From cd855469543648f887f0cfb98fa505971d1ef25f Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Wed, 10 Apr 2019 15:09:53 +0100 Subject: [PATCH 001/308] Add option type and null --- specs/simple-serialize.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index b78eff93e5..1f2c019490 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -30,6 +30,7 @@ This is a **work in progress** describing typing, serialization and Merkleizatio * `"uintN"`: `N`-bit unsigned integer (where `N in [8, 16, 32, 64, 128, 256]`) * `"bool"`: `True` or `False` +* '"null"': `Null` ### Composite types @@ -39,6 +40,8 @@ This is a **work in progress** describing typing, serialization and Merkleizatio * angle bracket notation `[type, N]`, e.g. `["uint64", N]` * **list**: ordered variable-length homogenous collection of values * angle bracket notation `[type]`, e.g. `["uint64"]` +* **option**: option type containing one of the given subtypes + * round bracket notation `(type1, type2, ...)`, e.g. `("uint64", "null")` We recursively define "variable-size" types to be lists and all types that contains a variable-size type. All other types are said to be "fixed-size". @@ -70,7 +73,13 @@ assert value in (True, False) return b"\x01" if value is True else b"\x00" ``` -### Vectors, containers, lists +### `"null"` + +```python +return b"" +``` + +### Vectors, containers, lists, options If `value` is fixed-size: @@ -87,6 +96,17 @@ serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, "lit return serialized_length + serialized_bytes ``` +If `value` is an option type: + +Define value as an object that has properties `value.value` with the contained value, and `value.type_index` which indexes the type. + +```python +serialized_bytes = serialize(value.value) +serialized_type_index = value.type_index.to_bytes(BYTES_PER_LENGTH_PREFIX, "little") +return serialized_length + serialized_bytes +``` + + ## Deserialization Because serialization is an injective function (i.e. two distinct objects of the same type will serialize to different values) any bytestring has at most one object it could deserialize to. Efficient algorithms for computing this object can be found in [the implementations](#implementations). @@ -98,6 +118,7 @@ We first define helper functions: * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. * `merkleize`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. * `mix_in_length`: Given a Merkle root `root` and a length `length` (`"uint256"` little-endian serialization) return `hash(root + length)`. +* `mix_in_type`: Given a Merkle root `root` and a type_index `type_index` (`"uint256"` little-endian serialization) return `hash(root + type_index)`. We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: @@ -105,6 +126,7 @@ We now define Merkleization `hash_tree_root(value)` of an object `value` recursi * `mix_in_length(merkleize(pack(value)), len(value))` if `value` is a list of basic objects * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container * `mix_in_length(merkleize([hash_tree_root(element) for element in value]), len(value))` if `value` is a list of composite objects +* `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of option type ## Self-signed containers From 283ba8f7611e3112f1466eac63c0fc2966f94c79 Mon Sep 17 00:00:00 2001 From: jannikluhn Date: Thu, 11 Apr 2019 16:00:53 +0100 Subject: [PATCH 002/308] Update specs/simple-serialize.md Co-Authored-By: dankrad --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 1f2c019490..5b302e2f52 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -103,7 +103,7 @@ Define value as an object that has properties `value.value` with the contained v ```python serialized_bytes = serialize(value.value) serialized_type_index = value.type_index.to_bytes(BYTES_PER_LENGTH_PREFIX, "little") -return serialized_length + serialized_bytes +return serialized_type_index + serialized_bytes ``` From 2017ce96149306721bfd8d18553f531594b937b5 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 11 Apr 2019 16:05:16 +0100 Subject: [PATCH 003/308] Rename "option" -> "union"; "null" only in unions --- specs/simple-serialize.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 5b302e2f52..8a8c784463 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -30,7 +30,9 @@ This is a **work in progress** describing typing, serialization and Merkleizatio * `"uintN"`: `N`-bit unsigned integer (where `N in [8, 16, 32, 64, 128, 256]`) * `"bool"`: `True` or `False` -* '"null"': `Null` +* `"null"`: `None` + +The type `"null"` is only legal as one of several type in a `union` type. ### Composite types @@ -40,7 +42,7 @@ This is a **work in progress** describing typing, serialization and Merkleizatio * angle bracket notation `[type, N]`, e.g. `["uint64", N]` * **list**: ordered variable-length homogenous collection of values * angle bracket notation `[type]`, e.g. `["uint64"]` -* **option**: option type containing one of the given subtypes +* **union**: union type containing one of the given subtypes * round bracket notation `(type1, type2, ...)`, e.g. `("uint64", "null")` We recursively define "variable-size" types to be lists and all types that contains a variable-size type. All other types are said to be "fixed-size". @@ -79,7 +81,7 @@ return b"\x01" if value is True else b"\x00" return b"" ``` -### Vectors, containers, lists, options +### Vectors, containers, lists, unions If `value` is fixed-size: @@ -96,7 +98,7 @@ serialized_length = len(serialized_bytes).to_bytes(BYTES_PER_LENGTH_PREFIX, "lit return serialized_length + serialized_bytes ``` -If `value` is an option type: +If `value` is an union type: Define value as an object that has properties `value.value` with the contained value, and `value.type_index` which indexes the type. @@ -126,7 +128,7 @@ We now define Merkleization `hash_tree_root(value)` of an object `value` recursi * `mix_in_length(merkleize(pack(value)), len(value))` if `value` is a list of basic objects * `merkleize([hash_tree_root(element) for element in value])` if `value` is a vector of composite objects or a container * `mix_in_length(merkleize([hash_tree_root(element) for element in value]), len(value))` if `value` is a list of composite objects -* `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of option type +* `mix_in_type(merkleize(value.value), value.type_index)` if `value` is of union type ## Self-signed containers From 22d44969f75af3a9dfc76aeafb584c4e40599247 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 16 Apr 2019 14:54:17 +1000 Subject: [PATCH 004/308] Add initial libp2p standardization --- specs/networking/libp2p-standardization.md | 131 +++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 specs/networking/libp2p-standardization.md diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md new file mode 100644 index 0000000000..ddefca02ab --- /dev/null +++ b/specs/networking/libp2p-standardization.md @@ -0,0 +1,131 @@ +ETH 2.0 Networking Spec - Libp2p standard protocols +=== + +# Abstract + +Ethereum 2.0 clients plan to use the libp2p protocol networking stack for +mainnet release. This document aims to standardize the libp2p client protocols, +configuration and messaging formats. + +# Libp2p Protocols + +## Gossipsub + +#### Protocol id: `/meshsub/1.0.0` + +*Note: Parameters listed here are subject to a large-scale network feasibility +study* + +The [Gossipsub](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) +protocol will be used for block and attestation propagation across the +network. + +### Configuration Parameters + +Gossipsub has a number of internal configuration parameters which directly +effect the network performance. Clients can implement independently, however +we aim to standardize these across clients to optimize the gossip network for +propagation times and message duplication. Current network-related defaults are: + +``` +( + // The target number of peers in the overlay mesh network (D in the libp2p specs). + mesh_size: 6 + // The minimum number of peers in the mesh network before adding more (D_lo in the libp2p specs). + mesh_lo: 4 + // The maximum number of peers in the mesh network before removing some (D_high in the libp2p sepcs). + mesh_high: 12 + // The number of peers to gossip to during a heartbeat (D_lazy in the libp2p sepcs). + gossip_lazy: 6 // defaults to `mesh_size` + // Time to live for fanout peers (seconds). + fanout_ttl: 60 + // The number of heartbeats to gossip about. + gossip_history: 3 + // Time between each heartbeat (seconds). + heartbeat_interval: 1 +) +``` + +### Topics + +*The Go and Js implementations use string topics - This is likely to be +updated to topic hashes in later versions - https://github.com/libp2p/rust-libp2p/issues/473* + +For Eth2.0 clients, topics will be sent as `SHA2-256` hashes of the topic string. + +There is one dedicated topic for propagating beacon blocks and aggregated +attestations across the network. This topic will have the string +`beacon_chain`. Each shard will have it's own topic allowing relevant parties +to subscribe to in order to receive local shard attestations. The shard topics are +prefixed with `shard` followed by the number of the shard. For example, +messages relating to shard 10, will have the topic string `shard10`. + +### Messages + +Messages sent across gossipsub are fixed-size length-prefixed byte arrays. +Each message has a maximum size of 512KB (estimated from expected largest uncompressed +block size). + +The byte array is prefixed with a unsigned 64 bit length number encoded as an +`unsigned varint` (https://github.com/multiformats/unsigned-varint). Gossipsub messages therefore take the form: +``` ++--------------------------+ +| message length | ++--------------------------+ +| | +| body (<1M) | +| | ++--------------------------+ +``` + +The body of the message is an SSZ-encoded object representing either a +beacon block or attestation. The type of objected is determined via a prefixed +nibble. Currently there are two objects that are sent across the gossip +network. They are (with their corresponding nibble specification): + +- `0x1`: Beacon block +- `0x2`: Attestation + +The body therefore takes the form: +``` ++--------------------------+ +| type nibble | ++--------------------------+ +| | +| SSZ-encoded object | +| | ++--------------------------+ +``` + +## Eth-2 RPC + +#### Protocol Id: `/eth/serenity/beacon/rpc/1` + +The [RPC Interface](./rpc-interface.md) is specified in this repository. + + +## Identify + +#### Protocol Id: `/ipfs/id/1.0.0` (to be updated to `/p2p/id/1.0.0`) + +The Identify protocol (defined in go - [identify-go](https://github.com/ipfs/go-ipfs/blob/master/core/commands/id.go) and rust [rust-identify](https://github.com/libp2p/rust-libp2p/blob/master/protocols/identify/src/lib.rs)) +allows a node A to query another node B which information B knows about A. This also includes the addresses B is listening on. + +This protocol allows nodes to discover addresses of other nodes to be added to +peer discovery. It further allows nodes to determine the capabilities of all it's connected +peers. + +### Configuration Parameters + +The protocol has two configurable parameters, which can be used to identify the +type of connecting node. Suggested format: +``` + version: `/eth/serenity/1.0.0` + user_agent: +``` + +## Discovery + +#### Protocol Id: `/eth/serenity/disc/1.0.0` + +The discovery protocol to be determined. From 63bdf95e79e2dcced7e2f5ba3945700972a5ef9d Mon Sep 17 00:00:00 2001 From: dankrad Date: Sat, 20 Apr 2019 11:46:31 +0100 Subject: [PATCH 005/308] Update simple-serialize.md --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 8a8c784463..c9f845873a 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -45,7 +45,7 @@ The type `"null"` is only legal as one of several type in a `union` type. * **union**: union type containing one of the given subtypes * round bracket notation `(type1, type2, ...)`, e.g. `("uint64", "null")` -We recursively define "variable-size" types to be lists and all types that contains a variable-size type. All other types are said to be "fixed-size". +We recursively define "variable-size" types to be lists and unions and all types that contain a variable-size type. All other types are said to be "fixed-size". ### Aliases From a481a4e96cf29d13dd19fef33172d0cefa390e04 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 20 Apr 2019 20:57:50 +1000 Subject: [PATCH 006/308] Update simple-serialize.md --- specs/simple-serialize.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index c9f845873a..2d9b715b3a 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -32,7 +32,7 @@ This is a **work in progress** describing typing, serialization and Merkleizatio * `"bool"`: `True` or `False` * `"null"`: `None` -The type `"null"` is only legal as one of several type in a `union` type. +The `"null"` type is only legal as a union sub-type. ### Composite types @@ -108,7 +108,6 @@ serialized_type_index = value.type_index.to_bytes(BYTES_PER_LENGTH_PREFIX, "litt return serialized_type_index + serialized_bytes ``` - ## Deserialization Because serialization is an injective function (i.e. two distinct objects of the same type will serialize to different values) any bytestring has at most one object it could deserialize to. Efficient algorithms for computing this object can be found in [the implementations](#implementations). From f7ef9a1ba5e7b863538ab48097060f98ec536412 Mon Sep 17 00:00:00 2001 From: Matthew Slipper Date: Sat, 20 Apr 2019 17:26:07 -0700 Subject: [PATCH 007/308] Don't use SSZ in RPC request/response wrappers --- specs/networking/rpc-interface.md | 42 +++++++++++++++++-------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/specs/networking/rpc-interface.md b/specs/networking/rpc-interface.md index 5d408b5a07..ca6008a406 100644 --- a/specs/networking/rpc-interface.md +++ b/specs/networking/rpc-interface.md @@ -39,32 +39,36 @@ To facilitate RPC-over-`libp2p`, a single protocol name is used: `/eth/serenity/ Remote method calls are wrapped in a "request" structure: ``` -( - id: uint64 - method_id: uint16 - body: Request -) + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ id (uint64) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| method_id (uint16) | body_len (uint32) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | body (body_len bytes) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ``` and their corresponding responses are wrapped in a "response" structure: ``` -( - id: uint64 - response_code: uint16 - result: bytes -) + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | ++ id (uint64) + +| | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| response_code (uint16) | result_len (uint32) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +| | result (result_len bytes) | ++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ``` -If an error occurs, a variant of the response structure is returned: - -``` -( - id: uint64 - response_code: uint16 - result: bytes -) -``` +Note that the above structures are NOT encoded as SSZ but rather as sequences of bytes according to the packet diagrams above. This is because SSZ does not support structures without an explicit schema. Since the `body` and `result` fields depend on the value of `method_id` and `response_code`, a schema for the above structure cannot be known beforehand. The details of the RPC-Over-`libp2p` protocol are similar to [JSON-RPC 2.0](https://www.jsonrpc.org/specification). Specifically: From b83a7c4a23c8b27df64e92efcd05eb201b3244e2 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 24 Apr 2019 16:44:22 +1000 Subject: [PATCH 008/308] Add @prestonvanloon and @djrtwo's comments for muliple beacon topics --- specs/networking/libp2p-standardization.md | 42 +++++++++------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index ddefca02ab..126ed53b4a 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -53,12 +53,20 @@ updated to topic hashes in later versions - https://github.com/libp2p/rust-libp2 For Eth2.0 clients, topics will be sent as `SHA2-256` hashes of the topic string. -There is one dedicated topic for propagating beacon blocks and aggregated -attestations across the network. This topic will have the string -`beacon_chain`. Each shard will have it's own topic allowing relevant parties -to subscribe to in order to receive local shard attestations. The shard topics are -prefixed with `shard` followed by the number of the shard. For example, -messages relating to shard 10, will have the topic string `shard10`. +There are two main topics used to propagate attestations and beacon blocks to +all nodes on the network. + +- The `beacon_block` topic - This topic is used solely for propagating new + beacon blocks to all nodes on the networks. +- The `beacon_attestation` topic - This topic is used to for propagate + aggregated attestations to subscribing nodes (typically block proposers) to + be included into future blocks. Attestations will be aggregated in their + respective subnets before publishing on this topic. + +Shards will be grouped into their own subnets (defined by a shard topic). The +number of shard subnets will be defined via `SHARD_SUBNET_COUNT` and the shard +`shard_number % SHARD_SUBNET_COUNT` will be assigned to the topic: +`shard{shard_number % SHARD_SUBNET_COUNT}`. ### Messages @@ -78,24 +86,9 @@ The byte array is prefixed with a unsigned 64 bit length number encoded as an +--------------------------+ ``` -The body of the message is an SSZ-encoded object representing either a -beacon block or attestation. The type of objected is determined via a prefixed -nibble. Currently there are two objects that are sent across the gossip -network. They are (with their corresponding nibble specification): - -- `0x1`: Beacon block -- `0x2`: Attestation - -The body therefore takes the form: -``` -+--------------------------+ -| type nibble | -+--------------------------+ -| | -| SSZ-encoded object | -| | -+--------------------------+ -``` +The body of the message is an SSZ-encoded object. For the `beacon_block` topic, +this will be a `beacon_block`. For the `beacon_attestation` topic, this will be +an `attestation`. ## Eth-2 RPC @@ -103,7 +96,6 @@ The body therefore takes the form: The [RPC Interface](./rpc-interface.md) is specified in this repository. - ## Identify #### Protocol Id: `/ipfs/id/1.0.0` (to be updated to `/p2p/id/1.0.0`) From 101449e71a3492ff97c649c706866a0da2ec6962 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Sat, 27 Apr 2019 21:00:50 +0100 Subject: [PATCH 009/308] Define null as alias of {} --- specs/simple-serialize.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index abef5c6696..6858561e8c 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -8,12 +8,13 @@ This is a **work in progress** describing typing, serialization and Merkleizatio - [Typing](#typing) - [Basic types](#basic-types) - [Composite types](#composite-types) + - [Illegal empty composites](#illegal-empty-composites) - [Aliases](#aliases) - [Default values](#default-values) - [Serialization](#serialization) - [`"uintN"`](#uintn) - [`"bool"`](#bool) - - [Containers, vectors, lists](#containers-vectors-lists) + - [Vectors, containers, lists, unions](#vectors-containers-lists-unions) - [Deserialization](#deserialization) - [Merkleization](#merkleization) - [Self-signed containers](#self-signed-containers) @@ -32,9 +33,6 @@ This is a **work in progress** describing typing, serialization and Merkleizatio * `"uintN"`: `N`-bit unsigned integer (where `N in [8, 16, 32, 64, 128, 256]`) * `"bool"`: `True` or `False` -* `"null"`: `None` - -The `"null"` type is only legal as a union sub-type. ### Composite types @@ -49,6 +47,10 @@ The `"null"` type is only legal as a union sub-type. We recursively define "variable-size" types to be lists and unions and all types that contain a variable-size type. All other types are said to be "fixed-size". +#### Illegal empty composites + +The empty container `{}` (except as the `"null"` type inside a union, see below) and the empty fixed length list `[type, 0]` are **not** legal types. + ### Aliases For convenience we alias: @@ -56,6 +58,9 @@ For convenience we alias: * `"byte"` to `"uint8"` (this is a basic type) * `"bytes"` to `["byte"]` (this is *not* a basic type) * `"bytesN"` to `["byte", N]` (this is *not* a basic type) +* `"null"`: `{}`, i.e. the empty container + +The `"null"` type is only legal as a union sub-type. ### Default values From 4a483309a544dcd24095e5bb9d5a05b832e97b3b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 1 May 2019 12:39:07 +0200 Subject: [PATCH 010/308] Update specs/simple-serialize.md Co-Authored-By: dankrad --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 6858561e8c..79ed0ef7b1 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -113,7 +113,7 @@ fixed_parts = [part if part != None else variable_offsets[i] for i, part in enum return b"".join(fixed_parts + variable_parts) ``` -If `value` is an union type: +If `value` is a union type: Define value as an object that has properties `value.value` with the contained value, and `value.type_index` which indexes the type. From d0447022cb4982e544abdea22e616e2cc7c44444 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 1 May 2019 12:39:24 +0200 Subject: [PATCH 011/308] Update specs/simple-serialize.md Co-Authored-By: dankrad --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 79ed0ef7b1..0e0468eb70 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -119,7 +119,7 @@ Define value as an object that has properties `value.value` with the contained v ```python serialized_bytes = serialize(value.value) -serialized_type_index = value.type_index.to_bytes(BYTES_PER_LENGTH_PREFIX, "little") +serialized_type_index = value.type_index.to_bytes(BYTES_PER_LENGTH_OFFSET, "little") return serialized_type_index + serialized_bytes ``` From cc22432bb91626d48ba62e786da72891f60424de Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 1 May 2019 12:39:44 +0200 Subject: [PATCH 012/308] Update specs/simple-serialize.md Co-Authored-By: dankrad --- specs/simple-serialize.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 0e0468eb70..f6b5cb232b 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -45,6 +45,8 @@ This is a **work in progress** describing typing, serialization and Merkleizatio * **union**: union type containing one of the given subtypes * round bracket notation `(type1, type2, ...)`, e.g. `("uint64", "null")` +#### Variable-size and fixed-size + We recursively define "variable-size" types to be lists and unions and all types that contain a variable-size type. All other types are said to be "fixed-size". #### Illegal empty composites From 8b316c6db4bcbd96416a1a12e76f57a626f03df6 Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 1 May 2019 12:04:27 +0100 Subject: [PATCH 013/308] Start moving state_transition.py to state transitition spec The state transition spec should be reasonably self-contained, limiting the amount of "magic" outside of it. This PR is a first step in this direction, specifically for operation processing. --- specs/core/0_beacon-chain.md | 42 ++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 55791e25f5..bfb51e99ac 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1691,11 +1691,27 @@ def process_eth1_data(state: BeaconState, block: BeaconBlock) -> None: #### Operations -##### Proposer slashings +The sub-sections below define helper functions, one per operation type. The full processing of operations is done by running `process_operations(state, block.body)`. + +```python +def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: + assert len(body.deposits) == min(MAX_DEPOSITS, state.latest_eth1_data.deposit_count - state.deposit_index) + assert len(body.transfers) == len(set(body.transfers)) -Verify that `len(block.body.proposer_slashings) <= MAX_PROPOSER_SLASHINGS`. + for operations, max_operations, function in { + (body.proposer_slashings, MAX_PROPOSER_SLASHINGS, process_proposer_slashing), + (body.attester_slashings, MAX_ATTESTER_SLASHINGS, attester_slashings), + (body.attestations, MAX_ATTESTATIONS, process_attestation), + (body.deposits, MAX_DEPOSITS, process_deposit), + (body.voluntary_exits, MAX_VOLUNTARY_EXITS, process_voluntary_exit), + (body.transfers, MAX_TRANSFERS, process_transfer), + }: + assert len(operations) <= max_operations + for operation in operations: + function(state, operation) +``` -For each `proposer_slashing` in `block.body.proposer_slashings`, run the following function: +##### Proposer slashings ```python def process_proposer_slashing(state: BeaconState, @@ -1721,10 +1737,6 @@ def process_proposer_slashing(state: BeaconState, ##### Attester slashings -Verify that `len(block.body.attester_slashings) <= MAX_ATTESTER_SLASHINGS`. - -For each `attester_slashing` in `block.body.attester_slashings`, run the following function: - ```python def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSlashing) -> None: @@ -1759,10 +1771,6 @@ def process_attester_slashing(state: BeaconState, ##### Attestations -Verify that `len(block.body.attestations) <= MAX_ATTESTATIONS`. - -For each `attestation` in `block.body.attestations`, run the following function: - ```python def process_attestation(state: BeaconState, attestation: Attestation) -> None: """ @@ -1801,10 +1809,6 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ##### Deposits -Verify that `len(block.body.deposits) == min(MAX_DEPOSITS, state.latest_eth1_data.deposit_count - state.deposit_index)`. - -For each `deposit` in `block.body.deposits`, run the following function: - ```python def process_deposit(state: BeaconState, deposit: Deposit) -> None: """ @@ -1851,10 +1855,6 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: ##### Voluntary exits -Verify that `len(block.body.voluntary_exits) <= MAX_VOLUNTARY_EXITS`. - -For each `exit` in `block.body.voluntary_exits`, run the following function: - ```python def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: """ @@ -1879,10 +1879,6 @@ def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: ##### Transfers -Verify that `len(block.body.transfers) <= MAX_TRANSFERS` and that all transfers are distinct. - -For each `transfer` in `block.body.transfers`, run the following function: - ```python def process_transfer(state: BeaconState, transfer: Transfer) -> None: """ From 591a2b47c8af28a43f5349435749d7f60f9894d4 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 1 May 2019 12:08:15 +0100 Subject: [PATCH 014/308] Simplify state_transition.py --- specs/core/0_beacon-chain.md | 6 +- .../eth2spec/phase0/state_transition.py | 69 +------------------ 2 files changed, 5 insertions(+), 70 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index bfb51e99ac..e57fce0374 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1698,14 +1698,14 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: assert len(body.deposits) == min(MAX_DEPOSITS, state.latest_eth1_data.deposit_count - state.deposit_index) assert len(body.transfers) == len(set(body.transfers)) - for operations, max_operations, function in { + for operations, max_operations, function in ( (body.proposer_slashings, MAX_PROPOSER_SLASHINGS, process_proposer_slashing), - (body.attester_slashings, MAX_ATTESTER_SLASHINGS, attester_slashings), + (body.attester_slashings, MAX_ATTESTER_SLASHINGS, process_attester_slashing), (body.attestations, MAX_ATTESTATIONS, process_attestation), (body.deposits, MAX_DEPOSITS, process_deposit), (body.voluntary_exits, MAX_VOLUNTARY_EXITS, process_voluntary_exit), (body.transfers, MAX_TRANSFERS, process_transfer), - }: + ): assert len(operations) <= max_operations for operation in operations: function(state, operation) diff --git a/test_libs/pyspec/eth2spec/phase0/state_transition.py b/test_libs/pyspec/eth2spec/phase0/state_transition.py index 1bef358d49..2aa0a38a6c 100644 --- a/test_libs/pyspec/eth2spec/phase0/state_transition.py +++ b/test_libs/pyspec/eth2spec/phase0/state_transition.py @@ -14,76 +14,11 @@ ) -def expected_deposit_count(state: BeaconState) -> int: - return min( - spec.MAX_DEPOSITS, - state.latest_eth1_data.deposit_count - state.deposit_index - ) - - -def process_operation_type(state: BeaconState, - operations: List[Any], - max_operations: int, - tx_fn: Callable[[BeaconState, Any], None]) -> None: - assert len(operations) <= max_operations - for operation in operations: - tx_fn(state, operation) - - -def process_operations(state: BeaconState, block: BeaconBlock) -> None: - process_operation_type( - state, - block.body.proposer_slashings, - spec.MAX_PROPOSER_SLASHINGS, - spec.process_proposer_slashing, - ) - - process_operation_type( - state, - block.body.attester_slashings, - spec.MAX_ATTESTER_SLASHINGS, - spec.process_attester_slashing, - ) - - process_operation_type( - state, - block.body.attestations, - spec.MAX_ATTESTATIONS, - spec.process_attestation, - ) - - assert len(block.body.deposits) == expected_deposit_count(state) - process_operation_type( - state, - block.body.deposits, - spec.MAX_DEPOSITS, - spec.process_deposit, - ) - - process_operation_type( - state, - block.body.voluntary_exits, - spec.MAX_VOLUNTARY_EXITS, - spec.process_voluntary_exit, - ) - - assert len(block.body.transfers) == len(set(block.body.transfers)) - process_operation_type( - state, - block.body.transfers, - spec.MAX_TRANSFERS, - spec.process_transfer, - ) - - -def process_block(state: BeaconState, - block: BeaconBlock, - verify_state_root: bool=False) -> None: +def process_block(state: BeaconState, block: BeaconBlock, verify_state_root) -> None: spec.process_block_header(state, block) spec.process_randao(state, block) spec.process_eth1_data(state, block) - - process_operations(state, block) + spec.process_operations(state, block.body) if verify_state_root: spec.verify_block_state_root(state, block) From 5df79d7565fd4fc49fe4367cb40afa7b121d5718 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 1 May 2019 13:14:10 +0100 Subject: [PATCH 015/308] Get rid of state_transition.py --- specs/core/0_beacon-chain.md | 79 ++++++++----------- .../eth2spec/phase0/state_transition.py | 47 ----------- .../test_process_attestation.py | 4 +- .../test_process_block_header.py | 2 +- .../test_process_crosslinks.py | 4 +- test_libs/pyspec/tests/helpers.py | 8 +- test_libs/pyspec/tests/test_finality.py | 5 +- test_libs/pyspec/tests/test_sanity.py | 6 +- 8 files changed, 43 insertions(+), 112 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/phase0/state_transition.py diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e57fce0374..50e390326c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -105,7 +105,6 @@ - [Registry updates](#registry-updates) - [Slashings](#slashings) - [Final updates](#final-updates) - - [Per-slot processing](#per-slot-processing) - [Per-block processing](#per-block-processing) - [Block header](#block-header) - [RANDAO](#randao) @@ -1264,27 +1263,27 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], ## Beacon chain state transition function -We now define the state transition function. At a high level, the state transition is made up of four parts: +The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. -1. State caching, which happens at the start of every slot. -2. The per-epoch transitions, which happens at the start of the first slot of every epoch. -3. The per-slot transitions, which happens at every slot. -4. The per-block transitions, which happens at every block. - -Transition section notes: -* The state caching caches the state root of the previous slot and updates block and state roots records. -* The per-epoch transitions focus on the [validator](#dfn-validator) registry, including adjusting balances and activating and exiting [validators](#dfn-validator), as well as processing crosslinks and managing block justification/finalization. -* The per-slot transitions focus on the slot counter. -* The per-block transitions generally focus on verifying aggregate signatures and saving temporary records relating to the per-block activity in the `BeaconState`. - -Beacon blocks that trigger unhandled Python exceptions (e.g. out-of-range list accesses) and failed `assert`s during the state transition are considered invalid. +```python +def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: + assert state.slot < block.slot + while state.slot < block.slot: + # State caching at the start of every slot + cache_state(state) + # Per-epoch processing at the start of the first slot of every epoch + if (state.slot + 1) % SLOTS_PER_EPOCH == 0: + process_epoch_transition(state) + # Slot incrementing + state.slot += 1 + # Block processing at every block + process_block(state, block) +``` -Note: If there are skipped slots between a block and its parent block, run the steps in the [state-root](#state-caching), [per-epoch](#per-epoch-processing), and [per-slot](#per-slot-processing) sections once for each skipped slot and then once for the slot containing the new block. +Note: Beacon blocks that trigger unhandled Python exceptions (e.g. out-of-range list accesses) and failed `assert`s during the state transition are considered invalid. ### State caching -At every `slot > GENESIS_SLOT` run the following function: - ```python def cache_state(state: BeaconState) -> None: # Cache latest known state root (for previous slot) @@ -1302,12 +1301,18 @@ def cache_state(state: BeaconState) -> None: ### Per-epoch processing -The steps below happen when `state.slot > GENESIS_SLOT and (state.slot + 1) % SLOTS_PER_EPOCH == 0`. +```python +def process_epoch_transition(state: BeaconState) -> None: + process_justification_and_finalization(state) + process_crosslinks(state) + process_rewards_and_penalties(state) + process_registry_updates(state) + process_slashings(state) + process_final_updates(state) +``` #### Helper functions -We define epoch transition helper functions: - ```python def get_total_active_balance(state: BeaconState) -> Gwei: return get_total_balance(state, get_active_validator_indices(state, get_current_epoch(state))) @@ -1387,8 +1392,6 @@ def get_earliest_attestation(state: BeaconState, attestations: List[PendingAttes #### Justification and finalization -Run the following function: - ```python def process_justification_and_finalization(state: BeaconState) -> None: if get_current_epoch(state) <= GENESIS_EPOCH + 1: @@ -1436,8 +1439,6 @@ def process_justification_and_finalization(state: BeaconState) -> None: #### Crosslinks -Run the following function: - ```python def process_crosslinks(state: BeaconState) -> None: state.previous_crosslinks = [c for c in state.current_crosslinks] @@ -1453,8 +1454,6 @@ def process_crosslinks(state: BeaconState) -> None: #### Rewards and penalties -First, we define additional helpers: - ```python def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: adjusted_quotient = integer_squareroot(get_total_active_balance(state)) // BASE_REWARD_QUOTIENT @@ -1526,8 +1525,6 @@ def get_crosslink_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: return [rewards, penalties] ``` -Run the following function: - ```python def process_rewards_and_penalties(state: BeaconState) -> None: if get_current_epoch(state) == GENESIS_EPOCH: @@ -1542,8 +1539,6 @@ def process_rewards_and_penalties(state: BeaconState) -> None: #### Registry updates -Run the following function: - ```python def process_registry_updates(state: BeaconState) -> None: # Process activation eligibility and ejections @@ -1568,8 +1563,6 @@ def process_registry_updates(state: BeaconState) -> None: #### Slashings -Run the following function: - ```python def process_slashings(state: BeaconState) -> None: current_epoch = get_current_epoch(state) @@ -1592,8 +1585,6 @@ def process_slashings(state: BeaconState) -> None: #### Final updates -Run the following function: - ```python def process_final_updates(state: BeaconState) -> None: current_epoch = get_current_epoch(state) @@ -1632,19 +1623,17 @@ def process_final_updates(state: BeaconState) -> None: state.current_epoch_attestations = [] ``` -### Per-slot processing - -At every `slot > GENESIS_SLOT` run the following function: +### Per-block processing ```python -def advance_slot(state: BeaconState) -> None: - state.slot += 1 +def process_block(state: BeaconState, block: BeaconBlock) -> None: + process_block_header(state, block) + process_randao(state, block) + process_eth1_data(state, block) + process_operations(state, block.body) + # verify_block_state_root(state, block) ``` -### Per-block processing - -For every `block` except the genesis block, run `process_block_header(state, block)`, `process_randao(state, block)` and `process_eth1_data(state, block)`. - #### Block header ```python @@ -1691,8 +1680,6 @@ def process_eth1_data(state: BeaconState, block: BeaconBlock) -> None: #### Operations -The sub-sections below define helper functions, one per operation type. The full processing of operations is done by running `process_operations(state, block.body)`. - ```python def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: assert len(body.deposits) == min(MAX_DEPOSITS, state.latest_eth1_data.deposit_count - state.deposit_index) @@ -1913,8 +1900,6 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: #### State root verification -Verify the block's `state_root` by running the following function: - ```python def verify_block_state_root(state: BeaconState, block: BeaconBlock) -> None: assert block.state_root == hash_tree_root(state) diff --git a/test_libs/pyspec/eth2spec/phase0/state_transition.py b/test_libs/pyspec/eth2spec/phase0/state_transition.py deleted file mode 100644 index 2aa0a38a6c..0000000000 --- a/test_libs/pyspec/eth2spec/phase0/state_transition.py +++ /dev/null @@ -1,47 +0,0 @@ -from . import spec - - -from typing import ( - Any, - Callable, - List -) - -from .spec import ( - BeaconState, - BeaconBlock, - Slot, -) - - -def process_block(state: BeaconState, block: BeaconBlock, verify_state_root) -> None: - spec.process_block_header(state, block) - spec.process_randao(state, block) - spec.process_eth1_data(state, block) - spec.process_operations(state, block.body) - if verify_state_root: - spec.verify_block_state_root(state, block) - - -def process_epoch_transition(state: BeaconState) -> None: - spec.process_justification_and_finalization(state) - spec.process_crosslinks(state) - spec.process_rewards_and_penalties(state) - spec.process_registry_updates(state) - spec.process_slashings(state) - spec.process_final_updates(state) - - -def state_transition_to(state: BeaconState, up_to: Slot) -> BeaconState: - while state.slot < up_to: - spec.cache_state(state) - if (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0: - process_epoch_transition(state) - spec.advance_slot(state) - - -def state_transition(state: BeaconState, - block: BeaconBlock, - verify_state_root: bool=False) -> BeaconState: - state_transition_to(state, block.slot) - process_block(state, block, verify_state_root) diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index 1be60c860b..105d1e0a41 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -3,13 +3,11 @@ import eth2spec.phase0.spec as spec -from eth2spec.phase0.state_transition import ( - state_transition, -) from eth2spec.phase0.spec import ( get_current_epoch, process_attestation, slot_to_epoch, + state_transition, ) from tests.helpers import ( build_empty_block_for_next_slot, diff --git a/test_libs/pyspec/tests/block_processing/test_process_block_header.py b/test_libs/pyspec/tests/block_processing/test_process_block_header.py index b35b0a9c11..6fd6e674e3 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/tests/block_processing/test_process_block_header.py @@ -5,10 +5,10 @@ from eth2spec.phase0.spec import ( get_beacon_proposer_index, cache_state, - advance_slot, process_block_header, ) from tests.helpers import ( + advance_slot, build_empty_block_for_next_slot, next_slot, ) diff --git a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py index fe694724a6..60e0dec535 100644 --- a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py @@ -3,13 +3,11 @@ import eth2spec.phase0.spec as spec -from eth2spec.phase0.state_transition import ( - state_transition, -) from eth2spec.phase0.spec import ( cache_state, get_crosslink_deltas, process_crosslinks, + state_transition, ) from tests.helpers import ( add_attestation_to_state, diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 63e4cd710f..2cb272ff3e 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -2,9 +2,6 @@ from py_ecc import bls -from eth2spec.phase0.state_transition import ( - state_transition, -) import eth2spec.phase0.spec as spec from eth2spec.utils.minimal_ssz import signing_root from eth2spec.phase0.spec import ( @@ -38,6 +35,7 @@ get_shard_delta, hash_tree_root, slot_to_epoch, + state_transition, verify_merkle_branch, hash, ) @@ -53,6 +51,10 @@ pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} +def advance_slot(state) -> None: + state.slot += 1 + + def get_balance(state, index): return state.balances[index] diff --git a/test_libs/pyspec/tests/test_finality.py b/test_libs/pyspec/tests/test_finality.py index ca048c2b2a..816dfd6bd2 100644 --- a/test_libs/pyspec/tests/test_finality.py +++ b/test_libs/pyspec/tests/test_finality.py @@ -4,9 +4,6 @@ import eth2spec.phase0.spec as spec -from eth2spec.phase0.state_transition import ( - state_transition, -) from .helpers import ( build_empty_block_for_next_slot, fill_aggregate_attestation, @@ -67,7 +64,7 @@ def next_epoch_with_attestations(state, fill_aggregate_attestation(post_state, prev_attestation) block.body.attestations.append(prev_attestation) - state_transition(post_state, block) + spec.state_transition(post_state, block) blocks.append(block) return state, blocks, post_state diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index b7d31f122a..4d826fc130 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -20,13 +20,10 @@ get_state_root, get_current_epoch, get_domain, - advance_slot, cache_state, verify_merkle_branch, - hash, -) -from eth2spec.phase0.state_transition import ( state_transition, + hash, ) from eth2spec.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, @@ -34,6 +31,7 @@ get_merkle_root, ) from .helpers import ( + advance_slot, get_balance, build_deposit_data, build_empty_block_for_next_slot, From 2e63a9b5a0322e0e87f72baee3a279999dbdddc7 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 1 May 2019 13:29:03 +0100 Subject: [PATCH 016/308] clean up --- specs/core/0_beacon-chain.md | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 50e390326c..8f69213bfe 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -97,7 +97,7 @@ - [On genesis](#on-genesis) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - [State caching](#state-caching) - - [Per-epoch processing](#per-epoch-processing) + - [Epoch processing](#epoch-processing) - [Helper functions](#helper-functions-1) - [Justification and finalization](#justification-and-finalization) - [Crosslinks](#crosslinks) @@ -105,7 +105,7 @@ - [Registry updates](#registry-updates) - [Slashings](#slashings) - [Final updates](#final-updates) - - [Per-block processing](#per-block-processing) + - [Block processing](#block-processing) - [Block header](#block-header) - [RANDAO](#randao) - [Eth1 data](#eth1-data) @@ -1263,7 +1263,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], ## Beacon chain state transition function -The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. +The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled exception (e.g. failed `assert`s and out-of-range list accesses) are considered invalid. ```python def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: @@ -1271,17 +1271,15 @@ def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: while state.slot < block.slot: # State caching at the start of every slot cache_state(state) - # Per-epoch processing at the start of the first slot of every epoch + # Epoch processing at the start of the first slot of every epoch if (state.slot + 1) % SLOTS_PER_EPOCH == 0: - process_epoch_transition(state) + process_epoch(state) # Slot incrementing state.slot += 1 # Block processing at every block process_block(state, block) ``` -Note: Beacon blocks that trigger unhandled Python exceptions (e.g. out-of-range list accesses) and failed `assert`s during the state transition are considered invalid. - ### State caching ```python @@ -1299,10 +1297,10 @@ def cache_state(state: BeaconState) -> None: state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = latest_block_root ``` -### Per-epoch processing +### Epoch processing ```python -def process_epoch_transition(state: BeaconState) -> None: +def process_epoch(state: BeaconState) -> None: process_justification_and_finalization(state) process_crosslinks(state) process_rewards_and_penalties(state) @@ -1623,7 +1621,7 @@ def process_final_updates(state: BeaconState) -> None: state.current_epoch_attestations = [] ``` -### Per-block processing +### Block processing ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: From a33ee00239f0ecd1f8bb557372844250431d0f68 Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 1 May 2019 13:52:37 +0100 Subject: [PATCH 017/308] Update simple-serialize.md --- specs/simple-serialize.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index f6b5cb232b..b7a1375e4f 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -8,9 +8,10 @@ This is a **work in progress** describing typing, serialization and Merkleizatio - [Typing](#typing) - [Basic types](#basic-types) - [Composite types](#composite-types) - - [Illegal empty composites](#illegal-empty-composites) + - [Variable-size and fixed-size](#variable-size-and-fixed-size) - [Aliases](#aliases) - [Default values](#default-values) + - [Illegal types](#illegal-types) - [Serialization](#serialization) - [`"uintN"`](#uintn) - [`"bool"`](#bool) @@ -45,14 +46,10 @@ This is a **work in progress** describing typing, serialization and Merkleizatio * **union**: union type containing one of the given subtypes * round bracket notation `(type1, type2, ...)`, e.g. `("uint64", "null")` -#### Variable-size and fixed-size +### Variable-size and fixed-size We recursively define "variable-size" types to be lists and unions and all types that contain a variable-size type. All other types are said to be "fixed-size". -#### Illegal empty composites - -The empty container `{}` (except as the `"null"` type inside a union, see below) and the empty fixed length list `[type, 0]` are **not** legal types. - ### Aliases For convenience we alias: @@ -62,19 +59,20 @@ For convenience we alias: * `"bytesN"` to `["byte", N]` (this is *not* a basic type) * `"null"`: `{}`, i.e. the empty container -The `"null"` type is only legal as a union sub-type. - ### Default values The default value of a type upon initialization is recursively defined using `0` for `"uintN"`, `False` for `"bool"`, and `[]` for lists. +### Illegal types + +Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `"null"` type is only legal as a union subtype. + ## Serialization We recursively define the `serialize` function which consumes an object `value` (of the type specified) and returns a bytestring of type `"bytes"`. > *Note*: In the function definitions below (`serialize`, `hash_tree_root`, `signing_root`, `is_variable_size`, etc.) objects implicitly carry their type. - ### `"uintN"` ```python From 563df146b9073da8f91b6ad200cd19273ea8f2ac Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 1 May 2019 13:55:02 +0100 Subject: [PATCH 018/308] Update simple-serialize.md --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index b7a1375e4f..21a87a6f9e 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -44,7 +44,7 @@ This is a **work in progress** describing typing, serialization and Merkleizatio * **list**: ordered variable-length homogeneous collection of values * angle bracket notation `[type]`, e.g. `["uint64"]` * **union**: union type containing one of the given subtypes - * round bracket notation `(type1, type2, ...)`, e.g. `("uint64", "null")` + * round bracket notation `(type_1, type_2, ...)`, e.g. `("uint64", "null")` ### Variable-size and fixed-size From 9b24d06b2cbfa494a8169eebc3f97f32b9ab3399 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 1 May 2019 14:16:55 +0100 Subject: [PATCH 019/308] Cleanup --- specs/core/0_beacon-chain.md | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8f69213bfe..153e42cf67 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1263,18 +1263,19 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], ## Beacon chain state transition function -The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled exception (e.g. failed `assert`s and out-of-range list accesses) are considered invalid. +The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. ```python def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: assert state.slot < block.slot + # Slot processing (including slots with no blocks) while state.slot < block.slot: # State caching at the start of every slot cache_state(state) # Epoch processing at the start of the first slot of every epoch if (state.slot + 1) % SLOTS_PER_EPOCH == 0: process_epoch(state) - # Slot incrementing + # Increment slot number state.slot += 1 # Block processing at every block process_block(state, block) @@ -1284,17 +1285,15 @@ def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: ```python def cache_state(state: BeaconState) -> None: - # Cache latest known state root (for previous slot) - latest_state_root = hash_tree_root(state) - state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = latest_state_root + # Cache state and block roots of previous slot + previous_state_root = hash_tree_root(state) + state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root + previous_block_root = signing_root(state.latest_block_header) + state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root - # Store latest known state root (for previous slot) in latest_block_header if it is empty + # Cache previous state root in latest_block_header, if empty if state.latest_block_header.state_root == ZERO_HASH: - state.latest_block_header.state_root = latest_state_root - - # Cache latest known block root (for previous slot) - latest_block_root = signing_root(state.latest_block_header) - state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = latest_block_root + state.latest_block_header.state_root = previous_state_root ``` ### Epoch processing From 7980cf3ef79ee4f7ee418de5c8169907280169a2 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 1 May 2019 15:07:55 +0100 Subject: [PATCH 020/308] Fix --- specs/core/0_beacon-chain.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 153e42cf67..4c1498ccfb 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1285,15 +1285,18 @@ def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: ```python def cache_state(state: BeaconState) -> None: - # Cache state and block roots of previous slot + # Cache state root of previous slot previous_state_root = hash_tree_root(state) state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root - previous_block_root = signing_root(state.latest_block_header) - state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root # Cache previous state root in latest_block_header, if empty if state.latest_block_header.state_root == ZERO_HASH: state.latest_block_header.state_root = previous_state_root + + # Cache block root of previous slot + previous_block_root = signing_root(state.latest_block_header) + state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root + ``` ### Epoch processing From b1930d22394911c9549f479adb92f62ec41b0958 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Wed, 1 May 2019 15:12:49 +0100 Subject: [PATCH 021/308] Union default values --- specs/simple-serialize.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 21a87a6f9e..6adc9c4b84 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -61,11 +61,11 @@ For convenience we alias: ### Default values -The default value of a type upon initialization is recursively defined using `0` for `"uintN"`, `False` for `"bool"`, and `[]` for lists. +The default value of a type upon initialization is recursively defined using `0` for `"uintN"`, `False` for `"bool"`, and `[]` for lists. Unions default to the first type in the union (with type index zero), which is `"null"` if present in the union. ### Illegal types -Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `"null"` type is only legal as a union subtype. +Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `"null"` type is only legal as the first type in a union subtype (i.e., with type index zero). ## Serialization From faf36e056f223927564d2fbf327926ddc455ce59 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Wed, 1 May 2019 16:10:01 -0700 Subject: [PATCH 022/308] Add notes to compute fork choice at genesis --- specs/core/0_fork-choice.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 0b19f1559e..cb11ea8606 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -13,6 +13,8 @@ - [Time parameters](#time-parameters) - [Beacon chain processing](#beacon-chain-processing) - [Beacon chain fork choice rule](#beacon-chain-fork-choice-rule) + - [Implementation notes](#implementation-notes) + - [Justification and finality at genesis](#justification-and-finality-at-genesis) @@ -99,3 +101,9 @@ def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) # Ties broken by favoring block with lexicographically higher root head = max(children, key=lambda x: (get_vote_count(x), hash_tree_root(x))) ``` + +## Implementation notes + +### Justification and finality at genesis + +Clients may choose to refer to the justification and finality data in a given `BeaconState` to determine the finalized/justified head. During the early epochs right after genesis, the justification and finality data are not stored in the `BeaconState`. In their place are "empty" values; for example, the 32-byte zero value as the `finalized_root`. Clients wishing to compute the fork choice in these early epochs should work around this fact of the `BeaconState` to recognize that the genesis epoch and root of the genesis block are _both_ the finalized and justified heads until updated via the state transition function defined in [Phase 0 -- The Beacon Chain](./0_beacon-chain.md). Solutions will be language-specific but one possibility is to treat the zero-value hash as an exceptional case that references the genesis block/epoch. From d28e5e4843ae39fba177e8c8744bd248cd938c15 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Wed, 1 May 2019 16:26:18 -0700 Subject: [PATCH 023/308] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 55791e25f5..68e7ff4840 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1151,6 +1151,9 @@ def get_delayed_activation_exit_epoch(epoch: Epoch) -> Epoch: ```python def get_churn_limit(state: BeaconState) -> int: + """ + Return the churn limit based on the active validator count. + """ return max( MIN_PER_EPOCH_CHURN_LIMIT, len(get_active_validator_indices(state, get_current_epoch(state))) // CHURN_LIMIT_QUOTIENT From 68488a34d0bb5d47f3b4324ee3858d852f4505dd Mon Sep 17 00:00:00 2001 From: terence tsao Date: Wed, 1 May 2019 16:33:21 -0700 Subject: [PATCH 024/308] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ad7204f214..27ff7fcb1e 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Core specifications for eth2.0 client validation can be found in [specs/core](sp * [The Beacon Chain](specs/core/0_beacon-chain.md) * [Fork Choice](specs/core/0_fork-choice.md) * [Deposit Contract](specs/core/0_deposit-contract.md) -* [Honest validator implementation doc](specs/validator/0_beacon-chain-validator.md) +* [Honest Validator](specs/validator/0_beacon-chain-validator.md) ### Phase 1 * [Custody Game](specs/core/1_custody-game.md) From bbca108a80effe0e167132f88dd1a6899bb86c70 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 2 May 2019 16:34:47 +1000 Subject: [PATCH 025/308] Add Transport and lower-level libp2p specifications --- specs/networking/libp2p-standardization.md | 79 ++++++++++++++++++++-- 1 file changed, 74 insertions(+), 5 deletions(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index 126ed53b4a..c5add549a9 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -9,6 +9,57 @@ configuration and messaging formats. # Libp2p Protocols +## Transport + +This section details the libp2p transport layer that underlies the +[protocols](#protocols) that are listed in this document. + +Libp2p allows composition of multiple transports. Eth2.0 clients should support +TCP/IP and optionally websockets. Websockets are useful for implementations +running in the browser and therefore native clients would ideally support these implementations +by supporting websockets. + +An ideal libp2p transport would therefore be TCP/IP with a fallback to +websockets. + +### Encryption + +Libp2p currently offers [Secio](https://github.com/libp2p/specs/pull/106) which +can upgrade a transport which will then encrypt all future communication. Secio +generates a symmetric ephemeral key which peers use to encrypt their +communication. It can support a range of ciphers and currently supports key +derivation for elliptic curve-based public keys. + +Current defaults are: +- Key agreement: `ECDH-P256` (also supports `ECDH-P384`) +- Cipher: `AES-128` (also supports `AES-256`, `TwofishCTR`) +- Digests: `SHA256` (also supports `SHA512`) + + +## Protocols + +This section lists the necessary libp2p protocols required by Ethereum 2.0 +running a libp2p network stack. + +## Multistream-select + +#### Protocol id: `/multistream/1.0.0` + +Clients running libp2p should support the [multistream-select](https://github.com/multiformats/multistream-select/) +protocol which allows clients to negotiate libp2p protocols establish streams +per protocol. + +## Multiplexing + +Libp2p allows clients to compose multiple multiplexing methods. Clients should +support [mplex](https://github.com/libp2p/specs/tree/master/mplex) and +optionally [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md) +(these can be composed). + +**Mplex protocol id: `/mplex/6.7.0`** + +**Yamux protocol id: `/yamux/1.0.0`** + ## Gossipsub #### Protocol id: `/meshsub/1.0.0` @@ -70,11 +121,14 @@ number of shard subnets will be defined via `SHARD_SUBNET_COUNT` and the shard ### Messages -Messages sent across gossipsub are fixed-size length-prefixed byte arrays. -Each message has a maximum size of 512KB (estimated from expected largest uncompressed -block size). +#### Libp2p Specification -The byte array is prefixed with a unsigned 64 bit length number encoded as an +*This section simply outlines the data sent across the wire as specified by +libp2p - this section is aimed at gossipsub implementers to standardize their implementation of this protocol* + +Libp2p raw gossipsub messages are sent across the wire as fixed-size length-prefixed byte arrays. + +The byte array is prefixed with an unsigned 64 bit length number encoded as an `unsigned varint` (https://github.com/multiformats/unsigned-varint). Gossipsub messages therefore take the form: ``` +--------------------------+ @@ -86,7 +140,17 @@ The byte array is prefixed with a unsigned 64 bit length number encoded as an +--------------------------+ ``` -The body of the message is an SSZ-encoded object. For the `beacon_block` topic, +The body represents a protobuf-encoded [Message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24). + +In the following section we discuss the data being sent in the `data` field of +the protobuf gossipsub `Message`. + +#### Eth2.0 Specifics + +Each message has a maximum size of 512KB (estimated from expected largest uncompressed +block size). + +The `data` that is sent in a Gossipsub message is an SSZ-encoded object. For the `beacon_block` topic, this will be a `beacon_block`. For the `beacon_attestation` topic, this will be an `attestation`. @@ -100,6 +164,9 @@ The [RPC Interface](./rpc-interface.md) is specified in this repository. #### Protocol Id: `/ipfs/id/1.0.0` (to be updated to `/p2p/id/1.0.0`) +*To be updated to incorporate discv5* + + The Identify protocol (defined in go - [identify-go](https://github.com/ipfs/go-ipfs/blob/master/core/commands/id.go) and rust [rust-identify](https://github.com/libp2p/rust-libp2p/blob/master/protocols/identify/src/lib.rs)) allows a node A to query another node B which information B knows about A. This also includes the addresses B is listening on. @@ -120,4 +187,6 @@ type of connecting node. Suggested format: #### Protocol Id: `/eth/serenity/disc/1.0.0` +*To be updated to incorporate discv5* + The discovery protocol to be determined. From 78181834ab6dbbfdc58a295d71d1309ba5829957 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 2 May 2019 16:38:23 +1000 Subject: [PATCH 026/308] Update specs/networking/libp2p-standardization.md Co-Authored-By: AgeManning --- specs/networking/libp2p-standardization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index c5add549a9..5fe26f435a 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -109,7 +109,7 @@ all nodes on the network. - The `beacon_block` topic - This topic is used solely for propagating new beacon blocks to all nodes on the networks. -- The `beacon_attestation` topic - This topic is used to for propagate +- The `beacon_attestation` topic - This topic is used to propagate aggregated attestations to subscribing nodes (typically block proposers) to be included into future blocks. Attestations will be aggregated in their respective subnets before publishing on this topic. From c761fbc3181fcf94ef9201a8174806bacd859caa Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 2 May 2019 09:24:24 +0100 Subject: [PATCH 027/308] Clean up verify_indexed_attestation Cosmetic changes: * Add 4 lines of comments (now every statement has a comment) * Avoid unnecessary `assert` (the end goal for me is for `assert`s to be exclusive to the operation processing helpers). * Merge `return`s into one (increase readability, reduce verbosity) * Use shorter-named `bit_0_indices` and `bit_1_indices` helper variables Substantive change: * Remove the condition that `len(0_indices) + len(1_indices) > 0`. This condition is redundant in the context of `process_attester_slashing` because of `slashed_any`. It is largely artificial in `process_attestation` where validators are incentivised to maximise new attestations. --- specs/core/0_beacon-chain.md | 54 +++++++++++++++++------------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 2a0b0c11d7..963c2ac444 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1018,37 +1018,33 @@ def convert_to_indexed(state: BeaconState, attestation: Attestation) -> IndexedA ```python def verify_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: """ - Verify validity of ``indexed_attestation`` fields. + Verify validity of ``indexed_attestation``. """ - custody_bit_0_indices = indexed_attestation.custody_bit_0_indices - custody_bit_1_indices = indexed_attestation.custody_bit_1_indices + bit_0_indices = indexed_attestation.custody_bit_0_indices + bit_1_indices = indexed_attestation.custody_bit_1_indices - # Ensure no duplicate indices across custody bits - assert len(set(custody_bit_0_indices).intersection(set(custody_bit_1_indices))) == 0 - - if len(custody_bit_1_indices) > 0: # [TO BE REMOVED IN PHASE 1] - return False - - if not (1 <= len(custody_bit_0_indices) + len(custody_bit_1_indices) <= MAX_INDICES_PER_ATTESTATION): - return False - - if custody_bit_0_indices != sorted(custody_bit_0_indices): - return False - - if custody_bit_1_indices != sorted(custody_bit_1_indices): - return False - - return bls_verify_multiple( - pubkeys=[ - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_0_indices]), - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in custody_bit_1_indices]), - ], - message_hashes=[ - hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b0)), - hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), - ], - signature=indexed_attestation.signature, - domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target_epoch), + return ( + # Verify no index has custody bit equal to 1 [to be removed in phase 1] + len(bit_1_indices) == 0 and + # Verify max number of indices + len(bit_0_indices) + len(bit_1_indices) <= MAX_INDICES_PER_ATTESTATION and + # Verify index sets are disjoint + len(set(bit_0_indices).intersection(bit_1_indices)) == 0 and + # Verify indices are sorted + bit_0_indices == sorted(bit_0_indices) and bit_1_indices == sorted(bit_1_indices) and + # Verify aggregate signature + bls_verify_multiple( + pubkeys=[ + bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in bit_0_indices]), + bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in bit_1_indices]), + ], + message_hashes=[ + hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b0)), + hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), + ], + signature=indexed_attestation.signature, + domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target_epoch), + ) ) ``` From 973f07223537e33dd05587b895143eb45ee6e3d0 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Thu, 2 May 2019 09:25:29 +0100 Subject: [PATCH 028/308] Remove unnecessary test --- .../block_processing/test_process_attestation.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index bcf71376ce..165f0c84a9 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -142,14 +142,3 @@ def test_non_empty_custody_bitfield(state): pre_state, post_state = run_attestation_processing(state, attestation, False) return pre_state, attestation, post_state - - -def test_empty_aggregation_bitfield(state): - attestation = get_valid_attestation(state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - attestation.aggregation_bitfield = b'\x00' * len(attestation.aggregation_bitfield) - - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state From 9f2cdd9c7807265edcb259dd91d37a9ca51988ca Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 2 May 2019 11:07:25 +0100 Subject: [PATCH 029/308] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4c1498ccfb..86caf7402d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1266,19 +1266,22 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. ```python -def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: +def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconBlock: + # Block must post-date the state assert state.slot < block.slot - # Slot processing (including slots with no blocks) + # Process slots (including those with no blocks) since block while state.slot < block.slot: - # State caching at the start of every slot + # Cache state at the start of every slot cache_state(state) - # Epoch processing at the start of the first slot of every epoch + # Process epoch at the start of the first slot of every epoch if (state.slot + 1) % SLOTS_PER_EPOCH == 0: process_epoch(state) # Increment slot number state.slot += 1 - # Block processing at every block + # Process block process_block(state, block) + # Return post-state + return state ``` ### State caching From 5d688ddb9d5c156762ca2482c201374d3607e58e Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 2 May 2019 20:45:39 -0700 Subject: [PATCH 030/308] Update property name --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 82f740f7db..10a228e508 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -181,7 +181,7 @@ epoch_signature = bls_sign( `block.eth1_data` is a mechanism used by block proposers vote on a recent Ethereum 1.0 block hash and an associated deposit root found in the Ethereum 1.0 deposit contract. When consensus is formed, `state.latest_eth1_data` is updated, and validator deposits up to this root can be processed. The deposit root can be calculated by calling the `get_deposit_root()` function of the deposit contract using the post-state of the block hash. * Let `D` be the set of `Eth1DataVote` objects `vote` in `state.eth1_data_votes` where: - * `vote.eth1_data.block_hash` is the hash of an eth1.0 block that is (i) part of the canonical chain, (ii) >= `ETH1_FOLLOW_DISTANCE` blocks behind the head, and (iii) newer than `state.latest_eth1_data.block_data`. + * `vote.eth1_data.block_hash` is the hash of an eth1.0 block that is (i) part of the canonical chain, (ii) >= `ETH1_FOLLOW_DISTANCE` blocks behind the head, and (iii) newer than `state.latest_eth1_data.block_hash`. * `vote.eth1_data.deposit_count` is the deposit count of the eth1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. * `vote.eth1_data.deposit_root` is the deposit root of the eth1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. * If `D` is empty: From 43e85f76d99b3d6c3e0becd7364c14e4ecb4f80c Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 2 May 2019 21:05:51 -0700 Subject: [PATCH 031/308] Update name of constant max deposit amount is now in spirit named max effective balance --- specs/validator/0_beacon-chain-validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 82f740f7db..0b23d9f46a 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -101,12 +101,12 @@ In phase 0, all incoming validator deposits originate from the Ethereum 1.0 PoW To submit a deposit: * Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](../core/0_beacon-chain.md#depositdata) SSZ object. -* Let `amount` be the amount in Gwei to be deposited by the validator where `MIN_DEPOSIT_AMOUNT <= amount <= MAX_DEPOSIT_AMOUNT`. +* Let `amount` be the amount in Gwei to be deposited by the validator where `MIN_DEPOSIT_AMOUNT <= amount <= MAX_EFFECTIVE_BALANCE`. * Set `deposit_data.amount = amount`. * Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=DOMAIN_DEPOSIT`. * Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])` along with a deposit of `amount` Gwei. -_Note_: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validator_registry` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_DEPOSIT_AMOUNT`. +_Note_: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validator_registry` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. ### Process deposit From 4ca2f1182732d2dc88e3edf8fe60383e3dbc681d Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 3 May 2019 17:20:54 +0800 Subject: [PATCH 032/308] Custody game changes (#866) * Custody game changes 1. Don't store the full chunk bits, instead only store a Merkle root. Increased history size complexity from `N` to `N + log(N)` but with the benefit of decreasing storage requirements from `N` to a single 32 byte hash. 2. `custody_bit` is computed as the first bit of the hash of the custody bits, not the xor. This allows us to more safely use functions with more risky security assumptions for computing the chunk mix. * Update specs/core/1_custody-game.md * Update specs/core/1_custody-game.md * Update specs/core/1_custody-game.md * Update specs/core/1_custody-game.md * XOR aggregation before SHA256 to reduce number of hashes * Simplifed get_chunk_bits_root * standalone -> indexed * Fix missing "data" and ToC --- specs/core/1_custody-game.md | 55 +++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 9f1c0e66a3..0f1555e625 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -32,6 +32,7 @@ - [`empty`](#empty) - [`get_crosslink_chunk_count`](#get_crosslink_chunk_count) - [`get_custody_chunk_bit`](#get_custody_chunk_bit) + - [`get_chunk_bits_root`](#get_chunk_bits_root) - [`epoch_to_custody_period`](#epoch_to_custody_period) - [`replace_empty_or_append`](#replace_empty_or_append) - [`verify_custody_key`](#verify_custody_key) @@ -148,7 +149,8 @@ This document details the beacon chain additions and changes in Phase 1 of Ether 'responder_index': ValidatorIndex, 'deadline': Epoch, 'crosslink_data_root': Hash, - 'chunk_bits': Bitfield, + 'chunk_count': 'uint64', + 'chunk_bits_merkle_root': Hash, 'responder_key': BLSSignature, } ``` @@ -160,7 +162,9 @@ This document details the beacon chain additions and changes in Phase 1 of Ether 'challenge_index': 'uint64', 'chunk_index': 'uint64', 'chunk': ['byte', BYTES_PER_CUSTODY_CHUNK], - 'branch': [Hash], + 'data_branch': [Hash], + 'chunk_bits_branch': [Hash], + 'chunk_bits_leaf': Hash, } ``` @@ -233,6 +237,17 @@ def get_custody_chunk_bit(key: BLSSignature, chunk: bytes) -> bool: return get_bitfield_bit(hash(challenge.responder_key + chunk), 0) ``` +### `get_chunk_bits_root` + +```python +def get_chunk_bits_root(chunk_bitfield: Bitfield) -> Bytes32: + aggregated_bits = bytearray([0] * 32) + for i in range(0, len(chunk_bitfield), 32): + for j in range(32): + aggregated_bits[j] ^= chunk_bitfield[i+j] + return hash(aggregated_bits) +``` + ### `epoch_to_custody_period` ```python @@ -326,7 +341,7 @@ For each `challenge` in `block.body.custody_chunk_challenges`, run the following def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge) -> None: # Verify the attestation - assert verify_standalone_attestation(state, convert_to_standalone(state, challenge.attestation)) + assert verify_indexed_attestation(state, convert_to_indexed(state, challenge.attestation)) # Verify it is not too late to challenge assert slot_to_epoch(challenge.attestation.data.slot) >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY responder = state.validator_registry[challenge.responder_index] @@ -380,7 +395,7 @@ def process_bit_challenge(state: BeaconState, # Verify the challenger is not slashed assert challenger.slashed is False # Verify the attestation - assert verify_standalone_attestation(state, convert_to_standalone(state, challenge.attestation)) + assert verify_indexed_attestation(state, convert_to_indexed(state, challenge.attestation)) # Verify the attestation is eligible for challenging responder = state.validator_registry[challenge.responder_index] min_challengeable_epoch = responder.exit_epoch - EPOCHS_PER_CUSTODY_PERIOD * (1 + responder.max_reveal_lateness) @@ -403,20 +418,18 @@ def process_bit_challenge(state: BeaconState, # Verify the chunk count chunk_count = get_custody_chunk_count(challenge.attestation) assert verify_bitfield(challenge.chunk_bits, chunk_count) - # Verify the xor of the chunk bits does not equal the custody bit - chunk_bits_xor = 0b0 - for i in range(chunk_count): - chunk_bits_xor ^ get_bitfield_bit(challenge.chunk_bits, i) + # Verify the first bit of the hash of the chunk bits does not equal the custody bit custody_bit = get_bitfield_bit(attestation.custody_bitfield, attesters.index(responder_index)) - assert custody_bit != chunk_bits_xor + assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) # Add new bit challenge record new_record = CustodyBitChallengeRecord( challenge_index=state.custody_challenge_index, challenger_index=challenge.challenger_index, responder_index=challenge.responder_index, deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE - crosslink_data_root=challenge.attestation.crosslink_data_root, - chunk_bits=challenge.chunk_bits, + crosslink_data_root=challenge.attestation.data.crosslink_data_root, + chunk_count=chunk_count, + chunk_bits_merkle_root=merkle_root(pad_to_power_of_2((challenge.chunk_bits))), responder_key=challenge.responder_key, ) replace_empty_or_append(state.custody_bit_challenge_records, new_record) @@ -451,10 +464,12 @@ def process_chunk_challenge_response(state: BeaconState, challenge: CustodyChunkChallengeRecord) -> None: # Verify chunk index assert response.chunk_index == challenge.chunk_index + # Verify bit challenge data is null + assert response.chunk_bits_branch == [] and response.chunk_bits_leaf == ZERO_HASH # Verify the chunk matches the crosslink data root assert verify_merkle_branch( leaf=hash_tree_root(response.chunk), - branch=response.branch, + branch=response.data_branch, depth=challenge.depth, index=response.chunk_index, root=challenge.crosslink_data_root, @@ -472,17 +487,25 @@ def process_bit_challenge_response(state: BeaconState, response: CustodyResponse, challenge: CustodyBitChallengeRecord) -> None: # Verify chunk index - assert response.chunk_index < len(challenge.chunk_bits) + assert response.chunk_index < challenge.chunk_count # Verify the chunk matches the crosslink data root assert verify_merkle_branch( leaf=hash_tree_root(response.chunk), - branch=response.branch, - depth=math.log2(next_power_of_two(len(challenge.chunk_bits))), + branch=response.data_branch, + depth=math.log2(next_power_of_two(challenge.chunk_count)), index=response.chunk_index, root=challenge.crosslink_data_root, ) + # Verify the chunk bit leaf matches the challenge data + assert verify_merkle_branch( + leaf=response.chunk_bits_leaf, + branch=response.chunk_bits_branch, + depth=math.log2(next_power_of_two(challenge.chunk_count) // 256), + index=response.chunk_index // 256, + root=challenge.chunk_bits_merkle_root + ) # Verify the chunk bit does not match the challenge chunk bit - assert get_custody_chunk_bit(challenge.responder_key, response.chunk) != get_bitfield_bit(challenge.chunk_bits, response.chunk_index) + assert get_custody_chunk_bit(challenge.responder_key, response.chunk) != get_bitfield_bit(challenge.chunk_bits_leaf, response.chunk_index % 256) # Clear the challenge records = state.custody_bit_challenge_records records[records.index(challenge)] = CustodyBitChallengeRecord() From c0f345309393eb8842fbd96fce65a71d5245ce62 Mon Sep 17 00:00:00 2001 From: dankrad Date: Fri, 3 May 2019 11:30:55 +0200 Subject: [PATCH 033/308] RANDAO reveal slashing, custody period staggering and integration of custody and RANDAO reveals (#880) * Add RandaoRevealSlashing for early Randao reveals * add some randao reveal slashing tests * Unifying RANDAO reveal slashing and Custody reveal slashing; implemented more lenient penalty for not-to-early reveals * Fix custody reveal domain * Also test exposed_randao_reveals * Re-add exposed_randao_reveals to validator initialization * Fix tests * Unified Randao Reveal slashing mechanism and Custody Reveal, CUSTODY_PERIOD_TO_RANDAO_PADDING = EPOCHS_PER_CUSTODY_PERIOD * Move exposed_randao_reveals into separate data structure to keep validator record clean * new shiny staggered custody periods * Fixes style and type hinting * removes whitespace * Cleans up multi-line conditionals * Make RANDAO key reveal penalties proportional to block rewards * Minor typos * Minor typos * Fixes off-by one error * Removes unnecicary whitepsace * Clean up comments; add test for key reveal too far in the future * Reduce the CUSTODY_PERIOD_TO_RANDAO_PADDING again * Fix max_proposer_slot_reward * Fix types * Move test * Fix RandaoKeyReveal tests * Move all RANDAO key reveal to phase 1 * Factor out signature checking * Some fixes * Update specs/core/1_custody-game.md Co-Authored-By: dankrad * Addressing Vitalik's suggestions: Separate RANDAO and Custody key reveals; append the cleanup of RANDAO reveals instead of adding a new function * Remove remnants of verify_custody_key * RandaoKeyReveal -> EarlyDerivedSecretReveal * Make penalty proportional to number of secrets already exposed * Update specs/core/1_custody-game.md Co-Authored-By: dankrad * Update specs/core/1_custody-game.md Co-Authored-By: dankrad * Update specs/core/1_custody-game.md Co-Authored-By: dankrad --- specs/core/1_custody-game.md | 317 +++++++++++++----- .../phase1_test_process_randao_key_reveal.py | 109 ++++++ test_libs/pyspec/tests/helpers_phase1.py | 50 +++ 3 files changed, 398 insertions(+), 78 deletions(-) create mode 100644 test_libs/pyspec/tests/block_processing_phase1/phase1_test_process_randao_key_reveal.py create mode 100644 test_libs/pyspec/tests/helpers_phase1.py diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 0f1555e625..d565266110 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -14,6 +14,7 @@ - [Misc](#misc) - [Time parameters](#time-parameters) - [Max operations per block](#max-operations-per-block) + - [Reward and penalty quotients](#reward-and-penalty-quotients) - [Signature domains](#signature-domains) - [Data structures](#data-structures) - [Custody objects](#custody-objects) @@ -22,7 +23,9 @@ - [`CustodyChunkChallengeRecord`](#custodychunkchallengerecord) - [`CustodyBitChallengeRecord`](#custodybitchallengerecord) - [`CustodyResponse`](#custodyresponse) + - [New Beacon operations](#new-beacon-operations) - [`CustodyKeyReveal`](#custodykeyreveal) + - [`EarlyDerivedSecretReveal`](#earlyderivedsecretreveal) - [Phase 0 container updates](#phase-0-container-updates) - [`Validator`](#validator) - [`BeaconState`](#beaconstate) @@ -32,17 +35,19 @@ - [`empty`](#empty) - [`get_crosslink_chunk_count`](#get_crosslink_chunk_count) - [`get_custody_chunk_bit`](#get_custody_chunk_bit) + - [`get_randao_epoch_for_custody_period`](#get_randao_epoch_for_custody_period) + - [`get_validators_custody_reveal_period`](#get_validators_custody_reveal_period) - [`get_chunk_bits_root`](#get_chunk_bits_root) - - [`epoch_to_custody_period`](#epoch_to_custody_period) - [`replace_empty_or_append`](#replace_empty_or_append) - - [`verify_custody_key`](#verify_custody_key) - [Per-block processing](#per-block-processing) - [Operations](#operations) - - [Custody reveals](#custody-reveals) + - [Custody key reveals](#custody-key-reveals) + - [Early derived secret reveals](#early-derived-secret-reveals) - [Chunk challenges](#chunk-challenges) - [Bit challenges](#bit-challenges) - [Custody responses](#custody-responses) - [Per-epoch processing](#per-epoch-processing) + - [Handling of custody-related deadlines](#handling-of-custody-related-deadlines) @@ -80,24 +85,32 @@ This document details the beacon chain additions and changes in Phase 1 of Ether | Name | Value | Unit | Duration | | - | - | :-: | :-: | | `MAX_CHUNK_CHALLENGE_DELAY` | `2**11` (= 2,048) | epochs | ~9 days | -| `EPOCHS_PER_CUSTODY_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | | `CUSTODY_RESPONSE_DEADLINE` | `2**14` (= 16,384) | epochs | ~73 days | +| `RANDAO_PENALTY_EPOCHS` | `2**1` (= 2) | epochs | 12.8 minutes | +| `EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS` | `2**14` | epochs | ~73 days | +| `EPOCHS_PER_CUSTODY_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | +| `CUSTODY_PERIOD_TO_RANDAO_PADDING` | `2**11` (= 2,048) | epochs | ~9 days | +| `MAX_REVEAL_LATENESS_DECREMENT` | `2**7` (= 128) | epochs | ~14 hours | ### Max operations per block | Name | Value | | - | - | | `MAX_CUSTODY_KEY_REVEALS` | `2**4` (= 16) | +| `MAX_EARLY_DERIVED_SECRET_REVEALS` | `1` | | `MAX_CUSTODY_CHUNK_CHALLENGES` | `2**2` (= 4) | | `MAX_CUSTODY_BIT_CHALLENGES` | `2**2` (= 4) | | `MAX_CUSTODY_RESPONSES` | `2**5` (= 32) | +### Reward and penalty quotients + +| `EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE` | `2**1` (= 2) | + ### Signature domains | Name | Value | | - | - | -| `DOMAIN_CUSTODY_KEY_REVEAL` | `6` | -| `DOMAIN_CUSTODY_BIT_CHALLENGE` | `7` | +| `DOMAIN_CUSTODY_BIT_CHALLENGE` | `6` | ## Data structures @@ -168,15 +181,35 @@ This document details the beacon chain additions and changes in Phase 1 of Ether } ``` +### New Beacon operations + #### `CustodyKeyReveal` ```python { - 'revealer_index': ValidatorIndex, - 'period': 'uint64', - 'key': BLSSignature, - 'masker_index': ValidatorIndex, - 'mask': Hash, + # Index of the validator whose key is being revealed + 'revealer_index': 'uint64', + # Reveal (masked signature) + 'reveal': 'bytes96', +} +``` + +#### `EarlyDerivedSecretReveal` + +Represents an early (punishable) reveal of one of the derived secrets, where derived secrets are RANDAO reveals and custody reveals (both are part of the same domain). + +```python +{ + # Index of the validator whose key is being revealed + 'revealed_index': 'uint64', + # RANDAO epoch of the key that is being revealed + 'epoch': 'uint64', + # Reveal (masked signature) + 'reveal': 'bytes96', + # Index of the validator who revealed (whistleblower) + 'masker_index': 'uint64', + # Mask used to hide the actual reveal signature (prevent reveal from being stolen) + 'mask': 'bytes32', } ``` @@ -187,7 +220,10 @@ Add the following fields to the end of the specified container objects. Fields w #### `Validator` ```python - 'custody_reveal_index': 'uint64', + # next_custody_reveal_period is initialised to the custody period + # (of the particular validator) in which the validator is activated + # = get_validators_custody_reveal_period(...) + 'next_custody_reveal_period': 'uint64', 'max_reveal_lateness': 'uint64', ``` @@ -197,15 +233,20 @@ Add the following fields to the end of the specified container objects. Fields w 'custody_chunk_challenge_records': [CustodyChunkChallengeRecord], 'custody_bit_challenge_records': [CustodyBitChallengeRecord], 'custody_challenge_index': 'uint64', + + # Future derived secrets already exposed; contains the indices of the exposed validator + # at RANDAO reveal period % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS + 'exposed_derived_secrets': [['uint64'], EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS], ``` #### `BeaconBlockBody` ```python - 'custody_key_reveals': [CustodyKeyReveal], 'custody_chunk_challenges': [CustodyChunkChallenge], 'custody_bit_challenges': [CustodyBitChallenge], 'custody_responses': [CustodyResponse], + 'custody_key_reveals': [CustodyKeyReveal], + 'early_derived_secret_reveals': [EarlyDerivedSecretReveal], ``` ## Helpers @@ -248,15 +289,33 @@ def get_chunk_bits_root(chunk_bitfield: Bitfield) -> Bytes32: return hash(aggregated_bits) ``` -### `epoch_to_custody_period` +### `get_randao_epoch_for_custody_period` ```python -def epoch_to_custody_period(epoch: Epoch) -> int: - return epoch // EPOCHS_PER_CUSTODY_PERIOD +def get_randao_epoch_for_custody_period(period: int, validator_index: ValidatorIndex) -> Epoch: + next_period_start = (period + 1) * EPOCHS_PER_CUSTODY_PERIOD - validator_index % EPOCHS_PER_CUSTODY_PERIOD + return next_period_start + CUSTODY_PERIOD_TO_RANDAO_PADDING +``` + +### `get_validators_custody_reveal_period` + + ```python +def get_validators_custody_reveal_period(state: BeaconState, + validator_index: ValidatorIndex, + epoch: Epoch=None) -> int: + ''' + This function returns the reveal period for a given validator. + If no epoch is supplied, the current epoch is assumed. + Note: This function implicitly requires that validators are not removed from the + validator set in fewer than EPOCHS_PER_CUSTODY_PERIOD epochs + ''' + epoch = get_current_epoch(state) if epoch is None else epoch + return (epoch + validator_index % EPOCHS_PER_CUSTODY_PERIOD) // EPOCHS_PER_CUSTODY_PERIOD ``` ### `replace_empty_or_append` + ```python def replace_empty_or_append(list: List[Any], new_element: Any) -> int: for i in range(len(list)): @@ -267,68 +326,131 @@ def replace_empty_or_append(list: List[Any], new_element: Any) -> int: return len(list) - 1 ``` -### `verify_custody_key` - -```python -def verify_custody_key(state: BeaconState, reveal: CustodyKeyReveal) -> bool: - # Case 1: non-masked non-punitive non-early reveal - pubkeys = [state.validator_registry[reveal.revealer_index].pubkey] - message_hashes = [hash_tree_root(reveal.period)] - - # Case 2: masked punitive early reveal - # Masking prevents proposer stealing the whistleblower reward - # Secure under the aggregate extraction infeasibility assumption - # See pages 11-12 of https://crypto.stanford.edu/~dabo/pubs/papers/aggreg.pdf - if reveal.mask != ZERO_HASH: - pubkeys.append(state.validator_registry[reveal.masker_index].pubkey) - message_hashes.append(reveal.mask) - - return bls_verify_multiple( - pubkeys=pubkeys, - message_hashes=message_hashes, - signature=reveal.key, - domain=get_domain( - fork=state.fork, - epoch=reveal.period * EPOCHS_PER_CUSTODY_PERIOD, - domain_type=DOMAIN_CUSTODY_KEY_REVEAL, - ), - ) -``` - ## Per-block processing ### Operations Add the following operations to the per-block processing, in order the given below and after all other operations in phase 0. -#### Custody reveals +#### Custody key reveals Verify that `len(block.body.custody_key_reveals) <= MAX_CUSTODY_KEY_REVEALS`. For each `reveal` in `block.body.custody_key_reveals`, run the following function: ```python -def process_custody_reveal(state: BeaconState, +def process_custody_key_reveal(state: BeaconState, reveal: CustodyKeyReveal) -> None: - assert verify_custody_key(state, reveal) + + """ + Process ``CustodyKeyReveal`` operation. + Note that this function mutates ``state``. + """ + revealer = state.validator_registry[reveal.revealer_index] - current_custody_period = epoch_to_custody_period(get_current_epoch(state)) - - # Case 1: non-masked non-punitive non-early reveal - if reveal.mask == ZERO_HASH: - assert reveal.period == epoch_to_custody_period(revealer.activation_epoch) + revealer.custody_reveal_index - # Revealer is active or exited - assert is_active_validator(revealer, get_current_epoch(state)) or revealer.exit_epoch > get_current_epoch(state) - revealer.custody_reveal_index += 1 - revealer.max_reveal_lateness = max(revealer.max_reveal_lateness, current_custody_period - reveal.period) - proposer_index = get_beacon_proposer_index(state) - increase_balance(state, proposer_index, base_reward(state, index) // MINOR_REWARD_QUOTIENT) + epoch_to_sign = get_randao_epoch_for_custody_period(revealer.next_custody_reveal_period, reveal.revealed_index) + + assert revealer.next_custody_reveal_period < get_validators_custody_reveal_period(state, reveal.revealed_index) + + # Revealed validator is active or exited, but not withdrawn + assert is_slashable_validator(revealer, get_current_epoch(state)) + + # Verify signature + assert bls_verify( + pubkey=revealer.pubkey, + message_hash=hash_tree_root(epoch_to_sign), + signature=reveal.reveal, + domain=get_domain( + state=state, + domain_type=DOMAIN_RANDAO, + message_epoch=epoch_to_sign, + ), + ) + + # Decrement max reveal lateness if response is timely + if revealer.next_custody_reveal_period == get_validators_custody_reveal_period(state, reveal.revealer_index) - 2: + revealer.max_reveal_lateness -= MAX_REVEAL_LATENESS_DECREMENT + revealer.max_reveal_lateness = max(revealed_validator.max_reveal_lateness, get_validators_custody_reveal_period(state, reveal.revealed_index) - revealer.next_custody_reveal_period) + + # Process reveal + revealer.next_custody_reveal_period += 1 + + # Reward Block Preposer + proposer_index = get_beacon_proposer_index(state) + increase_balance(state, proposer_index, base_reward(state, index) // MINOR_REWARD_QUOTIENT) +``` + +##### Early derived secret reveals + +Verify that `len(block.body.early_derived_secret_reveals) <= MAX_EARLY_DERIVED_SECRET_REVEALS`. + +For each `reveal` in `block.body.early_derived_secret_reveals`, run the following function: + +```python +def process_early_derived_secret_reveal(state: BeaconState, + reveal: EarlyDerivedSecretReveal) -> None: + """ + Process ``EarlyDerivedSecretReveal`` operation. + Note that this function mutates ``state``. + """ + + revealed_validator = state.validator_registry[reveal.revealed_index] + masker = state.validator_registry[reveal.masker_index] + + assert reveal.epoch >= get_current_epoch(state) + RANDAO_PENALTY_EPOCHS + assert reveal.epoch < get_current_epoch(state) + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS + assert revealed_validator.slashed is False + assert reveal.revealed_index not in state.exposed_derived_secrets[reveal.epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] + + # Verify signature correctness + masker = state.validator_registry[reveal.masker_index] + pubkeys = [revealed_validator.pubkey, masker.pubkey] + message_hashes = [ + hash_tree_root(reveal.epoch), + reveal.mask, + ] + + assert bls_verify_multiple( + pubkeys=pubkeys, + message_hashes=message_hashes, + signature=reveal.reveal, + domain=get_domain( + state=state, + domain_type=DOMAIN_RANDAO, + message_epoch=reveal.epoch, + ), + ) - # Case 2: masked punitive early reveal + if reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: + # Full slashing when the secret was revealed so early it may be a valid custody + # round key + slash_validator(state, reveal.revealed_index, reveal.masker_index) else: - assert reveal.period > current_custody_period - assert revealer.slashed is False - slash_validator(state, reveal.revealer_index, reveal.masker_index) + # Only a small penalty proportional to proposer slot reward for RANDAO reveal + # that does not interfere with the custody period + # The penalty is proportional to the max proposer reward + + # Calculate penalty + max_proposer_slot_reward = ( + get_base_reward(state, reveal.revealed_index) * + SLOTS_PER_EPOCH // + len(get_active_validator_indices(state, get_current_epoch(state))) // + PROPOSER_REWARD_QUOTIENT + ) + penalty = max_proposer_slot_reward * EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE * (len(state.exposed_derived_secrets[reveal.epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS]) + 1) + + # Apply penalty + proposer_index = get_beacon_proposer_index(state) + whistleblower_index = reveal.masker_index + whistleblowing_reward = penalty // WHISTLEBLOWING_REWARD_QUOTIENT + proposer_reward = whistleblowing_reward // PROPOSER_REWARD_QUOTIENT + increase_balance(state, proposer_index, proposer_reward) + increase_balance(state, whistleblower_index, whistleblowing_reward - proposer_reward) + decrease_balance(state, reveal.revealed_index, penalty) + + # Mark this derived secret as exposed so validator cannot be punished repeatedly + state.exposed_derived_secrets[reveal.epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS].append(reveal.revealed_index) + ``` #### Chunk challenges @@ -384,6 +506,7 @@ For each `challenge` in `block.body.custody_bit_challenges`, run the following f ```python def process_bit_challenge(state: BeaconState, challenge: CustodyBitChallenge) -> None: + # Verify challenge signature challenger = state.validator_registry[challenge.challenger_index] assert bls_verify( @@ -392,29 +515,43 @@ def process_bit_challenge(state: BeaconState, signature=challenge.signature, domain=get_domain(state, get_current_epoch(state), DOMAIN_CUSTODY_BIT_CHALLENGE), ) - # Verify the challenger is not slashed - assert challenger.slashed is False + assert is_slashable_validator(challenger, get_current_epoch(state)) + # Verify the attestation assert verify_indexed_attestation(state, convert_to_indexed(state, challenge.attestation)) # Verify the attestation is eligible for challenging responder = state.validator_registry[challenge.responder_index] - min_challengeable_epoch = responder.exit_epoch - EPOCHS_PER_CUSTODY_PERIOD * (1 + responder.max_reveal_lateness) - assert min_challengeable_epoch <= slot_to_epoch(challenge.attestation.data.slot) + assert (slot_to_epoch(challenge.attestation.data.slot) + responder.max_reveal_lateness <= + get_validators_custody_reveal_period(state, challenge.responder_index)) + # Verify the responder participated in the attestation attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) assert challenge.responder_index in attesters + # A validator can be the challenger or responder for at most one challenge at a time for record in state.custody_bit_challenge_records: assert record.challenger_index != challenge.challenger_index assert record.responder_index != challenge.responder_index - # Verify the responder key - assert verify_custody_key(state, CustodyKeyReveal( - revealer_index=challenge.responder_index, - period=epoch_to_custody_period(slot_to_epoch(attestation.data.slot)), - key=challenge.responder_key, - masker_index=0, - mask=ZERO_HASH, - )) + + # Verify the responder is a valid custody key + epoch_to_sign = get_randao_epoch_for_custody_period( + get_validators_custody_reveal_period( + state=state, + index=challenge.responder_index, + epoch=slot_to_epoch(attestation.data.slot), + challenge.responder_index + ) + assert bls_verify( + pubkey=responder.pubkey, + message_hash=hash_tree_root(epoch_to_sign), + signature=challenge.responder_key, + domain=get_domain( + state=state, + domain_type=DOMAIN_RANDAO, + message_epoch=epoch_to_sign, + ), + ) + # Verify the chunk count chunk_count = get_custody_chunk_count(challenge.attestation) assert verify_bitfield(challenge.chunk_bits, chunk_count) @@ -426,7 +563,7 @@ def process_bit_challenge(state: BeaconState, challenge_index=state.custody_challenge_index, challenger_index=challenge.challenger_index, responder_index=challenge.responder_index, - deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE + deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE, crosslink_data_root=challenge.attestation.data.crosslink_data_root, chunk_count=chunk_count, chunk_bits_merkle_root=merkle_root(pad_to_power_of_2((challenge.chunk_bits))), @@ -434,6 +571,7 @@ def process_bit_challenge(state: BeaconState, ) replace_empty_or_append(state.custody_bit_challenge_records, new_record) state.custody_challenge_index += 1 + # Postpone responder withdrawability responder.withdrawable_epoch = FAR_FUTURE_EPOCH ``` @@ -515,7 +653,20 @@ def process_bit_challenge_response(state: BeaconState, ## Per-epoch processing -Run `process_challenge_deadlines(state)` immediately after `process_ejections(state)`: +### Handling of custody-related deadlines + + Run `process_reveal_deadlines(state)` immediately after `process_ejections(state)`: + + ```python +def process_reveal_deadlines(state: BeaconState) -> None: + for index, validator in enumerate(state.validator_registry): + if (validator.latest_custody_reveal_period + + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) < + get_validators_custody_reveal_period(state, index)): + slash_validator(state, index) +``` + +Run `process_challenge_deadlines(state)` immediately after `process_reveal_deadlines(state)`: ```python def process_challenge_deadlines(state: BeaconState) -> None: @@ -532,16 +683,26 @@ def process_challenge_deadlines(state: BeaconState) -> None: records[records.index(challenge)] = CustodyBitChallengeRecord() ``` +Append this to `process_final_updates(state)`: + +```python + # Clean up exposed RANDAO key reveals + state.exposed_derived_secrets[current_epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] +``` + In `process_penalties_and_exits`, change the definition of `eligible` to the following (note that it is not a pure function because `state` is declared in the surrounding scope): ```python -def eligible(index): +def eligible(state: BeaconState, index: ValidatorIndex) -> bool: validator = state.validator_registry[index] # Cannot exit if there are still open chunk challenges if len([record for record in state.custody_chunk_challenge_records if record.responder_index == index]) > 0: return False + # Cannot exit if there are still open bit challenges + if len([record for record in state.custody_bit_challenge_records if record.responder_index == index]) > 0: + return False # Cannot exit if you have not revealed all of your custody keys - elif epoch_to_custody_period(revealer.activation_epoch) + validator.custody_reveal_index <= epoch_to_custody_period(validator.exit_epoch): + elif validator.next_custody_reveal_period <= get_validators_custody_reveal_period(state, index, validator.exit_epoch): return False # Cannot exit if you already have elif validator.withdrawable_epoch < FAR_FUTURE_EPOCH: diff --git a/test_libs/pyspec/tests/block_processing_phase1/phase1_test_process_randao_key_reveal.py b/test_libs/pyspec/tests/block_processing_phase1/phase1_test_process_randao_key_reveal.py new file mode 100644 index 0000000000..0be8ab4a97 --- /dev/null +++ b/test_libs/pyspec/tests/block_processing_phase1/phase1_test_process_randao_key_reveal.py @@ -0,0 +1,109 @@ +from copy import deepcopy +import pytest + +import eth2spec.phase1.spec as spec +from eth2spec.phase1.spec import ( + get_current_epoch, + process_randao_key_reveal, + RANDAO_PENALTY_EPOCHS, + CUSTODY_PERIOD_TO_RANDAO_PADDING, + RANDAO_PENALTY_MAX_FUTURE_EPOCHS, +) +from tests.helpers_phase1 import ( + get_valid_randao_key_reveal, +) + +mark entire file as 'randao_key_reveals' +pytestmark = pytest.mark.randao_key_reveals + + +def run_randao_key_reveal_processing(state, randao_key_reveal, valid=True): + """ + Run ``process_randao_key_reveal`` returning the pre and post state. + If ``valid == False``, run expecting ``AssertionError`` + """ + post_state = deepcopy(state) + + if not valid: + with pytest.raises(AssertionError): + process_randao_key_reveal(post_state, randao_key_reveal) + return state, None + + process_randao_key_reveal(post_state, randao_key_reveal) + + slashed_validator = post_state.validator_registry[randao_key_reveal.revealed_index] + + if randao_key_reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + # lost whistleblower reward + # FIXME: Currently broken because get_base_reward in genesis epoch is 0 + assert ( + post_state.balances[randao_key_reveal.revealed_index] < + state.balances[randao_key_reveal.revealed_index] + ) + + return state, post_state + + +def test_success(state): + randao_key_reveal = get_valid_randao_key_reveal(state) + + pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal) + + return pre_state, randao_key_reveal, post_state + + +def test_reveal_from_current_epoch(state): + randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state)) + + pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, False) + + return pre_state, randao_key_reveal, post_state + +# Not currently possible as we are testing at epoch 0 +# +#def test_reveal_from_past_epoch(state): +# randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state) - 1) +# +# pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, False) +# +# return pre_state, randao_key_reveal, post_state + +def test_reveal_with_custody_padding(state): + randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING) + pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, True) + + return pre_state, randao_key_reveal, post_state + +def test_reveal_with_custody_padding_minus_one(state): + randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING - 1) + pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, True) + + return pre_state, randao_key_reveal, post_state + +def test_double_reveal(state): + + randao_key_reveal1 = get_valid_randao_key_reveal(state, get_current_epoch(state) + RANDAO_PENALTY_EPOCHS + 1) + pre_state, intermediate_state = run_randao_key_reveal_processing(state, randao_key_reveal1) + + randao_key_reveal2 = get_valid_randao_key_reveal(intermediate_state, get_current_epoch(pre_state) + RANDAO_PENALTY_EPOCHS + 1) + intermediate_state_, post_state = run_randao_key_reveal_processing(intermediate_state, randao_key_reveal2, False) + + return pre_state, [randao_key_reveal1, randao_key_reveal2], post_state + +def test_revealer_is_slashed(state): + randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state)) + state.validator_registry[randao_key_reveal.revealed_index].slashed = True + + pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, False) + + return pre_state, randao_key_reveal, post_state + +def test_far_future_epoch(state): + randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state) + RANDAO_PENALTY_MAX_FUTURE_EPOCHS) + + pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, False) + + return pre_state, randao_key_reveal, post_state diff --git a/test_libs/pyspec/tests/helpers_phase1.py b/test_libs/pyspec/tests/helpers_phase1.py new file mode 100644 index 0000000000..aba93e1596 --- /dev/null +++ b/test_libs/pyspec/tests/helpers_phase1.py @@ -0,0 +1,50 @@ +from py_ecc import bls + +import eth2spec.phase1.spec as spec +from eth2spec.phase0.spec import ( + # constants + ZERO_HASH, + CUSTODY_PERIOD_TO_RANDAO_PADDING, + # SSZ + RandaoKeyReveal, + # functions + get_active_validator_indices, + get_current_epoch, + get_domain, + hash_tree_root, +) + +def get_valid_randao_key_reveal(state, epoch=None): + current_epoch = get_current_epoch(state) + revealed_index = get_active_validator_indices(state, current_epoch)[-1] + masker_index = get_active_validator_indices(state, current_epoch)[0] + + if epoch is None: + epoch = current_epoch + CUSTODY_PERIOD_TO_RANDAO_PADDING + + reveal = bls.sign( + message_hash=hash_tree_root(epoch), + privkey=pubkey_to_privkey[state.validator_registry[revealed_index].pubkey], + domain=get_domain( + state=state, + domain_type=spec.DOMAIN_RANDAO, + message_epoch=epoch, + ), + ) + mask = bls.sign( + message_hash=hash_tree_root(epoch), + privkey=pubkey_to_privkey[state.validator_registry[masker_index].pubkey], + domain=get_domain( + state=state, + domain_type=spec.DOMAIN_RANDAO, + message_epoch=epoch, + ), + ) + + return RandaoKeyReveal( + revealed_index=revealed_index, + epoch=epoch, + reveal=reveal, + masker_index=masker_index, + mask=mask, + ) From 66b152f79ecad957b38ecccc16ce2f131afd10cc Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 3 May 2019 05:07:11 -0500 Subject: [PATCH 034/308] Allow multiple bit challenges, and recover withdrawability Resolves #864 items 4, 7, 14 --- specs/core/1_custody-game.md | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index d565266110..0de84ff067 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -146,7 +146,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether 'challenge_index': 'uint64', 'challenger_index': ValidatorIndex, 'responder_index': ValidatorIndex, - 'deadline': Epoch, + 'inclusion_epoch': Epoch, 'crosslink_data_root': Hash, 'depth': 'uint64', 'chunk_index': 'uint64', @@ -160,7 +160,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether 'challenge_index': 'uint64', 'challenger_index': ValidatorIndex, 'responder_index': ValidatorIndex, - 'deadline': Epoch, + 'inclusion_epoch': Epoch, 'crosslink_data_root': Hash, 'chunk_count': 'uint64', 'chunk_bits_merkle_root': Hash, @@ -485,7 +485,7 @@ def process_chunk_challenge(state: BeaconState, challenge_index=state.custody_challenge_index, challenger_index=get_beacon_proposer_index(state), responder_index=challenge.responder_index - deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE, + inclusion_epoch=get_current_epoch(state), crosslink_data_root=challenge.attestation.data.crosslink_data_root, depth=depth, chunk_index=challenge.chunk_index, @@ -528,10 +528,9 @@ def process_bit_challenge(state: BeaconState, attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) assert challenge.responder_index in attesters - # A validator can be the challenger or responder for at most one challenge at a time + # A validator can be the challenger for at most one challenge at a time for record in state.custody_bit_challenge_records: assert record.challenger_index != challenge.challenger_index - assert record.responder_index != challenge.responder_index # Verify the responder is a valid custody key epoch_to_sign = get_randao_epoch_for_custody_period( @@ -563,7 +562,7 @@ def process_bit_challenge(state: BeaconState, challenge_index=state.custody_challenge_index, challenger_index=challenge.challenger_index, responder_index=challenge.responder_index, - deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE, + inclusion_epoch=get_current_epoch(state), crosslink_data_root=challenge.attestation.data.crosslink_data_root, chunk_count=chunk_count, chunk_bits_merkle_root=merkle_root(pad_to_power_of_2((challenge.chunk_bits))), @@ -604,6 +603,8 @@ def process_chunk_challenge_response(state: BeaconState, assert response.chunk_index == challenge.chunk_index # Verify bit challenge data is null assert response.chunk_bits_branch == [] and response.chunk_bits_leaf == ZERO_HASH + # Verify minimum delay + assert get_current_epoch(state) >= challenge.inclusion_epoch + ACTIVATION_EXIT_DELAY # Verify the chunk matches the crosslink data root assert verify_merkle_branch( leaf=hash_tree_root(response.chunk), @@ -626,6 +627,9 @@ def process_bit_challenge_response(state: BeaconState, challenge: CustodyBitChallengeRecord) -> None: # Verify chunk index assert response.chunk_index < challenge.chunk_count + # Verify responder has not been slashed + responder = state.validator_registry[record.responder_index] + assert not responder.slashed # Verify the chunk matches the crosslink data root assert verify_merkle_branch( leaf=hash_tree_root(response.chunk), @@ -671,13 +675,13 @@ Run `process_challenge_deadlines(state)` immediately after `process_reveal_deadl ```python def process_challenge_deadlines(state: BeaconState) -> None: for challenge in state.custody_chunk_challenge_records: - if get_current_epoch(state) > challenge.deadline: + if get_current_epoch(state) > challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: slash_validator(state, challenge.responder_index, challenge.challenger_index) records = state.custody_chunk_challenge_records records[records.index(challenge)] = CustodyChunkChallengeRecord() for challenge in state.custody_bit_challenge_records: - if get_current_epoch(state) > challenge.deadline: + if get_current_epoch(state) > challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: slash_validator(state, challenge.responder_index, challenge.challenger_index) records = state.custody_bit_challenge_records records[records.index(challenge)] = CustodyBitChallengeRecord() @@ -688,6 +692,18 @@ Append this to `process_final_updates(state)`: ```python # Clean up exposed RANDAO key reveals state.exposed_derived_secrets[current_epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] + # Reset withdrawable epochs if challenge records are empty + for index, validator in enumerate(state.validator_registry): + eligible = True + for records in (state.custody_chunk_challenge_records, state.bit_challenge_records): + for filter_func in (lambda rec: rec.challenger_index == index, lambda rec: rec.responder_index == index): + if len(list(filter(filter_func, records))) > 0: + eligible = False + if eligible: + if validator.exit_epoch == FAR_FUTURE_EPOCH: + validator.withdrawable_epoch = FAR_FUTURE_EPOCH + else: + validator.withdrawable_epoch = validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY ``` In `process_penalties_and_exits`, change the definition of `eligible` to the following (note that it is not a pure function because `state` is declared in the surrounding scope): From 2ccd357f0eab881eaf7f6bd39e09987467731c91 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 3 May 2019 21:05:54 +0800 Subject: [PATCH 035/308] Update specs/core/1_custody-game.md Co-Authored-By: vbuterin --- specs/core/1_custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 0de84ff067..307e18573d 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -628,7 +628,7 @@ def process_bit_challenge_response(state: BeaconState, # Verify chunk index assert response.chunk_index < challenge.chunk_count # Verify responder has not been slashed - responder = state.validator_registry[record.responder_index] + responder = state.validator_registry[challenge.responder_index] assert not responder.slashed # Verify the chunk matches the crosslink data root assert verify_merkle_branch( From 197a7200efaf1bf9024910e3c41563e3b0c96f77 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Fri, 3 May 2019 21:21:42 +0100 Subject: [PATCH 036/308] Add HW and Danny comments --- specs/core/0_beacon-chain.md | 61 +++++++++---------- .../test_process_block_header.py | 4 +- .../test_process_crosslinks.py | 4 +- test_libs/pyspec/tests/test_sanity.py | 4 +- 4 files changed, 36 insertions(+), 37 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 86caf7402d..b4b44e6282 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -96,7 +96,6 @@ - [`slash_validator`](#slash_validator) - [On genesis](#on-genesis) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - - [State caching](#state-caching) - [Epoch processing](#epoch-processing) - [Helper functions](#helper-functions-1) - [Justification and finalization](#justification-and-finalization) @@ -1267,36 +1266,36 @@ The post-state corresponding to a pre-state `state` and a block `block` is defin ```python def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconBlock: - # Block must post-date the state - assert state.slot < block.slot # Process slots (including those with no blocks) since block - while state.slot < block.slot: - # Cache state at the start of every slot - cache_state(state) - # Process epoch at the start of the first slot of every epoch - if (state.slot + 1) % SLOTS_PER_EPOCH == 0: - process_epoch(state) - # Increment slot number - state.slot += 1 + process_slots(state, block.slot) # Process block process_block(state, block) # Return post-state return state ``` -### State caching +```python +def process_slots(state: BeaconState, slot: Slot) -> None: + assert state.slot < slot + while state.slot < slot: + process_slot(state) + # Process epoch on the last slot of every epoch + if (state.slot + 1) % SLOTS_PER_EPOCH == 0: + process_epoch(state) + state.slot += 1 +``` ```python -def cache_state(state: BeaconState) -> None: - # Cache state root of previous slot +def process_slot(state: BeaconState) -> None: + # Cache state root previous_state_root = hash_tree_root(state) state.latest_state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root - # Cache previous state root in latest_block_header, if empty + # Cache latest block header state root if state.latest_block_header.state_root == ZERO_HASH: state.latest_block_header.state_root = previous_state_root - # Cache block root of previous slot + # Cache block root previous_block_root = signing_root(state.latest_block_header) state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root @@ -1631,10 +1630,10 @@ def process_final_updates(state: BeaconState) -> None: ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) - process_randao(state, block) - process_eth1_data(state, block) + process_randao(state, block.body) + process_eth1_data(state, block.body) process_operations(state, block.body) - # verify_block_state_root(state, block) + # process_state_root(state, block) ``` #### Block header @@ -1661,31 +1660,33 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: #### RANDAO ```python -def process_randao(state: BeaconState, block: BeaconBlock) -> None: +def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: proposer = state.validator_registry[get_beacon_proposer_index(state)] # Verify that the provided randao value is valid - assert bls_verify(proposer.pubkey, hash_tree_root(get_current_epoch(state)), block.body.randao_reveal, get_domain(state, DOMAIN_RANDAO)) + assert bls_verify(proposer.pubkey, hash_tree_root(get_current_epoch(state)), body.randao_reveal, get_domain(state, DOMAIN_RANDAO)) # Mix it in state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = ( xor(get_randao_mix(state, get_current_epoch(state)), - hash(block.body.randao_reveal)) + hash(body.randao_reveal)) ) ``` #### Eth1 data ```python -def process_eth1_data(state: BeaconState, block: BeaconBlock) -> None: - state.eth1_data_votes.append(block.body.eth1_data) - if state.eth1_data_votes.count(block.body.eth1_data) * 2 > SLOTS_PER_ETH1_VOTING_PERIOD: - state.latest_eth1_data = block.body.eth1_data +def process_eth1_data(state: BeaconState, body: BeaconBlockBody) -> None: + state.eth1_data_votes.append(body.eth1_data) + if state.eth1_data_votes.count(body.eth1_data) * 2 > SLOTS_PER_ETH1_VOTING_PERIOD: + state.latest_eth1_data = body.eth1_data ``` #### Operations ```python def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: + # Verify that outstanding deposits are processed up to the maximum number of deposits assert len(body.deposits) == min(MAX_DEPOSITS, state.latest_eth1_data.deposit_count - state.deposit_index) + # Verify that there are no duplicate transfers assert len(body.transfers) == len(set(body.transfers)) for operations, max_operations, function in ( @@ -1704,8 +1705,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: ##### Proposer slashings ```python -def process_proposer_slashing(state: BeaconState, - proposer_slashing: ProposerSlashing) -> None: +def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSlashing) -> None: """ Process ``ProposerSlashing`` operation. Note that this function mutates ``state``. @@ -1728,8 +1728,7 @@ def process_proposer_slashing(state: BeaconState, ##### Attester slashings ```python -def process_attester_slashing(state: BeaconState, - attester_slashing: AttesterSlashing) -> None: +def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSlashing) -> None: """ Process ``AttesterSlashing`` operation. Note that this function mutates ``state``. @@ -1904,6 +1903,6 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: #### State root verification ```python -def verify_block_state_root(state: BeaconState, block: BeaconBlock) -> None: +def process_state_root(state: BeaconState, block: BeaconBlock) -> None: assert block.state_root == hash_tree_root(state) ``` diff --git a/test_libs/pyspec/tests/block_processing/test_process_block_header.py b/test_libs/pyspec/tests/block_processing/test_process_block_header.py index 6fd6e674e3..149bab5141 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/tests/block_processing/test_process_block_header.py @@ -4,7 +4,7 @@ from eth2spec.phase0.spec import ( get_beacon_proposer_index, - cache_state, + process_slot, process_block_header, ) from tests.helpers import ( @@ -18,7 +18,7 @@ def prepare_state_for_header_processing(state): - cache_state(state) + process_slot(state) advance_slot(state) diff --git a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py index 60e0dec535..4e3d4b5d89 100644 --- a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py @@ -4,7 +4,7 @@ import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import ( - cache_state, + process_slot, get_crosslink_deltas, process_crosslinks, state_transition, @@ -33,7 +33,7 @@ def run_process_crosslinks(state, valid=True): state_transition(state, block) # cache state before epoch transition - cache_state(state) + process_slot(state) post_state = deepcopy(state) process_crosslinks(post_state) diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 4d826fc130..7fff6fb550 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -20,7 +20,7 @@ get_state_root, get_current_epoch, get_domain, - cache_state, + process_slot, verify_merkle_branch, state_transition, hash, @@ -51,7 +51,7 @@ def test_slot_transition(state): test_state = deepcopy(state) - cache_state(test_state) + process_slot(test_state) advance_slot(test_state) assert test_state.slot == state.slot + 1 assert get_state_root(test_state, state.slot) == state.hash_tree_root() From 4c1073fa2f898de7ffe26fd8aa197d38c25a8f35 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 3 May 2019 22:22:19 +0100 Subject: [PATCH 037/308] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b4b44e6282..8d999c65cc 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1279,7 +1279,7 @@ def process_slots(state: BeaconState, slot: Slot) -> None: assert state.slot < slot while state.slot < slot: process_slot(state) - # Process epoch on the last slot of every epoch + # Process epoch on the first slot of the next epoch if (state.slot + 1) % SLOTS_PER_EPOCH == 0: process_epoch(state) state.slot += 1 From d5d2f7835aed7d4e2400ebce09af54fa70613173 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 3 May 2019 22:51:59 -0500 Subject: [PATCH 038/308] Rearranging one >= statement to make it more non-underflow-friendly --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 420ea3ef90..36ac927961 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1799,7 +1799,7 @@ def process_voluntary_exit(state: BeaconState, exit: VoluntaryExit) -> None: # Exits must specify an epoch when they become valid; they are not valid before then assert get_current_epoch(state) >= exit.epoch # Verify the validator has been active long enough - assert get_current_epoch(state) - validator.activation_epoch >= PERSISTENT_COMMITTEE_PERIOD + assert get_current_epoch(state) >= validator.activation_epoch + PERSISTENT_COMMITTEE_PERIOD # Verify signature domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, exit.epoch) assert bls_verify(validator.pubkey, signing_root(exit), exit.signature, domain) From a6e825d46056a4a92473e52f1ea0bcb19210e67b Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 5 May 2019 12:04:34 +0100 Subject: [PATCH 039/308] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 56 +++++++++++++++--------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 36ac927961..addaa865ee 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -279,6 +279,8 @@ The types are defined topologically to aid in facilitating an executable version ```python { + # Shard number + 'shard': 'uint64', # Epoch number 'epoch': 'uint64', # Root of the previous crosslink @@ -315,9 +317,7 @@ The types are defined topologically to aid in facilitating an executable version 'target_root': 'bytes32', # Crosslink vote - 'shard': 'uint64', - 'previous_crosslink_root': 'bytes32', - 'crosslink_data_root': 'bytes32', + 'crosslink': Crosslink, } ``` @@ -765,7 +765,7 @@ def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: def get_attestation_slot(state: BeaconState, attestation: Attestation) -> Slot: epoch = attestation.data.target_epoch committee_count = get_epoch_committee_count(state, epoch) - offset = (attestation.data.shard + SHARD_COUNT - get_epoch_start_shard(state, epoch)) % SHARD_COUNT + offset = (attestation.data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, epoch)) % SHARD_COUNT return get_epoch_start_slot(epoch) + offset // (committee_count // SLOTS_PER_EPOCH) ``` @@ -927,7 +927,7 @@ def get_attesting_indices(state: BeaconState, """ Return the sorted attesting indices corresponding to ``attestation_data`` and ``bitfield``. """ - committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.shard) + committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) assert verify_bitfield(bitfield, len(committee)) return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) ``` @@ -1296,28 +1296,18 @@ def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestat return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) ``` -```python -def get_crosslink_from_attestation_data(state: BeaconState, data: AttestationData) -> Crosslink: - return Crosslink( - epoch=min(data.target_epoch, state.current_crosslinks[data.shard].epoch + MAX_CROSSLINK_EPOCHS), - previous_crosslink_root=data.previous_crosslink_root, - crosslink_data_root=data.crosslink_data_root, - ) -``` - ```python def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[Crosslink, List[ValidatorIndex]]: - shard_attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.shard == shard] - shard_crosslinks = [get_crosslink_from_attestation_data(state, a.data) for a in shard_attestations] + shard_attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.crosslink.shard == shard] candidate_crosslinks = [ - c for c in shard_crosslinks + c for c in [a.data.crosslink for a in shard_attestations] if hash_tree_root(state.current_crosslinks[shard]) in (c.previous_crosslink_root, hash_tree_root(c)) ] if len(candidate_crosslinks) == 0: return Crosslink(), [] def get_attestations_for(crosslink: Crosslink) -> List[PendingAttestation]: - return [a for a in shard_attestations if get_crosslink_from_attestation_data(state, a.data) == crosslink] + return [a for a in shard_attestations if a.data.crosslink == crosslink] # Winning crosslink has the crosslink data root with the most balance voting for it (ties broken lexicographically) winning_crosslink = max(candidate_crosslinks, key=lambda crosslink: ( get_attesting_balance(state, get_attestations_for(crosslink)), crosslink.crosslink_data_root @@ -1705,30 +1695,30 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: attestation_slot = get_attestation_slot(state, attestation) assert attestation_slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation_slot + SLOTS_PER_EPOCH - # Check target epoch, source epoch, source root, and source crosslink data = attestation.data - assert (data.target_epoch, data.source_epoch, data.source_root, data.previous_crosslink_root) in { - (get_current_epoch(state), state.current_justified_epoch, state.current_justified_root, hash_tree_root(state.current_crosslinks[data.shard])), - (get_previous_epoch(state), state.previous_justified_epoch, state.previous_justified_root, hash_tree_root(state.previous_crosslinks[data.shard])), - } - - # Check crosslink data root - assert data.crosslink_data_root == ZERO_HASH # [to be removed in phase 1] - - # Check signature and bitfields - assert verify_indexed_attestation(state, convert_to_indexed(state, attestation)) - - # Cache pending attestation pending_attestation = PendingAttestation( data=data, aggregation_bitfield=attestation.aggregation_bitfield, inclusion_delay=state.slot - attestation_slot, proposer_index=get_beacon_proposer_index(state), ) + + assert data.target_epoch in (get_previous_epoch(state), get_current_epoch(state)) + if data.target_epoch == get_previous_epoch(state): + ffg_data = (state.previous_justified_epoch, state.previous_justified_root, get_previous_epoch(state)) + previous_crosslink = state.previous_crosslinks[data.crosslink.shard] + state.previous_epoch_attestations.append(pending_attestation) if data.target_epoch == get_current_epoch(state): + ffg_data = (state.current_justified_epoch, state.current_justified_root, get_current_epoch(state)) + previous_crosslink = state.current_crosslinks[data.crosslink.shard] state.current_epoch_attestations.append(pending_attestation) - else: - state.previous_epoch_attestations.append(pending_attestation) + + # Check FFG data, crosslink data, and signature + assert ffg_data == (data.source_epoch, data.source_root, data.target_epoch) + assert data.crosslink.epoch == min(data.target_epoch, previous_crosslink.epoch + MAX_CROSSLINK_EPOCHS) + assert data.crosslink.previous_crosslink_root == hash_tree_root(previous_crosslink) + assert data.crosslink.crosslink_data_root == ZERO_HASH # [to be removed in phase 1] + assert verify_indexed_attestation(state, convert_to_indexed(state, attestation)) ``` ##### Deposits From 5fb32fd19b780d172e8738d887fd0675df7ec3ef Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Sun, 5 May 2019 12:10:39 +0100 Subject: [PATCH 040/308] Fix tests --- configs/constant_presets/mainnet.yaml | 4 ++-- configs/constant_presets/minimal.yaml | 4 ++-- specs/core/0_beacon-chain.md | 6 +++--- specs/core/1_custody-game.md | 16 ++++++++-------- specs/core/1_shard-data-chains.md | 16 ++++++++-------- specs/validator/0_beacon-chain-validator.md | 4 ++-- .../block_processing/test_process_attestation.py | 4 ++-- .../epoch_processing/test_process_crosslinks.py | 12 ++++++------ test_libs/pyspec/tests/helpers.py | 15 ++++++++++----- 9 files changed, 43 insertions(+), 38 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 72d0fdc8f2..6ac3f422f5 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -72,7 +72,7 @@ MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 # 2**11 (= 2,048) epochs 9 days PERSISTENT_COMMITTEE_PERIOD: 2048 # 2**6 (= 64) epochs ~7 hours -MAX_CROSSLINK_EPOCHS: 64 +MAX_EPOCHS_PER_CROSSLINK: 64 # 2**2 (= 4) epochs 25.6 minutes MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 @@ -124,4 +124,4 @@ DOMAIN_RANDAO: 1 DOMAIN_ATTESTATION: 2 DOMAIN_DEPOSIT: 3 DOMAIN_VOLUNTARY_EXIT: 4 -DOMAIN_TRANSFER: 5 \ No newline at end of file +DOMAIN_TRANSFER: 5 diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 0a6cab6872..caae4623ba 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -71,7 +71,7 @@ MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 # 2**11 (= 2,048) epochs 9 days PERSISTENT_COMMITTEE_PERIOD: 2048 # 2**6 (= 64) epochs ~7 hours -MAX_CROSSLINK_EPOCHS: 64 +MAX_EPOCHS_PER_CROSSLINK: 64 # 2**2 (= 4) epochs 25.6 minutes MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 @@ -123,4 +123,4 @@ DOMAIN_RANDAO: 1 DOMAIN_ATTESTATION: 2 DOMAIN_DEPOSIT: 3 DOMAIN_VOLUNTARY_EXIT: 4 -DOMAIN_TRANSFER: 5 \ No newline at end of file +DOMAIN_TRANSFER: 5 diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index addaa865ee..156a2523d2 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -206,10 +206,10 @@ These configurations are updated for releases, but may be out of sync during `de | `SLOTS_PER_HISTORICAL_ROOT` | `2**13` (= 8,192) | slots | ~13 hours | | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `2**8` (= 256) | epochs | ~27 hours | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | 9 days | -| `MAX_CROSSLINK_EPOCHS` | `2**6` (= 64) | epochs | ~7 hours | +| `MAX_EPOCHS_PER_CROSSLINK` | `2**6` (= 64) | epochs | ~7 hours | | `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `2**2` (= 4) | epochs | 25.6 minutes | -* `MAX_CROSSLINK_EPOCHS` should be a small constant times `SHARD_COUNT // SLOTS_PER_EPOCH` +* `MAX_EPOCHS_PER_CROSSLINK` should be a small constant times `SHARD_COUNT // SLOTS_PER_EPOCH` ### State list lengths @@ -1715,7 +1715,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Check FFG data, crosslink data, and signature assert ffg_data == (data.source_epoch, data.source_root, data.target_epoch) - assert data.crosslink.epoch == min(data.target_epoch, previous_crosslink.epoch + MAX_CROSSLINK_EPOCHS) + assert data.crosslink.epoch == min(data.target_epoch, previous_crosslink.epoch + MAX_EPOCHS_PER_CROSSLINK) assert data.crosslink.previous_crosslink_root == hash_tree_root(previous_crosslink) assert data.crosslink.crosslink_data_root == ZERO_HASH # [to be removed in phase 1] assert verify_indexed_attestation(state, convert_to_indexed(state, attestation)) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index d565266110..f386ee76e5 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -265,7 +265,7 @@ The `empty` function accepts and SSZ type as input and returns an object of that def get_custody_chunk_count(attestation: Attestation) -> int: crosslink_start_epoch = attestation.data.latest_crosslink.epoch crosslink_end_epoch = slot_to_epoch(attestation.data.slot) - crosslink_crosslink_length = min(MAX_CROSSLINK_EPOCHS, end_epoch - start_epoch) + crosslink_crosslink_length = min(MAX_EPOCHS_PER_CROSSLINK, end_epoch - start_epoch) chunks_per_epoch = 2 * BYTES_PER_SHARD_BLOCK * SLOTS_PER_EPOCH // BYTES_PER_CUSTODY_CHUNK return crosslink_crosslink_length * chunks_per_epoch ``` @@ -426,10 +426,10 @@ def process_early_derived_secret_reveal(state: BeaconState, # round key slash_validator(state, reveal.revealed_index, reveal.masker_index) else: - # Only a small penalty proportional to proposer slot reward for RANDAO reveal + # Only a small penalty proportional to proposer slot reward for RANDAO reveal # that does not interfere with the custody period - # The penalty is proportional to the max proposer reward - + # The penalty is proportional to the max proposer reward + # Calculate penalty max_proposer_slot_reward = ( get_base_reward(state, reveal.revealed_index) * @@ -448,7 +448,7 @@ def process_early_derived_secret_reveal(state: BeaconState, increase_balance(state, whistleblower_index, whistleblowing_reward - proposer_reward) decrease_balance(state, reveal.revealed_index, penalty) - # Mark this derived secret as exposed so validator cannot be punished repeatedly + # Mark this derived secret as exposed so validator cannot be punished repeatedly state.exposed_derived_secrets[reveal.epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS].append(reveal.revealed_index) ``` @@ -474,7 +474,7 @@ def process_chunk_challenge(state: BeaconState, # Verify the challenge is not a duplicate for record in state.custody_chunk_challenge_records: assert ( - record.crosslink_data_root != challenge.attestation.data.crosslink_data_root or + record.crosslink_data_root != challenge.attestation.data.crosslink.crosslink_data_root or record.chunk_index != challenge.chunk_index ) # Verify depth @@ -486,7 +486,7 @@ def process_chunk_challenge(state: BeaconState, challenger_index=get_beacon_proposer_index(state), responder_index=challenge.responder_index deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE, - crosslink_data_root=challenge.attestation.data.crosslink_data_root, + crosslink_data_root=challenge.attestation.data.crosslink.crosslink_data_root, depth=depth, chunk_index=challenge.chunk_index, ) @@ -564,7 +564,7 @@ def process_bit_challenge(state: BeaconState, challenger_index=challenge.challenger_index, responder_index=challenge.responder_index, deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE, - crosslink_data_root=challenge.attestation.data.crosslink_data_root, + crosslink_data_root=challenge.attestation.data.crosslink.crosslink_data_root, chunk_count=chunk_count, chunk_bits_merkle_root=merkle_root(pad_to_power_of_2((challenge.chunk_bits))), responder_key=challenge.responder_key, diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 33ef8632b0..673a64a4a7 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -215,7 +215,7 @@ def get_shard_header(block: ShardBlock) -> ShardBlockHeader: def verify_shard_attestation_signature(state: BeaconState, attestation: ShardAttestation) -> None: data = attestation.data - persistent_committee = get_persistent_committee(state, data.shard, data.slot) + persistent_committee = get_persistent_committee(state, data.crosslink.shard, data.slot) assert verify_bitfield(attestation.aggregation_bitfield, len(persistent_committee)) pubkeys = [] for i, index in enumerate(persistent_committee): @@ -225,7 +225,7 @@ def verify_shard_attestation_signature(state: BeaconState, pubkeys.append(validator.pubkey) assert bls_verify( pubkey=bls_aggregate_pubkeys(pubkeys), - message_hash=data.shard_block_root, + message_hash=data.crosslink.shard_block_root, signature=attestation.aggregate_signature, domain=get_domain(state, slot_to_epoch(data.slot), DOMAIN_SHARD_ATTESTER) ) @@ -312,7 +312,7 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], for _, attestation in enumerate(block.attestations): assert max(GENESIS_SHARD_SLOT, block.slot - SLOTS_PER_EPOCH) <= attestation.data.slot assert attestation.data.slot <= block.slot - MIN_ATTESTATION_INCLUSION_DELAY - assert attestation.data.shard == block.shard + assert attestation.data.crosslink.shard == block.shard verify_shard_attestation_signature(beacon_state, attestation) # Check signature @@ -343,11 +343,11 @@ def is_valid_shard_attestation(valid_shard_blocks: List[ShardBlock], # Check shard block shard_block = next( block for block in valid_shard_blocks if - signing_root(block) == candidate.attestation.data.shard_block_root + signing_root(block) == candidate.attestation.data.crosslink.shard_block_root , None) assert shard_block != None assert shard_block.slot == attestation.data.slot - assert shard_block.shard == attestation.data.shard + assert shard_block.shard == attestation.data.crosslink.shard # Check signature verify_shard_attestation_signature(beacon_state, attestation) @@ -382,18 +382,18 @@ def is_valid_beacon_attestation(shard: Shard, else: previous_attestation = next( attestation for attestation in valid_attestations if - attestation.data.crosslink_data_root == candidate.data.previous_crosslink.crosslink_data_root + attestation.data.crosslink.crosslink_data_root == candidate.data.previous_crosslink.crosslink_data_root , None) assert previous_attestation != None assert candidate.data.previous_attestation.epoch < slot_to_epoch(candidate.data.slot) # Check crosslink data root start_epoch = state.latest_crosslinks[shard].epoch - end_epoch = min(slot_to_epoch(candidate.data.slot) - CROSSLINK_LOOKBACK, start_epoch + MAX_CROSSLINK_EPOCHS) + end_epoch = min(slot_to_epoch(candidate.data.slot) - CROSSLINK_LOOKBACK, start_epoch + MAX_EPOCHS_PER_CROSSLINK) blocks = [] for slot in range(start_epoch * SLOTS_PER_EPOCH, end_epoch * SLOTS_PER_EPOCH): blocks.append(shard_blocks[slot]) - assert candidate.data.crosslink_data_root == compute_crosslink_data_root(blocks) + assert candidate.data.crosslink.crosslink_data_root == compute_crosslink_data_root(blocks) return True ``` diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ca7f0eb3b3..61f80f68e2 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -271,7 +271,7 @@ _Note:_ This can be looked up in the state using: ##### Shard -Set `attestation_data.shard = shard` where `shard` is the shard associated with the validator's committee defined by `get_crosslink_committees_at_slot`. +Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associated with the validator's committee defined by `get_crosslink_committees_at_slot`. ##### Previous crosslink root @@ -279,7 +279,7 @@ Set `attestation_data.previous_crosslink_root = hash_tree_root(head_state.curren ##### Crosslink data root -Set `attestation_data.crosslink_data_root = ZERO_HASH`. +Set `attestation_data.crosslink.crosslink_data_root = ZERO_HASH`. _Note:_ This is a stub for phase 0. diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index bcf71376ce..24cafc2755 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -113,7 +113,7 @@ def test_non_zero_crosslink_data_root(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.crosslink_data_root = b'\x42' * 32 + attestation.data.crosslink.crosslink_data_root = b'\x42' * 32 pre_state, post_state = run_attestation_processing(state, attestation, False) @@ -126,7 +126,7 @@ def test_bad_previous_crosslink(state): for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): next_slot(state) - state.current_crosslinks[attestation.data.shard].epoch += 10 + state.current_crosslinks[attestation.data.crosslink.shard].epoch += 10 pre_state, post_state = run_attestation_processing(state, attestation, False) diff --git a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py index d6765e3a72..29e7347b12 100644 --- a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py @@ -64,7 +64,7 @@ def test_single_crosslink_update_from_current_epoch(state): pre_state, post_state = run_process_crosslinks(state) - shard = attestation.data.shard + shard = attestation.data.crosslink.shard assert post_state.previous_crosslinks[shard] != post_state.current_crosslinks[shard] assert pre_state.current_crosslinks[shard] != post_state.current_crosslinks[shard] @@ -84,11 +84,11 @@ def test_single_crosslink_update_from_previous_epoch(state): pre_state, post_state = run_process_crosslinks(state) crosslink_deltas = get_crosslink_deltas(state) - shard = attestation.data.shard + shard = attestation.data.crosslink.shard assert post_state.previous_crosslinks[shard] != post_state.current_crosslinks[shard] assert pre_state.current_crosslinks[shard] != post_state.current_crosslinks[shard] # ensure rewarded - for index in get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.shard): + for index in get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.crosslink.shard): assert crosslink_deltas[0][index] > 0 assert crosslink_deltas[1][index] == 0 @@ -108,7 +108,7 @@ def test_double_late_crosslink(state): for slot in range(spec.SLOTS_PER_EPOCH): attestation_2 = get_valid_attestation(state) - if attestation_2.data.shard == attestation_1.data.shard: + if attestation_2.data.crosslink.shard == attestation_1.data.crosslink.shard: break next_slot(state) fill_aggregate_attestation(state, attestation_2) @@ -124,12 +124,12 @@ def test_double_late_crosslink(state): pre_state, post_state = run_process_crosslinks(state) crosslink_deltas = get_crosslink_deltas(state) - shard = attestation_2.data.shard + shard = attestation_2.data.crosslink.shard # ensure that the current crosslinks were not updated by the second attestation assert post_state.previous_crosslinks[shard] == post_state.current_crosslinks[shard] # ensure no reward, only penalties for the failed crosslink - for index in get_crosslink_committee(state, attestation_2.data.target_epoch, attestation_2.data.shard): + for index in get_crosslink_committee(state, attestation_2.data.target_epoch, attestation_2.data.crosslink.shard): assert crosslink_deltas[0][index] == 0 assert crosslink_deltas[1][index] > 0 diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 3b9b6904d5..1ea299453d 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -10,6 +10,7 @@ from eth2spec.phase0.spec import ( # constants ZERO_HASH, + MAX_EPOCHS_PER_CROSSLINK, # SSZ Attestation, AttestationData, @@ -17,6 +18,7 @@ AttesterSlashing, BeaconBlock, BeaconBlockHeader, + Crosslink, Deposit, DepositData, Eth1Data, @@ -174,14 +176,17 @@ def build_attestation_data(state, slot, shard): crosslinks = state.current_crosslinks if slot_to_epoch(slot) == get_current_epoch(state) else state.previous_crosslinks return AttestationData( - shard=shard, beacon_block_root=block_root, source_epoch=justified_epoch, source_root=justified_block_root, target_epoch=slot_to_epoch(slot), target_root=epoch_boundary_root, - crosslink_data_root=spec.ZERO_HASH, - previous_crosslink_root=hash_tree_root(crosslinks[shard]), + crosslink=Crosslink( + shard=shard, + epoch=min(slot_to_epoch(slot), crosslinks[shard].epoch + MAX_EPOCHS_PER_CROSSLINK), + crosslink_data_root=spec.ZERO_HASH, + previous_crosslink_root=hash_tree_root(crosslinks[shard]), + ), ) @@ -288,7 +293,7 @@ def get_valid_attestation(state, slot=None): attestation_data = build_attestation_data(state, slot, shard) - crosslink_committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.shard) + crosslink_committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) committee_size = len(crosslink_committee) bitfield_length = (committee_size + 7) // 8 @@ -381,7 +386,7 @@ def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0) def fill_aggregate_attestation(state, attestation): - crosslink_committee = get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.shard) + crosslink_committee = get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.crosslink.shard) for i in range(len(crosslink_committee)): attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i) From bf20031755804aa98ea7f0b597f698efac3e01a2 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 5 May 2019 17:15:05 +0100 Subject: [PATCH 041/308] Cosmetic genesis cleanups --- specs/core/0_beacon-chain.md | 43 +++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 36ac927961..df871dc608 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -93,7 +93,10 @@ - [Routines for updating validator status](#routines-for-updating-validator-status) - [`initiate_validator_exit`](#initiate_validator_exit) - [`slash_validator`](#slash_validator) - - [On genesis](#on-genesis) + - [Genesis](#genesis) + - [`Eth2Genesis`](#eth2genesis) + - [Genesis state](#genesis-state) + - [Genesis block](#genesis-block) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - [State caching](#state-caching) - [Per-epoch processing](#per-epoch-processing) @@ -1171,30 +1174,29 @@ def slash_validator(state: BeaconState, slashed_index: ValidatorIndex, whistlebl decrease_balance(state, slashed_index, whistleblowing_reward) ``` -## On genesis +## Genesis -When enough full deposits have been made to the deposit contract, an `Eth2Genesis` log is emitted. Construct a corresponding `genesis_state` and `genesis_block` as follows: +### `Eth2Genesis` -* Let `genesis_validator_deposits` be the list of deposits, ordered chronologically, up to and including the deposit that triggered the `Eth2Genesis` log. -* Let `genesis_time` be the timestamp specified in the `Eth2Genesis` log. -* Let `genesis_eth1_data` be the `Eth1Data` object where: - * `genesis_eth1_data.deposit_root` is the `deposit_root` contained in the `Eth2Genesis` log. - * `genesis_eth1_data.deposit_count` is the `deposit_count` contained in the `Eth2Genesis` log. - * `genesis_eth1_data.block_hash` is the hash of the Ethereum 1.0 block that emitted the `Eth2Genesis` log. -* Let `genesis_state = get_genesis_beacon_state(genesis_validator_deposits, genesis_time, genesis_eth1_data)`. -* Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. +When enough deposits of size `MAX_EFFECTIVE_BALANCE` have been made to the deposit contract an `Eth2Genesis` log is emitted triggering the genesis of the beacon chain. Let: + +* `eth2genesis` be the object corresponding to `Eth2Genesis` +* `genesis_eth1_data` be object of type `Eth1Data` where + * `genesis_eth1_data.deposit_root = eth2genesis.deposit_root` + * `genesis_eth1_data.deposit_count = eth2genesis.deposit_count` + * `genesis_eth1_data.block_hash` is the hash of the Ethereum 1.0 block that emitted the `Eth2Genesis` log +* `genesis_deposits` be the object of type `List[Deposit]` with deposits ordered chronologically up to and including the deposit that triggered the `Eth2Genesis` log + +### Genesis state + +Let `genesis_state = get_genesis_beacon_state(eth2genesis.genesis_time, genesis_eth1_data, genesis_deposits)`. ```python -def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], - genesis_time: int, - genesis_eth1_data: Eth1Data) -> BeaconState: - """ - Get the genesis ``BeaconState``. - """ +def get_genesis_beacon_state(genesis_time: int, eth1_data: Eth1Data, deposits: List[Deposit]) -> BeaconState: state = BeaconState(genesis_time=genesis_time, latest_eth1_data=genesis_eth1_data) # Process genesis deposits - for deposit in genesis_validator_deposits: + for deposit in deposits: process_deposit(state, deposit) # Process genesis activations @@ -1203,6 +1205,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], validator.activation_eligibility_epoch = GENESIS_EPOCH validator.activation_epoch = GENESIS_EPOCH + # Populate latest_active_index_roots genesis_active_index_root = hash_tree_root(get_active_validator_indices(state, GENESIS_EPOCH)) for index in range(LATEST_ACTIVE_INDEX_ROOTS_LENGTH): state.latest_active_index_roots[index] = genesis_active_index_root @@ -1210,6 +1213,10 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], return state ``` +### Genesis block + +Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. + ## Beacon chain state transition function We now define the state transition function. At a high level, the state transition is made up of four parts: From a3bbf20bf732b77eaddabad5a82f89b0c9f612c2 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 5 May 2019 17:17:18 +0100 Subject: [PATCH 042/308] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index df871dc608..4d3263a404 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1189,10 +1189,10 @@ When enough deposits of size `MAX_EFFECTIVE_BALANCE` have been made to the depos ### Genesis state -Let `genesis_state = get_genesis_beacon_state(eth2genesis.genesis_time, genesis_eth1_data, genesis_deposits)`. +Let `genesis_state = get_genesis_beacon_state(genesis_deposits, eth2genesis.genesis_time, genesis_eth1_data)`. ```python -def get_genesis_beacon_state(genesis_time: int, eth1_data: Eth1Data, deposits: List[Deposit]) -> BeaconState: +def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, eth1_data: Eth1Data) -> BeaconState: state = BeaconState(genesis_time=genesis_time, latest_eth1_data=genesis_eth1_data) # Process genesis deposits From 6aff277b1a344a2ecf024ef0fa4e6a126d0cdecf Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 5 May 2019 17:20:25 +0100 Subject: [PATCH 043/308] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4d3263a404..381ef867d8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1192,7 +1192,7 @@ When enough deposits of size `MAX_EFFECTIVE_BALANCE` have been made to the depos Let `genesis_state = get_genesis_beacon_state(genesis_deposits, eth2genesis.genesis_time, genesis_eth1_data)`. ```python -def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, eth1_data: Eth1Data) -> BeaconState: +def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis_eth1_data: Eth1Data) -> BeaconState: state = BeaconState(genesis_time=genesis_time, latest_eth1_data=genesis_eth1_data) # Process genesis deposits From 2c3bbac8034d9ad7eeff35d95618e032d7a04056 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 5 May 2019 19:30:55 +0100 Subject: [PATCH 044/308] Fix #1050 --- specs/core/0_beacon-chain.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 36ac927961..a60245732c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -63,7 +63,7 @@ - [`get_epoch_committee_count`](#get_epoch_committee_count) - [`get_shard_delta`](#get_shard_delta) - [`get_epoch_start_shard`](#get_epoch_start_shard) - - [`get_attestation_slot`](#get_attestation_slot) + - [`get_attestation_data_slot`](#get_attestation_data_slot) - [`get_block_root_at_slot`](#get_block_root_at_slot) - [`get_block_root`](#get_block_root) - [`get_randao_mix`](#get_randao_mix) @@ -759,14 +759,13 @@ def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: return shard ``` -### `get_attestation_slot` +### `get_attestation_data_slot` ```python -def get_attestation_slot(state: BeaconState, attestation: Attestation) -> Slot: - epoch = attestation.data.target_epoch - committee_count = get_epoch_committee_count(state, epoch) - offset = (attestation.data.shard + SHARD_COUNT - get_epoch_start_shard(state, epoch)) % SHARD_COUNT - return get_epoch_start_slot(epoch) + offset // (committee_count // SLOTS_PER_EPOCH) +def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot: + committee_count = get_epoch_committee_count(state, data.target_epoch) + offset = (data.shard + SHARD_COUNT - get_epoch_start_shard(state, data.target_epoch)) % SHARD_COUNT + return get_epoch_start_slot(data.target_epoch) + offset // (committee_count // SLOTS_PER_EPOCH) ``` ### `get_block_root_at_slot` @@ -1279,7 +1278,7 @@ def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> List[P def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> List[PendingAttestation]: return [ a for a in get_matching_source_attestations(state, epoch) - if a.data.beacon_block_root == get_block_root_at_slot(state, get_attestation_slot(state, a)) + if a.data.beacon_block_root == get_block_root_at_slot(state, get_attestation_data_slot(state, a.data)) ] ``` @@ -1702,11 +1701,11 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: """ Process ``Attestation`` operation. """ - attestation_slot = get_attestation_slot(state, attestation) + data = attestation.data + attestation_slot = get_attestation_data_slot(state, data) assert attestation_slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation_slot + SLOTS_PER_EPOCH # Check target epoch, source epoch, source root, and source crosslink - data = attestation.data assert (data.target_epoch, data.source_epoch, data.source_root, data.previous_crosslink_root) in { (get_current_epoch(state), state.current_justified_epoch, state.current_justified_root, hash_tree_root(state.current_crosslinks[data.shard])), (get_previous_epoch(state), state.previous_justified_epoch, state.previous_justified_root, hash_tree_root(state.previous_crosslinks[data.shard])), From c7fea5ff38b260fccbe9416d5e0fd769884e5343 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 6 May 2019 12:28:16 +1000 Subject: [PATCH 045/308] Update libp2p-standardization based on latest comments --- specs/networking/libp2p-standardization.md | 65 ++++++++-------------- 1 file changed, 24 insertions(+), 41 deletions(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index c5add549a9..dec337f39d 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -7,7 +7,7 @@ Ethereum 2.0 clients plan to use the libp2p protocol networking stack for mainnet release. This document aims to standardize the libp2p client protocols, configuration and messaging formats. -# Libp2p Protocols +# Libp2p Components ## Transport @@ -19,8 +19,11 @@ TCP/IP and optionally websockets. Websockets are useful for implementations running in the browser and therefore native clients would ideally support these implementations by supporting websockets. -An ideal libp2p transport would therefore be TCP/IP with a fallback to -websockets. +An ideal libp2p transport would therefore support both TCP/IP and websockets. + +*Note: There is active development in libp2p to facilitate the +[QUIC](https://github.com/libp2p/go-libp2p-quic-transport) transport, which may +be adopted in the future* ### Encryption @@ -35,6 +38,11 @@ Current defaults are: - Cipher: `AES-128` (also supports `AES-256`, `TwofishCTR`) - Digests: `SHA256` (also supports `SHA512`) +*Note: Secio is being deprecated in favour of [TLS +1.3](https://github.com/libp2p/specs/blob/master/tls/tls.md). It is our +intention to transition to use TLS 1.3 for encryption between nodes, rather +than Secio.* + ## Protocols @@ -45,7 +53,8 @@ running a libp2p network stack. #### Protocol id: `/multistream/1.0.0` -Clients running libp2p should support the [multistream-select](https://github.com/multiformats/multistream-select/) +Clients running libp2p should support the +[multistream-select](https://github.com/multiformats/multistream-select/) protocol which allows clients to negotiate libp2p protocols establish streams per protocol. @@ -62,7 +71,7 @@ optionally [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md) ## Gossipsub -#### Protocol id: `/meshsub/1.0.0` +#### Protocol id: `/eth/serenity/gossipsub/1.0.0` *Note: Parameters listed here are subject to a large-scale network feasibility study* @@ -121,36 +130,14 @@ number of shard subnets will be defined via `SHARD_SUBNET_COUNT` and the shard ### Messages -#### Libp2p Specification - -*This section simply outlines the data sent across the wire as specified by -libp2p - this section is aimed at gossipsub implementers to standardize their implementation of this protocol* - -Libp2p raw gossipsub messages are sent across the wire as fixed-size length-prefixed byte arrays. - -The byte array is prefixed with an unsigned 64 bit length number encoded as an -`unsigned varint` (https://github.com/multiformats/unsigned-varint). Gossipsub messages therefore take the form: -``` -+--------------------------+ -| message length | -+--------------------------+ -| | -| body (<1M) | -| | -+--------------------------+ -``` +*Note: The message format here is Eth2.0-specific* -The body represents a protobuf-encoded [Message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24). +Each Gossipsub +[Message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24) +has a maximum size of 512KB (estimated from expected largest uncompressed block +size). -In the following section we discuss the data being sent in the `data` field of -the protobuf gossipsub `Message`. - -#### Eth2.0 Specifics - -Each message has a maximum size of 512KB (estimated from expected largest uncompressed -block size). - -The `data` that is sent in a Gossipsub message is an SSZ-encoded object. For the `beacon_block` topic, +The `data` field of a Gossipsub `Message` is an SSZ-encoded object. For the `beacon_block` topic, this will be a `beacon_block`. For the `beacon_attestation` topic, this will be an `attestation`. @@ -162,10 +149,10 @@ The [RPC Interface](./rpc-interface.md) is specified in this repository. ## Identify -#### Protocol Id: `/ipfs/id/1.0.0` (to be updated to `/p2p/id/1.0.0`) - -*To be updated to incorporate discv5* +**Note: This protocol is a placeholder and will be updated once the discv5 +discovery protocol is added to this document** +#### Protocol Id: `/eth/serentiy/id/1.0.0` The Identify protocol (defined in go - [identify-go](https://github.com/ipfs/go-ipfs/blob/master/core/commands/id.go) and rust [rust-identify](https://github.com/libp2p/rust-libp2p/blob/master/protocols/identify/src/lib.rs)) allows a node A to query another node B which information B knows about A. This also includes the addresses B is listening on. @@ -185,8 +172,4 @@ type of connecting node. Suggested format: ## Discovery -#### Protocol Id: `/eth/serenity/disc/1.0.0` - -*To be updated to incorporate discv5* - -The discovery protocol to be determined. +**To be updated to incorporate discv5** From 7d0a6191edade473053cec49ce560ea253a220fd Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 6 May 2019 09:09:37 -0600 Subject: [PATCH 046/308] simplify justificatio/finality notes at genesis --- specs/core/0_fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index cb11ea8606..8ec0a604cc 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -106,4 +106,4 @@ def lmd_ghost(store: Store, start_state: BeaconState, start_block: BeaconBlock) ### Justification and finality at genesis -Clients may choose to refer to the justification and finality data in a given `BeaconState` to determine the finalized/justified head. During the early epochs right after genesis, the justification and finality data are not stored in the `BeaconState`. In their place are "empty" values; for example, the 32-byte zero value as the `finalized_root`. Clients wishing to compute the fork choice in these early epochs should work around this fact of the `BeaconState` to recognize that the genesis epoch and root of the genesis block are _both_ the finalized and justified heads until updated via the state transition function defined in [Phase 0 -- The Beacon Chain](./0_beacon-chain.md). Solutions will be language-specific but one possibility is to treat the zero-value hash as an exceptional case that references the genesis block/epoch. +During genesis, justification and finality root fields within the `BeaconState` reference `ZERO_HASH` rather than a known block. `ZERO_HASH` in `previous_justified_root`, `current_justified_root`, and `finalized_root` should be considered as an alias to the root of the genesis block. From d1c96c1e0d3b97ac6b436cbaa070e4a39f6b5876 Mon Sep 17 00:00:00 2001 From: JSON <49416440+JSON@users.noreply.github.com> Date: Mon, 6 May 2019 10:30:32 -0500 Subject: [PATCH 047/308] Doc standardization (#1039) --- README.md | 8 +-- configs/constant_presets/README.md | 6 +- configs/fork_timelines/README.md | 7 ++- specs/bls_signature.md | 2 +- specs/core/0_beacon-chain.md | 52 ++++++++--------- specs/core/0_deposit-contract.md | 14 ++--- specs/core/0_fork-choice.md | 18 +++--- specs/core/1_custody-game.md | 36 ++++++------ specs/core/1_shard-data-chains.md | 14 ++--- specs/light_client/merkle_proofs.md | 23 ++++---- specs/light_client/sync_protocol.md | 6 +- specs/networking/messaging.md | 23 ++++---- specs/networking/node-identification.md | 13 ++--- specs/networking/rpc-interface.md | 59 ++++++++++--------- specs/simple-serialize.md | 39 +++++++------ specs/test_formats/README.md | 64 ++++++++++++--------- specs/test_formats/bls/README.md | 4 +- specs/test_formats/shuffling/README.md | 12 ++-- specs/test_formats/ssz_generic/README.md | 8 +-- specs/test_formats/ssz_static/README.md | 2 +- specs/test_formats/ssz_static/core.md | 8 +-- specs/validator/0_beacon-chain-validator.md | 60 +++++++++---------- test_generators/README.md | 14 ++--- test_libs/pyspec/README.md | 16 +++--- 24 files changed, 262 insertions(+), 246 deletions(-) diff --git a/README.md b/README.md index 27ff7fcb1e..4c4808059f 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,14 @@ [![Join the chat at https://gitter.im/ethereum/sharding](https://badges.gitter.im/ethereum/sharding.svg)](https://gitter.im/ethereum/sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -To learn more about sharding and eth2.0/Serenity, see the [sharding FAQ](https://github.com/ethereum/wiki/wiki/Sharding-FAQ) and the [research compendium](https://notes.ethereum.org/s/H1PGqDhpm). +To learn more about sharding and Ethereum 2.0 (Serenity), see the [sharding FAQ](https://github.com/ethereum/wiki/wiki/Sharding-FAQ) and the [research compendium](https://notes.ethereum.org/s/H1PGqDhpm). -This repo hosts the current eth2.0 specifications. Discussions about design rationale and proposed changes can be brought up and discussed as issues. Solidified, agreed upon changes to spec can be made through pull requests. +This repository hosts the current Eth 2.0 specifications. Discussions about design rationale and proposed changes can be brought up and discussed as issues. Solidified, agreed-upon changes to the spec can be made through pull requests. ## Specs -Core specifications for eth2.0 client validation can be found in [specs/core](specs/core). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are: +Core specifications for Eth 2.0 client validation can be found in [specs/core](specs/core). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are: ### Phase 0 * [The Beacon Chain](specs/core/0_beacon-chain.md) @@ -30,7 +30,7 @@ Core specifications for eth2.0 client validation can be found in [specs/core](sp * [Light client syncing protocol](specs/light_client/sync_protocol.md) -### Design goals +## Design goals The following are the broad design goals for Ethereum 2.0: * to minimize complexity, even at the cost of some losses in efficiency diff --git a/configs/constant_presets/README.md b/configs/constant_presets/README.md index 45148862e7..61c9a3a630 100644 --- a/configs/constant_presets/README.md +++ b/configs/constant_presets/README.md @@ -10,11 +10,11 @@ Later-fork constants can be ignored, e.g. ignore phase1 constants as a client th Each preset is a key-value mapping. **Key**: an `UPPER_SNAKE_CASE` (a.k.a. "macro case") formatted string, name of the constant. -**Value**: can be any of: + +**Value** can be either: - an unsigned integer number, can be up to 64 bits (incl.) - a hexadecimal string, prefixed with `0x` Presets may contain comments to describe the values. -See `mainnet.yaml` for a complete example. - +See [`mainnet.yaml`](./mainnet.yaml) for a complete example. diff --git a/configs/fork_timelines/README.md b/configs/fork_timelines/README.md index c93b415f50..da7445767c 100644 --- a/configs/fork_timelines/README.md +++ b/configs/fork_timelines/README.md @@ -3,16 +3,17 @@ This directory contains a set of fork timelines used for testing, testnets, and mainnet. A timeline file contains all the forks known for its target. -Later forks can be ignored, e.g. ignore fork `phase1` as a client that only supports phase 0 currently. +Later forks can be ignored, e.g. ignore fork `phase1` as a client that only supports Phase 0 currently. ## Format Each preset is a key-value mapping. **Key**: an `lower_snake_case` (a.k.a. "python case") formatted string, name of the fork. -**Value**: an unsigned integer number, epoch number of activation of the fork + +**Value**: an unsigned integer number, epoch number of activation of the fork. Timelines may contain comments to describe the values. -See `mainnet.yaml` for a complete example. +See [`mainnet.yaml`](./mainnet.yaml) for a complete example. diff --git a/specs/bls_signature.md b/specs/bls_signature.md index 18e2d8c9a0..d119c4499c 100644 --- a/specs/bls_signature.md +++ b/specs/bls_signature.md @@ -118,7 +118,7 @@ Let `bls_aggregate_signatures(signatures: List[Bytes96]) -> Bytes96` return `sig ## Signature verification -In the following `e` is the pairing function and `g` is the G1 generator with the following coordinates (see [here](https://github.com/zkcrypto/pairing/tree/master/src/bls12_381#g1)): +In the following, `e` is the pairing function and `g` is the G1 generator with the following coordinates (see [here](https://github.com/zkcrypto/pairing/tree/master/src/bls12_381#g1)): ```python g_x = 3685416753713387016781088315183077757961620795782546409894578378688607592378376318836054947676345821548104185464507 diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 36ac927961..7e135d0d0b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1,6 +1,6 @@ # Ethereum 2.0 Phase 0 -- The Beacon Chain -**NOTICE**: This document is a work in progress for researchers and implementers. +**Notice**: This document is a work-in-progress for researchers and implementers. ## Table of contents @@ -45,7 +45,7 @@ - [`BeaconBlock`](#beaconblock) - [Beacon state](#beacon-state) - [`BeaconState`](#beaconstate) - - [Custom Types](#custom-types) + - [Custom types](#custom-types) - [Helper functions](#helper-functions) - [`xor`](#xor) - [`hash`](#hash) @@ -134,25 +134,25 @@ Code snippets appearing in `this style` are to be interpreted as Python code. ## Terminology -* **Validator** - a registered participant in the beacon chain. You can become one by sending ether into the Ethereum 1.0 deposit contract. -* **Active validator** - an active participant in the Ethereum 2.0 consensus invited to, among other things, propose and attest to blocks and vote for crosslinks. -* **Committee** - a (pseudo-) randomly sampled subset of [active validators](#dfn-active-validator). When a committee is referred to collectively, as in "this committee attests to X", this is assumed to mean "some subset of that committee that contains enough [validators](#dfn-validator) that the protocol recognizes it as representing the committee". -* **Proposer** - the [validator](#dfn-validator) that creates a beacon chain block. -* **Attester** - a [validator](#dfn-validator) that is part of a committee that needs to sign off on a beacon chain block while simultaneously creating a link (crosslink) to a recent shard block on a particular shard chain. -* **Beacon chain** - the central PoS chain that is the base of the sharding system. -* **Shard chain** - one of the chains on which user transactions take place and account data is stored. -* **Block root** - a 32-byte Merkle root of a beacon chain block or shard chain block. Previously called "block hash". -* **Crosslink** - a set of signatures from a committee attesting to a block in a shard chain that can be included into the beacon chain. Crosslinks are the main means by which the beacon chain "learns about" the updated state of shard chains. -* **Slot** - a period during which one proposer has the ability to create a beacon chain block and some attesters have the ability to make attestations. -* **Epoch** - an aligned span of slots during which all [validators](#dfn-validator) get exactly one chance to make an attestation. -* **Finalized**, **justified** - see the [Casper FFG paper](https://arxiv.org/abs/1710.09437). -* **Withdrawal period** - the number of slots between a [validator](#dfn-validator) exit and the [validator](#dfn-validator) balance being withdrawable. -* **Genesis time** - the Unix time of the genesis beacon chain block at slot 0. +* **Validator**—a registered participant in the beacon chain. You can become one by sending ether into the Ethereum 1.0 deposit contract. +* **Active validator**—an active participant in the Ethereum 2.0 consensus invited to, among other things, propose and attest to blocks and vote for crosslinks. +* **Committee**—a (pseudo-) randomly sampled subset of [active validators](#dfn-active-validator). When a committee is referred to collectively, as in "this committee attests to X", this is assumed to mean "some subset of that committee that contains enough [validators](#dfn-validator) that the protocol recognizes it as representing the committee". +* **Proposer**—the [validator](#dfn-validator) that creates a beacon chain block. +* **Attester**—a [validator](#dfn-validator) that is part of a committee that needs to sign off on a beacon chain block while simultaneously creating a link (crosslink) to a recent shard block on a particular shard chain. +* **Beacon chain**—the central PoS chain that is the base of the sharding system. +* **Shard chain**—one of the chains on which user transactions take place and account data is stored. +* **Block root**—a 32-byte Merkle root of a beacon chain block or shard chain block. Previously called "block hash". +* **Crosslink**—a set of signatures from a committee attesting to a block in a shard chain that can be included into the beacon chain. Crosslinks are the main means by which the beacon chain "learns about" the updated state of shard chains. +* **Slot**—a period during which one proposer has the ability to create a beacon chain block and some attesters have the ability to make attestations. +* **Epoch**—an aligned span of slots during which all [validators](#dfn-validator) get exactly one chance to make an attestation. +* **Finalized**, **justified**—see the [Casper FFG paper](https://arxiv.org/abs/1710.09437). +* **Withdrawal period**—the number of slots between a [validator](#dfn-validator) exit and the [validator](#dfn-validator) balance being withdrawable. +* **Genesis time**—the Unix time of the genesis beacon chain block at slot 0. ## Constants -Note: the default mainnet values for the constants are included here for spec-design purposes. -The different configurations for mainnet, testnets, and yaml-based testing can be found in the `configs/constant_presets/` directory. +*Note*: The default mainnet values for the constants are included here for spec-design purposes. +The different configurations for mainnet, testnets, and YAML-based testing can be found in the `configs/constant_presets/` directory. These configurations are updated for releases, but may be out of sync during `dev` changes. ### Misc @@ -165,7 +165,7 @@ These configurations are updated for releases, but may be out of sync during `de | `MIN_PER_EPOCH_CHURN_LIMIT` | `2**2` (= 4) | | `CHURN_LIMIT_QUOTIENT` | `2**16` (= 65,536) | | `BASE_REWARDS_PER_EPOCH` | `5` | -| `SHUFFLE_ROUND_COUNT` | 90 | +| `SHUFFLE_ROUND_COUNT` | `90` | * For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) @@ -229,7 +229,7 @@ These configurations are updated for releases, but may be out of sync during `de | `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | | `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | -* **The `BASE_REWARD_QUOTIENT` is NOT final. Once all other protocol details are finalized it will be adjusted, to target a theoretical maximum total issuance of `2**21` ETH per year if `2**27` ETH is validating (and therefore `2**20` per year if `2**25` ETH is validating, etc etc)** +* **The `BASE_REWARD_QUOTIENT` is NOT final. Once all other protocol details are finalized, it will be adjusted to target a theoretical maximum total issuance of `2**21` ETH per year if `2**27` ETH is validating (and therefore `2**20` per year if `2**25` ETH is validating, etc.)** * The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (~18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating [validators](#dfn-validator) to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline [validators](#dfn-validator) after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)` so after `INVERSE_SQRT_E_DROP_TIME` epochs it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. ### Max operations per block @@ -587,7 +587,7 @@ The types are defined topologically to aid in facilitating an executable version } ``` -## Custom Types +## Custom types We define the following Python custom types for type hinting and readability: @@ -604,7 +604,7 @@ We define the following Python custom types for type hinting and readability: ## Helper functions -Note: The definitions below are for specification purposes and are not necessarily optimal implementations. +*Note*: The definitions below are for specification purposes and are not necessarily optimal implementations. ### `xor` @@ -617,7 +617,7 @@ def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: The `hash` function is SHA256. -Note: We aim to migrate to a S[T/N]ARK-friendly hash function in a future Ethereum 2.0 deployment phase. +*Note*: We aim to migrate to a S[T/N]ARK-friendly hash function in a future Ethereum 2.0 deployment phase. ### `hash_tree_root` @@ -1121,7 +1121,7 @@ def get_churn_limit(state: BeaconState) -> int: ### Routines for updating validator status -Note: All functions in this section mutate `state`. +*Note*: All functions in this section mutate `state`. #### `initiate_validator_exit` @@ -1227,7 +1227,7 @@ Transition section notes: Beacon blocks that trigger unhandled Python exceptions (e.g. out-of-range list accesses) and failed `assert`s during the state transition are considered invalid. -Note: If there are skipped slots between a block and its parent block, run the steps in the [state-root](#state-caching), [per-epoch](#per-epoch-processing), and [per-slot](#per-slot-processing) sections once for each skipped slot and then once for the slot containing the new block. +*Note*: If there are skipped slots between a block and its parent block, run the steps in the [state-root](#state-caching), [per-epoch](#per-epoch-processing), and [per-slot](#per-slot-processing) sections once for each skipped slot and then once for the slot containing the new block. ### State caching @@ -1634,7 +1634,7 @@ def process_eth1_data(state: BeaconState, block: BeaconBlock) -> None: #### Operations -Note: All functions in this section mutate `state`. +*Note*: All functions in this section mutate `state`. ##### Proposer slashings diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index 6843e407ec..917adaecb1 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -1,6 +1,6 @@ # Ethereum 2.0 Phase 0 -- Deposit Contract -**NOTICE**: This document is a work in progress for researchers and implementers. +**Notice**: This document is a work-in-progress for researchers and implementers. ## Table of contents @@ -24,7 +24,7 @@ ## Introduction -This document represents is the specification for the beacon chain deposit contract, part of Ethereum 2.0 phase 0. +This document represents the specification for the beacon chain deposit contract, part of Ethereum 2.0 Phase 0. ## Constants @@ -40,11 +40,11 @@ This document represents is the specification for the beacon chain deposit contr | - | - | | `DEPOSIT_CONTRACT_ADDRESS` | **TBD** | | `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | -| `CHAIN_START_FULL_DEPOSIT_THRESHOLD` | `2**16` (=65,536) | +| `CHAIN_START_FULL_DEPOSIT_THRESHOLD` | `2**16` (= 65,536) | ## Ethereum 1.0 deposit contract -The initial deployment phases of Ethereum 2.0 are implemented without consensus changes to Ethereum 1.0. A deposit contract at address `DEPOSIT_CONTRACT_ADDRESS` is added to Ethereum 1.0 for deposits of ETH to the beacon chain. Validator balances will be withdrawable to the shards in phase 2, i.e. when the EVM2.0 is deployed and the shards have state. +The initial deployment phases of Ethereum 2.0 are implemented without consensus changes to Ethereum 1.0. A deposit contract at address `DEPOSIT_CONTRACT_ADDRESS` is added to Ethereum 1.0 for deposits of ETH to the beacon chain. Validator balances will be withdrawable to the shards in Phase 2 (i.e. when the EVM 2.0 is deployed and the shards have state). ### Arguments @@ -52,7 +52,7 @@ The deposit contract has a `deposit` function which takes the amount in Ethereum #### Withdrawal credentials -One of the `DepositData` fields is `withdrawal_credentials`. It is a commitment to credentials for withdrawals to shards. The first byte of `withdrawal_credentials` is a version number. As of now the only expected format is as follows: +One of the `DepositData` fields is `withdrawal_credentials`. It is a commitment to credentials for withdrawals to shards. The first byte of `withdrawal_credentials` is a version number. As of now, the only expected format is as follows: * `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX_BYTE` * `withdrawal_credentials[1:] == hash(withdrawal_pubkey)[1:]` where `withdrawal_pubkey` is a BLS pubkey @@ -84,10 +84,10 @@ When `CHAIN_START_FULL_DEPOSIT_THRESHOLD` of full deposits have been made, the d The source for the Vyper contract lives in a [separate repository](https://github.com/ethereum/deposit_contract) at [https://github.com/ethereum/deposit_contract/blob/master/deposit_contract/contracts/validator_registration.v.py](https://github.com/ethereum/deposit_contract/blob/master/deposit_contract/contracts/validator_registration.v.py). -Note: to save ~10x on gas this contract uses a somewhat unintuitive progressive Merkle root calculation algo that requires only O(log(n)) storage. See https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py for an implementation of the same algo in python tested for correctness. +*Note*: To save ~10x on gas, this contract uses a somewhat unintuitive progressive Merkle root calculation algo that requires only O(log(n)) storage. See https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py for an implementation of the same algo in Python tested for correctness. For convenience, we provide the interface to the contract here: * `__init__()`: initializes the contract * `get_deposit_root() -> bytes32`: returns the current root of the deposit tree -* `deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])`: adds a deposit instance to the deposit tree, incorporating the input arguments and the value transferred in the given call. Note: the amount of value transferred *must* be at least `MIN_DEPOSIT_AMOUNT`. Each of these constants are specified in units of Gwei. +* `deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])`: adds a deposit instance to the deposit tree, incorporating the input arguments and the value transferred in the given call. *Note*: The amount of value transferred *must* be at least `MIN_DEPOSIT_AMOUNT`. Each of these constants are specified in units of Gwei. diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 23534ec4e0..549e9e2077 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -1,6 +1,6 @@ # Ethereum 2.0 Phase 0 -- Beacon Chain Fork Choice -**NOTICE**: This document is a work in progress for researchers and implementers. +**Notice**: This document is a work-in-progress for researchers and implementers. ## Table of contents @@ -20,7 +20,7 @@ ## Introduction -This document represents is the specification for the beacon chain fork choice rule, part of Ethereum 2.0 phase 0. +This document represents the specification for the beacon chain fork choice rule, part of Ethereum 2.0 Phase 0. ## Prerequisites @@ -42,17 +42,17 @@ Processing the beacon chain is similar to processing the Ethereum 1.0 chain. Cli * An Ethereum 1.0 block pointed to by the `state.latest_eth1_data.block_hash` has been processed and accepted. * The node's Unix time is greater than or equal to `state.genesis_time + block.slot * SECONDS_PER_SLOT`. -Note: Leap seconds mean that slots will occasionally last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds, possibly several times a year. +*Note*: Leap seconds mean that slots will occasionally last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds, possibly several times a year. -Note: Nodes needs to have a clock that is roughly (i.e. within `SECONDS_PER_SLOT` seconds) synchronized with the other nodes. +*Note*: Nodes needs to have a clock that is roughly (i.e. within `SECONDS_PER_SLOT` seconds) synchronized with the other nodes. ### Beacon chain fork choice rule -The beacon chain fork choice rule is a hybrid that combines justification and finality with Latest Message Driven (LMD) Greediest Heaviest Observed SubTree (GHOST). At any point in time a validator `v` subjectively calculates the beacon chain head as follows. +The beacon chain fork choice rule is a hybrid that combines justification and finality with Latest Message Driven (LMD) Greediest Heaviest Observed SubTree (GHOST). At any point in time, a validator `v` subjectively calculates the beacon chain head as follows. -* Abstractly define `Store` as the type of storage object for the chain data and `store` be the set of attestations and blocks that the validator `v` has observed and verified (in particular, block ancestors must be recursively verified). Attestations not yet included in any chain are still included in `store`. -* Let `finalized_head` be the finalized block with the highest epoch. (A block `B` is finalized if there is a descendant of `B` in `store` the processing of which sets `B` as finalized.) -* Let `justified_head` be the descendant of `finalized_head` with the highest epoch that has been justified for at least 1 epoch. (A block `B` is justified if there is a descendant of `B` in `store` the processing of which sets `B` as justified.) If no such descendant exists set `justified_head` to `finalized_head`. +* Abstractly define `Store` as the type of storage object for the chain data, and let `store` be the set of attestations and blocks that the validator `v` has observed and verified (in particular, block ancestors must be recursively verified). Attestations not yet included in any chain are still included in `store`. +* Let `finalized_head` be the finalized block with the highest epoch. (A block `B` is finalized if there is a descendant of `B` in `store`, the processing of which sets `B` as finalized.) +* Let `justified_head` be the descendant of `finalized_head` with the highest epoch that has been justified for at least 1 epoch. (A block `B` is justified if there is a descendant of `B` in `store` the processing of which sets `B` as justified.) If no such descendant exists, set `justified_head` to `finalized_head`. * Let `get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock` be the ancestor of `block` with slot number `slot`. The `get_ancestor` function can be defined recursively as: ```python @@ -70,7 +70,7 @@ def get_ancestor(store: Store, block: BeaconBlock, slot: Slot) -> BeaconBlock: * Let `get_latest_attestation(store: Store, index: ValidatorIndex) -> Attestation` be the attestation with the highest slot number in `store` from the validator with the given `index`. If several such attestations exist, use the one the validator `v` observed first. * Let `get_latest_attestation_target(store: Store, index: ValidatorIndex) -> BeaconBlock` be the target block in the attestation `get_latest_attestation(store, index)`. -* Let `get_children(store: Store, block: BeaconBlock) -> List[BeaconBlock]` returns the child blocks of the given `block`. +* Let `get_children(store: Store, block: BeaconBlock) -> List[BeaconBlock]` return the child blocks of the given `block`. * Let `justified_head_state` be the resulting `BeaconState` object from processing the chain up to the `justified_head`. * The `head` is `lmd_ghost(store, justified_head_state, justified_head)` where the function `lmd_ghost` is defined below. Note that the implementation below is suboptimal; there are implementations that compute the head in time logarithmic in slot count. diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index d565266110..b3ab519fa0 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -1,6 +1,6 @@ # Ethereum 2.0 Phase 1 -- Custody Game -**NOTICE**: This spec is a work-in-progress for researchers and implementers. +**Notice**: This document is a work-in-progress for researchers and implementers. ## Table of contents @@ -23,7 +23,7 @@ - [`CustodyChunkChallengeRecord`](#custodychunkchallengerecord) - [`CustodyBitChallengeRecord`](#custodybitchallengerecord) - [`CustodyResponse`](#custodyresponse) - - [New Beacon operations](#new-beacon-operations) + - [New beacon operations](#new-beacon-operations) - [`CustodyKeyReveal`](#custodykeyreveal) - [`EarlyDerivedSecretReveal`](#earlyderivedsecretreveal) - [Phase 0 container updates](#phase-0-container-updates) @@ -53,22 +53,22 @@ ## Introduction -This document details the beacon chain additions and changes in Phase 1 of Ethereum 2.0 to support the shard data custody game, building upon the [phase 0](0_beacon-chain.md) specification. +This document details the beacon chain additions and changes in Phase 1 of Ethereum 2.0 to support the shard data custody game, building upon the [Phase 0](0_beacon-chain.md) specification. ## Terminology -* **Custody game**: -* **Custody period**: -* **Custody chunk**: -* **Custody chunk bit**: -* **Custody chunk challenge**: -* **Custody bit**: -* **Custody bit challenge**: -* **Custody key**: -* **Custody key reveal**: -* **Custody key mask**: -* **Custody response**: -* **Custody response deadline**: +* **Custody game**— +* **Custody period**— +* **Custody chunk**— +* **Custody chunk bit**— +* **Custody chunk challenge**— +* **Custody bit**— +* **Custody bit challenge**— +* **Custody key**— +* **Custody key reveal**— +* **Custody key mask**— +* **Custody response**— +* **Custody response deadline**— ## Constants @@ -181,7 +181,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether } ``` -### New Beacon operations +### New beacon operations #### `CustodyKeyReveal` @@ -220,7 +220,7 @@ Add the following fields to the end of the specified container objects. Fields w #### `Validator` ```python - # next_custody_reveal_period is initialised to the custody period + # next_custody_reveal_period is initialized to the custody period # (of the particular validator) in which the validator is activated # = get_validators_custody_reveal_period(...) 'next_custody_reveal_period': 'uint64', @@ -330,7 +330,7 @@ def replace_empty_or_append(list: List[Any], new_element: Any) -> int: ### Operations -Add the following operations to the per-block processing, in order the given below and after all other operations in phase 0. +Add the following operations to the per-block processing, in the order given below and after all other operations in Phase 0. #### Custody key reveals diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 33ef8632b0..14eb511931 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -1,13 +1,13 @@ # Ethereum 2.0 Phase 1 -- Shard Data Chains -**NOTICE**: This document is a work-in-progress for researchers and implementers. +**Notice**: This document is a work-in-progress for researchers and implementers. -## Table of Contents +## Table of contents -- [Ethereum 2.0 Phase 1 -- Shards Data Chains](#ethereum-20-phase-1----shard-data-chains) - - [Table of Contents](#table-of-contents) +- [Ethereum 2.0 Phase 1 -- Shard Data Chains](#ethereum-20-phase-1----shard-data-chains) + - [Table of contents](#table-of-contents) - [Introduction](#introduction) - [Constants](#constants) - [Misc](#misc) @@ -53,8 +53,8 @@ This document describes the shard data layer and the shard fork choice rule in P | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `CROSSLINK_LOOKBACK` | 2**0 (= 1) | epochs | 6.2 minutes | -| `PERSISTENT_COMMITTEE_PERIOD` | 2**11 (= 2,048) | epochs | ~9 days | +| `CROSSLINK_LOOKBACK` | `2**0` (= 1) | epochs | 6.2 minutes | +| `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | ### Signature domains @@ -363,7 +363,7 @@ Let: * `shard_blocks` be the `ShardBlock` list such that `shard_blocks[slot]` is the canonical `ShardBlock` for shard `shard` at slot `slot` * `beacon_state` be the canonical `BeaconState` * `valid_attestations` be the list of valid `Attestation`, recursively defined -* `candidate` be a candidate `Attestation` which is valid under phase 0 rules, and for which validity is to be determined under phase 1 rules by running `is_valid_beacon_attestation` +* `candidate` be a candidate `Attestation` which is valid under Phase 0 rules, and for which validity is to be determined under Phase 1 rules by running `is_valid_beacon_attestation` ```python def is_valid_beacon_attestation(shard: Shard, diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index b38167bb5f..f009d97375 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -1,16 +1,19 @@ -**NOTICE**: This document is a work-in-progress for researchers and implementers. +# Merkle proof formats -## Table of Contents +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents -- [Table of Contents](#table-of-contents) -- [Constants](#constants) -- [Generalized Merkle tree index](#generalized-merkle-tree-index) -- [SSZ object to index](#ssz-object-to-index) -- [Merkle multiproofs](#merkle-multiproofs) -- [MerklePartial](#merklepartial) - - [`SSZMerklePartial`](#sszmerklepartial) - - [Proofs for execution](#proofs-for-execution) +- [Merkle proof formats](#merkle-proof-formats) + - [Table of contents](#table-of-contents) + - [Constants](#constants) + - [Generalized Merkle tree index](#generalized-merkle-tree-index) + - [SSZ object to index](#ssz-object-to-index) + - [Merkle multiproofs](#merkle-multiproofs) + - [MerklePartial](#merklepartial) + - [`SSZMerklePartial`](#sszmerklepartial) + - [Proofs for execution](#proofs-for-execution) diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index 7cb1f69287..f6e3d22657 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -1,13 +1,13 @@ # Beacon Chain Light Client Syncing -__NOTICE__: This document is a work-in-progress for researchers and implementers. One of the design goals of the eth2 beacon chain is light-client friendliness, both to allow low-resource clients (mobile phones, IoT, etc) to maintain access to the blockchain in a reasonably safe way, but also to facilitate the development of "bridges" between the eth2 beacon chain and other chains. +**Notice**: This document is a work-in-progress for researchers and implementers. One of the design goals of the Eth 2.0 beacon chain is light-client friendliness, not only to allow low-resource clients (mobile phones, IoT, etc.) to maintain access to the blockchain in a reasonably safe way, but also to facilitate the development of "bridges" between the Eth 2.0 beacon chain and other chains. -## Table of Contents +## Table of contents - [Beacon Chain Light Client Syncing](#beacon-chain-light-client-syncing) - - [Table of Contents](#table-of-contents) + - [Table of contents](#table-of-contents) - [Preliminaries](#preliminaries) - [Expansions](#expansions) - [`get_active_validator_indices`](#get_active_validator_indices) diff --git a/specs/networking/messaging.md b/specs/networking/messaging.md index b64e1d5d83..d7cb5bb5bb 100644 --- a/specs/networking/messaging.md +++ b/specs/networking/messaging.md @@ -1,23 +1,22 @@ -ETH 2.0 Networking Spec - Messaging -=== +# Eth 2.0 Networking Spec - Messaging -# Abstract +## Abstract This specification describes how individual Ethereum 2.0 messages are represented on the wire. -The key words “MUST”, “MUST NOT”, “REQUIRED”, “SHALL”, “SHALL”, NOT", “SHOULD”, “SHOULD NOT”, “RECOMMENDED”, “MAY”, and “OPTIONAL” in this document are to be interpreted as described in RFC 2119. +The key words “MUST”, “MUST NOT”, “REQUIRED”, “SHALL”, “SHALL”, NOT", “SHOULD”, “SHOULD NOT”, “RECOMMENDED”, “MAY”, and “OPTIONAL” in this document are to be interpreted as described in [RFC 2119](https://tools.ietf.org/html/rfc2119). -# Motivation +## Motivation -This specification seeks to define a messaging protocol that is flexible enough to be changed easily as the ETH 2.0 specification evolves. +This specification seeks to define a messaging protocol that is flexible enough to be changed easily as the Eth 2.0 specification evolves. Note that while `libp2p` is the chosen networking stack for Ethereum 2.0, as of this writing some clients do not have workable `libp2p` implementations. To allow those clients to communicate, we define a message envelope that includes the body's compression, encoding, and body length. Once `libp2p` is available across all implementations, this message envelope will be removed because `libp2p` will negotiate the values defined in the envelope upfront. -# Specification +## Specification -## Message Structure +### Message structure -An ETH 2.0 message consists of an envelope that defines the message's compression, encoding, and length followed by the body itself. +An Eth 2.0 message consists of an envelope that defines the message's compression, encoding, and length followed by the body itself. Visually, a message looks like this: @@ -35,12 +34,12 @@ Visually, a message looks like this: +--------------------------+ ``` -Clients MUST ignore messages with mal-formed bodies. The compression/encoding nibbles MUST be one of the following values: +Clients MUST ignore messages with malformed bodies. The compression/encoding nibbles MUST be one of the following values: -## Compression Nibble Values +### Compression nibble values - `0x0`: no compression -## Encoding Nibble Values +### Encoding nibble values - `0x1`: SSZ diff --git a/specs/networking/node-identification.md b/specs/networking/node-identification.md index 0f1f9832b7..32ec4dfad1 100644 --- a/specs/networking/node-identification.md +++ b/specs/networking/node-identification.md @@ -1,13 +1,12 @@ -ETH 2.0 Networking Spec - Node Identification -=== +# Eth 2.0 Networking Spec - Node Identification -# Abstract +## Abstract This specification describes how Ethereum 2.0 nodes identify and address each other on the network. -The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL", NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119. +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL", NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC 2119](https://tools.ietf.org/html/rfc2119). -# Specification +## Specification Clients use Ethereum Node Records (as described in [EIP-778](http://eips.ethereum.org/EIPS/eip-778)) to discover one another. Each ENR includes, among other things, the following keys: @@ -21,11 +20,11 @@ The keys above are enough to construct a [multiaddr](https://github.com/multifor It is RECOMMENDED that clients set their TCP port to the default of `9000`. -## Peer ID Generation +### Peer ID generation The `libp2p` networking stack identifies peers via a "peer ID." Simply put, a node's Peer ID is the SHA2-256 `multihash` of the node's public key struct (serialized in protobuf, refer to the [Peer ID spec](https://github.com/libp2p/specs/pull/100)). `go-libp2p-crypto` contains the canonical implementation of how to hash `secp256k1` keys for use as a peer ID. -# See Also +## See also - [multiaddr](https://github.com/multiformats/multiaddr) - [multihash](https://multiformats.io/multihash/) diff --git a/specs/networking/rpc-interface.md b/specs/networking/rpc-interface.md index f1da8f7e3d..ab7a08392c 100644 --- a/specs/networking/rpc-interface.md +++ b/specs/networking/rpc-interface.md @@ -1,19 +1,18 @@ -ETH 2.0 Networking Spec - RPC Interface -=== +# Eth 2.0 Networking Spec - RPC Interface -# Abstract +## Abstract The Ethereum 2.0 networking stack uses two modes of communication: a broadcast protocol that gossips information to interested parties via GossipSub, and an RPC protocol that retrieves information from specific clients. This specification defines the RPC protocol. -The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL", NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119. +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL", NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC 2119](https://tools.ietf.org/html/rfc2119). -# Dependencies +## Dependencies This specification assumes familiarity with the [Messaging](./messaging.md), [Node Identification](./node-identification.md), and [Beacon Chain](../core/0_beacon-chain.md) specifications. # Specification -## Message Schemas +## Message schemas Message body schemas are notated like this: @@ -26,13 +25,13 @@ Message body schemas are notated like this: Embedded types are serialized as SSZ Containers unless otherwise noted. -All referenced data structures can be found in the [0-beacon-chain](../core/0_beacon-chain.md#data-structures) specification. +All referenced data structures can be found in the [Beacon Chain](../core/0_beacon-chain.md#data-structures) specification. -## `libp2p` Protocol Names +## `libp2p` protocol names -A "Protocol ID" in `libp2p` parlance refers to a human-readable identifier `libp2p` uses in order to identify sub-protocols and stream messages of different types over the same connection. Peers exchange supported protocol IDs via the `Identify` protocol upon connection. When opening a new stream, peers pin a particular protocol ID to it, and the stream remains contextualised thereafter. Since messages are sent inside a stream, they do not need to bear the protocol ID. +A "Protocol ID" in `libp2p` parlance refers to a human-readable identifier `libp2p` uses in order to identify sub-protocols and stream messages of different types over the same connection. Peers exchange supported protocol IDs via the `Identify` protocol upon connection. When opening a new stream, peers pin a particular protocol ID to it, and the stream remains contextualized thereafter. Since messages are sent inside a stream, they do not need to bear the protocol ID. -## RPC-Over-`libp2p` +## RPC-over-`libp2p` To facilitate RPC-over-`libp2p`, a single protocol name is used: `/eth/serenity/beacon/rpc/1`. The version number in the protocol name is neither backwards or forwards compatible, and will be incremented whenever changes to the below structures are required. @@ -88,7 +87,7 @@ The first 1,000 values in `response_code` are reserved for system use. The follo 3. `30`: Method not found. 4. `40`: Server error. -### Alternative for Non-`libp2p` Clients +### Alternative for non-`libp2p` clients Since some clients are waiting for `libp2p` implementations in their respective languages. As such, they MAY listen for raw TCP messages on port `9000`. To distinguish RPC messages from other messages on that port, a byte prefix of `ETH` (`0x455448`) MUST be prepended to all messages. This option will be removed once `libp2p` is ready in all supported languages. @@ -145,7 +144,7 @@ Root B ^ +---+ ``` -Once the handshake completes, the client with the higher `latest_finalized_epoch` or `best_slot` (if the clients have equal `latest_finalized_epoch`s) SHOULD request beacon block roots from its counterparty via `beacon_block_roots` (i.e., RPC method `10`). +Once the handshake completes, the client with the higher `latest_finalized_epoch` or `best_slot` (if the clients have equal `latest_finalized_epoch`s) SHOULD request beacon block roots from its counterparty via `beacon_block_roots` (i.e. RPC method `10`). ### Goodbye @@ -167,11 +166,11 @@ Client MAY send `goodbye` messages upon disconnection. The reason field MAY be o Clients MAY define custom goodbye reasons as long as the value is larger than `1000`. -### Get Status +### Get status **Method ID:** `2` -**Request Body:** +**Request body:** ``` ( @@ -181,7 +180,7 @@ Clients MAY define custom goodbye reasons as long as the value is larger than `1 ) ``` -**Response Body:** +**Response body:** ``` ( @@ -193,11 +192,11 @@ Clients MAY define custom goodbye reasons as long as the value is larger than `1 Returns metadata about the remote node. -### Request Beacon Block Roots +### Request beacon block roots **Method ID:** `10` -**Request Body** +**Request body** ``` ( @@ -206,7 +205,7 @@ Returns metadata about the remote node. ) ``` -**Response Body:** +**Response body:** ``` # BlockRootSlot @@ -222,11 +221,11 @@ Returns metadata about the remote node. Requests a list of block roots and slots from the peer. The `count` parameter MUST be less than or equal to `32768`. The slots MUST be returned in ascending slot order. -### Beacon Block Headers +### Beacon block headers **Method ID:** `11` -**Request Body** +**Request body** ``` ( @@ -237,7 +236,7 @@ Requests a list of block roots and slots from the peer. The `count` parameter MU ) ``` -**Response Body:** +**Response body:** ``` ( @@ -245,15 +244,15 @@ Requests a list of block roots and slots from the peer. The `count` parameter MU ) ``` -Requests beacon block headers from the peer starting from `(start_root, start_slot)`. The response MUST contain no more than `max_headers` headers. `skip_slots` defines the maximum number of slots to skip between blocks. For example, requesting blocks starting at slots `2` a `skip_slots` value of `1` would return the blocks at `[2, 4, 6, 8, 10]`. In cases where a slot is empty for a given slot number, the closest previous block MUST be returned. For example, if slot `4` were empty in the previous example, the returned array would contain `[2, 3, 6, 8, 10]`. If slot three were further empty, the array would contain `[2, 6, 8, 10]` - i.e., duplicate blocks MUST be collapsed. A `skip_slots` value of `0` returns all blocks. +Requests beacon block headers from the peer starting from `(start_root, start_slot)`. The response MUST contain no more than `max_headers` headers. `skip_slots` defines the maximum number of slots to skip between blocks. For example, requesting blocks starting at slots `2` a `skip_slots` value of `1` would return the blocks at `[2, 4, 6, 8, 10]`. In cases where a slot is empty for a given slot number, the closest previous block MUST be returned. For example, if slot `4` were empty in the previous example, the returned array would contain `[2, 3, 6, 8, 10]`. If slot three were further empty, the array would contain `[2, 6, 8, 10]`—i.e. duplicate blocks MUST be collapsed. A `skip_slots` value of `0` returns all blocks. The function of the `skip_slots` parameter helps facilitate light client sync - for example, in [#459](https://github.com/ethereum/eth2.0-specs/issues/459) - and allows clients to balance the peers from whom they request headers. Clients could, for instance, request every 10th block from a set of peers where each peer has a different starting block in order to populate block data. -### Beacon Block Bodies +### Beacon block bodies **Method ID:** `12` -**Request Body:** +**Request body:** ``` ( @@ -261,7 +260,7 @@ The function of the `skip_slots` parameter helps facilitate light client sync - ) ``` -**Response Body:** +**Response body:** ``` ( @@ -269,15 +268,15 @@ The function of the `skip_slots` parameter helps facilitate light client sync - ) ``` -Requests the `block_bodies` associated with the provided `block_roots` from the peer. Responses MUST return `block_roots` in the order provided in the request. If the receiver does not have a particular `block_root`, it must return a zero-value `block_body` (i.e., a `block_body` container with all zero fields). +Requests the `block_bodies` associated with the provided `block_roots` from the peer. Responses MUST return `block_roots` in the order provided in the request. If the receiver does not have a particular `block_root`, it must return a zero-value `block_body` (i.e. a `block_body` container with all zero fields). -### Beacon Chain State +### Beacon chain state -**Note:** This section is preliminary, pending the definition of the data structures to be transferred over the wire during fast sync operations. +*Note*: This section is preliminary, pending the definition of the data structures to be transferred over the wire during fast sync operations. **Method ID:** `13` -**Request Body:** +**Request body:** ``` ( @@ -285,7 +284,7 @@ Requests the `block_bodies` associated with the provided `block_roots` from the ) ``` -**Response Body:** TBD +**Response body:** TBD Requests contain the hashes of Merkle tree nodes that when merkleized yield the block's `state_root`. diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 0080f24471..343c967d4a 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -1,23 +1,28 @@ # SimpleSerialize (SSZ) -This is a **work in progress** describing typing, serialization and Merkleization of Ethereum 2.0 objects. +**Notice**: This document is a work-in-progress describing typing, serialization, and Merkleization of Eth 2.0 objects. ## Table of contents - -- [Constants](#constants) -- [Typing](#typing) - - [Basic types](#basic-types) - - [Composite types](#composite-types) - - [Aliases](#aliases) - - [Default values](#default-values) -- [Serialization](#serialization) - - [`"uintN"`](#uintn) - - [`"bool"`](#bool) - - [Containers, vectors, lists](#containers-vectors-lists) -- [Deserialization](#deserialization) -- [Merkleization](#merkleization) -- [Self-signed containers](#self-signed-containers) -- [Implementations](#implementations) + + +- [SimpleSerialize (SSZ)](#simpleserialize-ssz) + - [Table of contents](#table-of-contents) + - [Constants](#constants) + - [Typing](#typing) + - [Basic types](#basic-types) + - [Composite types](#composite-types) + - [Aliases](#aliases) + - [Default values](#default-values) + - [Serialization](#serialization) + - [`"uintN"`](#uintn) + - [`"bool"`](#bool) + - [Containers, vectors, lists](#containers-vectors-lists) + - [Deserialization](#deserialization) + - [Merkleization](#merkleization) + - [Self-signed containers](#self-signed-containers) + - [Implementations](#implementations) + + ## Constants @@ -133,4 +138,4 @@ Let `value` be a self-signed container object. The convention is that the signat | Go | Prysm | Prysmatic Labs | [https://github.com/prysmaticlabs/prysm/tree/master/shared/ssz](https://github.com/prysmaticlabs/prysm/tree/master/shared/ssz) | | Swift | Yeeth | Dean Eigenmann | [https://github.com/yeeth/SimpleSerialize.swift](https://github.com/yeeth/SimpleSerialize.swift) | | C# | | Jordan Andrews | [https://github.com/codingupastorm/csharp-ssz](https://github.com/codingupastorm/csharp-ssz) | -| C++ | | | [https://github.com/NAKsir-melody/cpp_ssz](https://github.com/NAKsir-melody/cpp_ssz) | +| C++ | | Jiyun Kim | [https://github.com/NAKsir-melody/cpp_ssz](https://github.com/NAKsir-melody/cpp_ssz) | diff --git a/specs/test_formats/README.md b/specs/test_formats/README.md index 273659ce93..4783dc475c 100644 --- a/specs/test_formats/README.md +++ b/specs/test_formats/README.md @@ -1,17 +1,27 @@ # General test format -This document defines the YAML format and structure used for ETH 2.0 testing. - -## ToC - -* [About](#about) -* [Glossary](#glossary) -* [Test format philosophy](#test-format-philosophy) -* [Test Suite](#test-suite) -* [Config](#config) -* [Fork-timeline](#fork-timeline) -* [Config sourcing](#config-sourcing) -* [Test structure](#test-structure) +This document defines the YAML format and structure used for Eth 2.0 testing. + +## Table of contents + + +- [General test format](#general-test-format) + - [Table of contents](#table-of-contents) + - [About](#about) + - [Test-case formats](#test-case-formats) + - [Glossary](#glossary) + - [Test format philosophy](#test-format-philosophy) + - [Config design](#config-design) + - [Fork config design](#fork-config-design) + - [Test completeness](#test-completeness) + - [Test suite](#test-suite) + - [Config](#config) + - [Fork-timeline](#fork-timeline) + - [Config sourcing](#config-sourcing) + - [Test structure](#test-structure) + - [Note for implementers](#note-for-implementers) + + ## About @@ -52,28 +62,28 @@ Test formats: ### Config design After long discussion, the following types of configured constants were identified: -- Never changing: genesis data +- Never changing: genesis data. - Changing, but reliant on old value: e.g. an epoch time may change, but if you want to do the conversion - `(genesis data, timestamp) -> epoch number` you end up needing both constants. + `(genesis data, timestamp) -> epoch number`, you end up needing both constants. - Changing, but kept around during fork transition: finalization may take a while, e.g. an executable has to deal with new deposits and old deposits at the same time. Another example may be economic constants. -- Additional, back-wards compatible: new constants are introduced for later phases +- Additional, backwards compatible: new constants are introduced for later phases. - Changing: there is a very small chance some constant may really be *replaced*. In this off-chance, it is likely better to include it as an additional variable, - and some clients may simply stop supporting the old one, if they do not want to sync from genesis. + and some clients may simply stop supporting the old one if they do not want to sync from genesis. Based on these types of changes, we model the config as a list of key value pairs, - that only grows with every fork (they may change in development versions of forks however, git manages this). -With this approach, configurations are backwards compatible (older clients ignore unknown variables), and easy to maintain. + that only grows with every fork (they may change in development versions of forks, however; git manages this). +With this approach, configurations are backwards compatible (older clients ignore unknown variables) and easy to maintain. ### Fork config design There are two types of fork-data: -1) timeline: when does a fork take place? -2) coverage: what forks are covered by a test? +1) Timeline: When does a fork take place? +2) Coverage: What forks are covered by a test? The first is neat to have as a separate form: we prevent duplication, and can run with different presets - (e.g. fork timeline for a minimal local test, for a public testnet, or for mainnet) + (e.g. fork timeline for a minimal local test, for a public testnet, or for mainnet). The second does not affect the result of the tests, it just states what is covered by the tests, so that the right suites can be executed to see coverage for a certain fork. @@ -90,7 +100,7 @@ The aim is to provide clients with a well-defined scope of work to run a particu - Clients that are not complete in functionality can choose to ignore suites that use certain test-runners, or specific handlers of these test-runners. - Clients that are on older versions can test their work based on older releases of the generated tests, and catch up with newer releases when possible. -## Test Suite +## Test suite ``` title: -- Display name for the test suite @@ -113,9 +123,9 @@ Separation of configuration and tests aims to: - Prevent duplication of configuration - Make all tests easy to upgrade (e.g. when a new config constant is introduced) - Clearly define which constants to use -- Shareable between clients, for cross-client short or long lived testnets +- Shareable between clients, for cross-client short- or long-lived testnets - Minimize the amounts of different constants permutations to compile as a client. - Note: Some clients prefer compile-time constants and optimizations. + *Note*: Some clients prefer compile-time constants and optimizations. They should compile for each configuration once, and run the corresponding tests per build target. The format is described in [`configs/constant_presets`](../../configs/constant_presets/README.md#format). @@ -124,9 +134,9 @@ The format is described in [`configs/constant_presets`](../../configs/constant_p ## Fork-timeline A fork timeline is (preferably) loaded in as a configuration object into a client, as opposed to the constants configuration: - - we do not allocate or optimize any code based on epoch numbers - - when we transition from one fork to the other, it is preferred to stay online. - - we may decide on an epoch number for a fork based on external events (e.g. Eth1 log event), + - We do not allocate or optimize any code based on epoch numbers. + - When we transition from one fork to the other, it is preferred to stay online. + - We may decide on an epoch number for a fork based on external events (e.g. Eth1 log event); a client should be able to activate a fork dynamically. The format is described in [`configs/fork_timelines`](../../configs/fork_timelines/README.md#format). diff --git a/specs/test_formats/bls/README.md b/specs/test_formats/bls/README.md index db63bba1d3..4d95bdfd77 100644 --- a/specs/test_formats/bls/README.md +++ b/specs/test_formats/bls/README.md @@ -1,7 +1,7 @@ # BLS tests A test type for BLS. Primarily geared towards verifying the *integration* of any BLS library. -We do not recommend to roll your own crypto, or use an untested BLS library. +We do not recommend rolling your own crypto or using an untested BLS library. The BLS test suite runner has the following handlers: @@ -12,4 +12,4 @@ The BLS test suite runner has the following handlers: - [`priv_to_pub`](./priv_to_pub.md) - [`sign_msg`](./sign_msg.md) -Note: signature-verification and aggregate-verify test cases are not yet supported. +*Note*: Signature-verification and aggregate-verify test cases are not yet supported. diff --git a/specs/test_formats/shuffling/README.md b/specs/test_formats/shuffling/README.md index 57be965657..25074742d5 100644 --- a/specs/test_formats/shuffling/README.md +++ b/specs/test_formats/shuffling/README.md @@ -1,16 +1,16 @@ # Test format: shuffling -The runner of the Shuffling test type has only one handler: `core` +The runner of the Shuffling test type has only one handler: `core`. -This does not mean however that testing is limited. +However, this does not mean that testing is limited. Clients may take different approaches to shuffling, for optimizing, and supporting advanced lookup behavior back in older history. For implementers, possible test runners implementing testing can include: -1) just test permute-index, run it for each index `i` in `range(count)`, and check against expected `output[i]` (default spec implementation) -2) test un-permute-index (the reverse lookup. Implemented by running the shuffling rounds in reverse: from `round_count-1` to `0`) -3) test the optimized complete shuffle, where all indices are shuffled at once, test output in one go. -4) test complete shuffle in reverse (reverse rounds, same as 2) +1) Just test permute-index, run it for each index `i` in `range(count)`, and check against expected `output[i]` (default spec implementation). +2) Test un-permute-index (the reverse lookup; implemented by running the shuffling rounds in reverse, from `round_count-1` to `0`). +3) Test the optimized complete shuffle, where all indices are shuffled at once; test output in one go. +4) Test complete shuffle in reverse (reverse rounds, same as #2). ## Test case format diff --git a/specs/test_formats/ssz_generic/README.md b/specs/test_formats/ssz_generic/README.md index 9fda0c368d..da0898087d 100644 --- a/specs/test_formats/ssz_generic/README.md +++ b/specs/test_formats/ssz_generic/README.md @@ -3,7 +3,7 @@ This set of test-suites provides general testing for SSZ: to instantiate any container/list/vector/other type from binary data. -Since SSZ is in a development-phase, not the full suite of features is covered yet. +Since SSZ is in a development-phase, the full suite of features is not covered yet. Note that these tests are based on the older SSZ package. The tests are still relevant, but limited in scope: more complex object encodings have changed since the original SSZ testing. @@ -11,10 +11,10 @@ The tests are still relevant, but limited in scope: A minimal but useful series of tests covering `uint` encoding and decoding is provided. This is a direct port of the older SSZ `uint` tests (minus outdated test cases). -[uint test format](./uint.md). +Test format documentation can be found here: [uint test format](./uint.md). -Note: the current phase-0 spec does not use larger uints, and uses byte vectors (fixed length) instead to represent roots etc. +*Note*: The current Phase 0 spec does not use larger uints, and uses byte vectors (fixed length) instead to represent roots etc. The exact uint lengths to support may be redefined in the future. -Extension of the SSZ tests collection is planned, with an update to the new spec-maintained `minimal_ssz.py`, +Extension of the SSZ tests collection is planned, with an update to the new spec-maintained `minimal_ssz.py`; see CI/testing issues for progress tracking. diff --git a/specs/test_formats/ssz_static/README.md b/specs/test_formats/ssz_static/README.md index 413b00c75b..1df2cb5f66 100644 --- a/specs/test_formats/ssz_static/README.md +++ b/specs/test_formats/ssz_static/README.md @@ -1,7 +1,7 @@ # SSZ, static tests This set of test-suites provides static testing for SSZ: - to instantiate just the known ETH-2.0 SSZ types from binary data. + to instantiate just the known Eth 2.0 SSZ types from binary data. This series of tests is based on the spec-maintained `minimal_ssz.py`, i.e. fully consistent with the SSZ spec. diff --git a/specs/test_formats/ssz_static/core.md b/specs/test_formats/ssz_static/core.md index 1d470c3381..64b09a3296 100644 --- a/specs/test_formats/ssz_static/core.md +++ b/specs/test_formats/ssz_static/core.md @@ -1,7 +1,7 @@ # Test format: SSZ static types The goal of this type is to provide clients with a solid reference for how the known SSZ objects should be encoded. -Each object described in the Phase-0 spec is covered. +Each object described in the Phase 0 spec is covered. This is important, as many of the clients aiming to serialize/deserialize objects directly into structs/classes do not support (or have alternatives for) generic SSZ encoding/decoding. This test-format ensures these direct serializations are covered. @@ -27,6 +27,6 @@ A test-runner can implement the following assertions: ## References -**`serialized`**: [SSZ serialization](../../simple-serialize.md#serialization) -**`root`** - [hash_tree_root](../../simple-serialize.md#merkleization) function -**`signing_root`** - [signing_root](../../simple-serialize.md#self-signed-containers) function +**`serialized`**—[SSZ serialization](../../simple-serialize.md#serialization) +**`root`**—[hash_tree_root](../../simple-serialize.md#merkleization) function +**`signing_root`**—[signing_root](../../simple-serialize.md#self-signed-containers) function diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ca7f0eb3b3..600ed08391 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -1,13 +1,13 @@ # Ethereum 2.0 Phase 0 -- Honest Validator -__NOTICE__: This document is a work-in-progress for researchers and implementers. This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](../core/0_beacon-chain.md) that describes the expected actions of a "validator" participating in the Ethereum 2.0 protocol. +**Notice**: This document is a work-in-progress for researchers and implementers. This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](../core/0_beacon-chain.md), which describes the expected actions of a "validator" participating in the Ethereum 2.0 protocol. -## Table of Contents +## Table of contents - [Ethereum 2.0 Phase 0 -- Honest Validator](#ethereum-20-phase-0----honest-validator) - - [Table of Contents](#table-of-contents) + - [Table of contents](#table-of-contents) - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Constants](#constants) @@ -96,7 +96,7 @@ The validator constructs their `withdrawal_credentials` via the following: ### Submit deposit -In phase 0, all incoming validator deposits originate from the Ethereum 1.0 PoW chain. Deposits are made to the [deposit contract](../core/0_deposit-contract.md) located at `DEPOSIT_CONTRACT_ADDRESS`. +In Phase 0, all incoming validator deposits originate from the Ethereum 1.0 PoW chain. Deposits are made to the [deposit contract](../core/0_deposit-contract.md) located at `DEPOSIT_CONTRACT_ADDRESS`. To submit a deposit: @@ -106,11 +106,11 @@ To submit a deposit: * Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=DOMAIN_DEPOSIT`. * Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])` along with a deposit of `amount` Gwei. -_Note_: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validator_registry` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. +*Note*: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validator_registry` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. ### Process deposit -Deposits cannot be processed into the beacon chain until the eth1.0 block in which they were deposited or any of its descendants is added to the beacon chain `state.eth1_data`. This takes _a minimum_ of `ETH1_FOLLOW_DISTANCE` eth1.0 blocks (~4 hours) plus `ETH1_DATA_VOTING_PERIOD` epochs (~1.7 hours). Once the requisite eth1.0 data is added, the deposit will normally be added to a beacon chain block and processed into the `state.validator_registry` within an epoch or two. The validator is then in a queue to be activated. +Deposits cannot be processed into the beacon chain until the Eth 1.0 block in which they were deposited or any of its descendants is added to the beacon chain `state.eth1_data`. This takes _a minimum_ of `ETH1_FOLLOW_DISTANCE` Eth 1.0 blocks (~4 hours) plus `ETH1_DATA_VOTING_PERIOD` epochs (~1.7 hours). Once the requisite Eth 1.0 data is added, the deposit will normally be added to a beacon chain block and processed into the `state.validator_registry` within an epoch or two. The validator is then in a queue to be activated. ### Validator index @@ -130,11 +130,11 @@ is_active = is_active_validator(validator, shuffling_epoch) Once a validator is activated, the validator is assigned [responsibilities](#beacon-chain-responsibilities) until exited. -_Note_: There is a maximum validator churn per finalized epoch so the delay until activation is variable depending upon finality, total active validator balance, and the number of validators in the queue to be activated. +*Note*: There is a maximum validator churn per finalized epoch so the delay until activation is variable depending upon finality, total active validator balance, and the number of validators in the queue to be activated. ## Beacon chain responsibilities -A validator has two primary responsibilities to the beacon chain -- [proposing blocks](block-proposal) and [creating attestations](attestations-1). Proposals happen infrequently, whereas attestations should be created once per epoch. +A validator has two primary responsibilities to the beacon chain: [proposing blocks](block-proposal) and [creating attestations](attestations-1). Proposals happen infrequently, whereas attestations should be created once per epoch. ### Block proposal @@ -148,7 +148,7 @@ There is one proposer per slot, so if there are N active validators any individu Set `block.slot = slot` where `slot` is the current slot at which the validator has been selected to propose. The `parent` selected must satisfy that `parent.slot < block.slot`. -_Note:_ there might be "skipped" slots between the `parent` and `block`. These skipped slots are processed in the state transition function without per-block processing. +*Note*: There might be "skipped" slots between the `parent` and `block`. These skipped slots are processed in the state transition function without per-block processing. ##### Parent root @@ -158,7 +158,7 @@ Set `block.previous_block_root = signing_root(parent)`. Set `block.state_root = hash_tree_root(state)` of the resulting `state` of the `parent -> block` state transition. -_Note_: To calculate `state_root`, the validator should first run the state transition function on an unsigned `block` containing a stub for the `state_root`. It is useful to be able to run a state transition function that does _not_ validate signatures or state root for this purpose. +*Note*: To calculate `state_root`, the validator should first run the state transition function on an unsigned `block` containing a stub for the `state_root`. It is useful to be able to run a state transition function that does _not_ validate signatures or state root for this purpose. ##### Randao reveal @@ -181,12 +181,12 @@ epoch_signature = bls_sign( `block.eth1_data` is a mechanism used by block proposers vote on a recent Ethereum 1.0 block hash and an associated deposit root found in the Ethereum 1.0 deposit contract. When consensus is formed, `state.latest_eth1_data` is updated, and validator deposits up to this root can be processed. The deposit root can be calculated by calling the `get_deposit_root()` function of the deposit contract using the post-state of the block hash. * Let `D` be the set of `Eth1DataVote` objects `vote` in `state.eth1_data_votes` where: - * `vote.eth1_data.block_hash` is the hash of an eth1.0 block that is (i) part of the canonical chain, (ii) >= `ETH1_FOLLOW_DISTANCE` blocks behind the head, and (iii) newer than `state.latest_eth1_data.block_hash`. - * `vote.eth1_data.deposit_count` is the deposit count of the eth1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. - * `vote.eth1_data.deposit_root` is the deposit root of the eth1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. + * `vote.eth1_data.block_hash` is the hash of an Eth 1.0 block that is (i) part of the canonical chain, (ii) >= `ETH1_FOLLOW_DISTANCE` blocks behind the head, and (iii) newer than `state.latest_eth1_data.block_hash`. + * `vote.eth1_data.deposit_count` is the deposit count of the Eth 1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. + * `vote.eth1_data.deposit_root` is the deposit root of the Eth 1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. * If `D` is empty: - * Let `block_hash` be the block hash of the `ETH1_FOLLOW_DISTANCE`'th ancestor of the head of the canonical eth1.0 chain. - * Let `deposit_root` and `deposit_count` be the deposit root and deposit count of the eth1.0 deposit contract in the post-state of the block referenced by `block_hash` + * Let `block_hash` be the block hash of the `ETH1_FOLLOW_DISTANCE`'th ancestor of the head of the canonical Eth 1.0 chain. + * Let `deposit_root` and `deposit_count` be the deposit root and deposit count of the Eth 1.0 deposit contract in the post-state of the block referenced by `block_hash` * Let `best_vote_data = Eth1Data(block_hash=block_hash, deposit_root=deposit_root, deposit_count=deposit_count)`. * If `D` is nonempty: * Let `best_vote_data` be the `eth1_data` of the member of `D` that has the highest `vote.vote_count`, breaking ties by favoring block hashes with higher associated block height. @@ -224,7 +224,7 @@ Up to `MAX_ATTESTATIONS` aggregate attestations can be included in the `block`. ##### Deposits -If there are any unprocessed deposits for the existing `state.latest_eth1_data` (i.e. `state.latest_eth1_data.deposit_count > state.deposit_index`), then pending deposits _must_ be added to the block. The expected number of deposits is exactly `min(MAX_DEPOSITS, latest_eth1_data.deposit_count - state.deposit_index)`. These [`deposits`](../core/0_beacon-chain.md#deposit) are constructed from the `Deposit` logs from the [Eth1.0 deposit contract](../core/0_deposit-contract) and must be processed in sequential order. The deposits included in the `block` must satisfy the verification conditions found in [deposits processing](../core/0_beacon-chain.md#deposits). +If there are any unprocessed deposits for the existing `state.latest_eth1_data` (i.e. `state.latest_eth1_data.deposit_count > state.deposit_index`), then pending deposits _must_ be added to the block. The expected number of deposits is exactly `min(MAX_DEPOSITS, latest_eth1_data.deposit_count - state.deposit_index)`. These [`deposits`](../core/0_beacon-chain.md#deposit) are constructed from the `Deposit` logs from the [Eth 1.0 deposit contract](../core/0_deposit-contract) and must be processed in sequential order. The deposits included in the `block` must satisfy the verification conditions found in [deposits processing](../core/0_beacon-chain.md#deposits). The `proof` for each deposit must be constructed against the deposit root contained in `state.latest_eth1_data` rather than the deposit root at the time the deposit was initially logged from the 1.0 chain. This entails storing a full deposit merkle tree locally and computing updated proofs against the `latest_eth1_data.deposit_root` as needed. See [`minimal_merkle.py`](https://github.com/ethereum/research/blob/master/spec_pythonizer/utils/merkle_minimal.py) for a sample implementation. @@ -236,7 +236,7 @@ Up to `MAX_VOLUNTARY_EXITS` [`VoluntaryExit`](../core/0_beacon-chain.md#voluntar A validator is expected to create, sign, and broadcast an attestation during each epoch. The slot during which the validator performs this role is any slot at which `get_crosslink_committees_at_slot(state, slot)` contains a committee that contains `validator_index`. -A validator should create and broadcast the attestation halfway through the `slot` during which the validator is assigned -- that is `SECONDS_PER_SLOT * 0.5` seconds after the start of `slot`. +A validator should create and broadcast the attestation halfway through the `slot` during which the validator is assigned ― that is, `SECONDS_PER_SLOT * 0.5` seconds after the start of `slot`. #### Attestation data @@ -265,7 +265,7 @@ Set `attestation_data.source_root = head_state.current_justified_root`. Set `attestation_data.target_root = signing_root(epoch_boundary)` where `epoch_boundary` is the block at the most recent epoch boundary. -_Note:_ This can be looked up in the state using: +*Note*: This can be looked up in the state using: * Let `epoch_start_slot = get_epoch_start_slot(get_current_epoch(head_state))`. * Set `epoch_boundary = head if epoch_start_slot == head_state.slot else get_block_root(state, epoch_start_slot)`. @@ -281,7 +281,7 @@ Set `attestation_data.previous_crosslink_root = hash_tree_root(head_state.curren Set `attestation_data.crosslink_data_root = ZERO_HASH`. -_Note:_ This is a stub for phase 0. +*Note*: This is a stub for Phase 0. #### Construct attestation @@ -298,14 +298,14 @@ Set `attestation.data = attestation_data` where `attestation_data` is the `Attes * Set `aggregation_bitfield[index_into_committee // 8] |= 2 ** (index_into_committee % 8)`. * Set `attestation.aggregation_bitfield = aggregation_bitfield`. -_Note_: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield)` should return a list of length equal to 1, containing `validator_index`. +*Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield)` should return a list of length equal to 1, containing `validator_index`. ##### Custody bitfield * Let `custody_bitfield` be a byte array filled with zeros of length `(len(committee) + 7) // 8`. * Set `attestation.custody_bitfield = custody_bitfield`. -_Note:_ This is a stub for phase 0. +*Note*: This is a stub for Phase 0. ##### Aggregate signature @@ -379,14 +379,14 @@ def is_proposer_at_slot(state: BeaconState, return get_beacon_proposer_index(state) == validator_index ``` -_Note_: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot. +*Note*: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot. ### Lookahead -The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing which must checked during the slot in question. +The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must checked during the slot in question. -`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in phase 1+). +`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in Phase 1+). Specifically, a validator should call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. @@ -394,19 +394,19 @@ Specifically, a validator should call `get_committee_assignment(state, next_epoc "Slashing" is the burning of some amount of validator funds and immediate ejection from the active validator set. In Phase 0, there are two ways in which funds can be slashed -- [proposer slashing](#proposer-slashing) and [attester slashing](#attester-slashing). Although being slashed has serious repercussions, it is simple enough to avoid being slashed all together by remaining _consistent_ with respect to the messages a validator has previously signed. -_Note_: Signed data must be within a sequential `Fork` context to conflict. Messages cannot be slashed across diverging forks. If the previous fork version is 1 and the chain splits into fork 2 and 102, messages from 1 can slashable against messages in forks 1, 2, and 102. Messages in 2 cannot be slashable against messages in 102 and vice versa. +*Note*: Signed data must be within a sequential `Fork` context to conflict. Messages cannot be slashed across diverging forks. If the previous fork version is 1 and the chain splits into fork 2 and 102, messages from 1 can slashable against messages in forks 1, 2, and 102. Messages in 2 cannot be slashable against messages in 102 and vice versa. ### Proposer slashing To avoid "proposer slashings", a validator must not sign two conflicting [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) where conflicting is defined as two distinct blocks within the same epoch. -_In phase 0, as long as the validator does not sign two different beacon blocks for the same epoch, the validator is safe against proposer slashings._ +*In Phase 0, as long as the validator does not sign two different beacon blocks for the same epoch, the validator is safe against proposer slashings.* -Specifically, when signing an `BeaconBlock`, a validator should perform the following steps in the following order: -1. Save a record to hard disk that an beacon block has been signed for the `epoch=slot_to_epoch(block.slot)`. +Specifically, when signing a `BeaconBlock`, a validator should perform the following steps in the following order: +1. Save a record to hard disk that a beacon block has been signed for the `epoch=slot_to_epoch(block.slot)`. 2. Generate and broadcast the block. -If the software crashes at some point within this routine, then when the validator comes back online the hard disk has the record of the _potentially_ signed/broadcast block and can effectively avoid slashing. +If the software crashes at some point within this routine, then when the validator comes back online the hard disk has the record of the *potentially* signed/broadcast block and can effectively avoid slashing. ### Attester slashing @@ -416,4 +416,4 @@ Specifically, when signing an `Attestation`, a validator should perform the foll 1. Save a record to hard disk that an attestation has been signed for source -- `attestation_data.source_epoch` -- and target -- `slot_to_epoch(attestation_data.slot)`. 2. Generate and broadcast attestation. -If the software crashes at some point within this routine, then when the validator comes back online the hard disk has the record of the _potentially_ signed/broadcast attestation and can effectively avoid slashing. +If the software crashes at some point within this routine, then when the validator comes back online the hard disk has the record of the *potentially* signed/broadcast attestation and can effectively avoid slashing. diff --git a/test_generators/README.md b/test_generators/README.md index 43bf7af031..f8124f9a78 100644 --- a/test_generators/README.md +++ b/test_generators/README.md @@ -2,7 +2,7 @@ This directory contains all the generators for YAML tests, consumed by Eth 2.0 client implementations. -Any issues with the generators and/or generated tests should be filed in the repository that hosts the generator outputs, here: [ethereum/eth2.0-tests](https://github.com/ethereum/eth2.0-tests). +Any issues with the generators and/or generated tests should be filed in the repository that hosts the generator outputs, here: [ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests). Whenever a release is made, the new tests are automatically built, and [eth2TestGenBot](https://github.com/eth2TestGenBot) commits the changes to the test repository. @@ -12,7 +12,7 @@ Whenever a release is made, the new tests are automatically built, and Prerequisites: - Python 3 installed - PIP 3 -- GNU make +- GNU Make ### Cleaning @@ -66,7 +66,7 @@ eth-utils==1.4.1 The config helper and pyspec is optional, but preferred. We encourage generators to derive tests from the spec itself in order to prevent code duplication and outdated tests. Applying configurations to the spec is simple and enables you to create test suites with different contexts. -Note: make sure to run `make pyspec` from the root of the specs repository in order to build the pyspec requirement. +*Note*: Make sure to run `make pyspec` from the root of the specs repository in order to build the pyspec requirement. Install all the necessary requirements (re-run when you add more): ```bash @@ -134,7 +134,7 @@ if __name__ == "__main__": Recommendations: - You can have more than just one suite creator, e.g. ` gen_runner.run_generator("foo", [bar_test_suite, abc_test_suite, example_test_suite])`. - You can concatenate lists of test cases if you don't want to split it up in suites, however, make sure they can be run with one handler. -- You can split your suite creators into different python files/packages; this is good for code organization. +- You can split your suite creators into different Python files/packages; this is good for code organization. - Use config "minimal" for performance, but also implement a suite with the default config where necessary. - You may be able to write your test suite creator in a way where it does not make assumptions on constants. If so, you can generate test suites with different configurations for the same scenario (see example). @@ -157,8 +157,8 @@ To add a new test generator that builds `New Tests`: [circleci config file](https://github.com/ethereum/eth2.0-test-generators/blob/master/.circleci/config.yml) if desired to increase code quality. -Note: you do not have to change the makefile. -However, if necessary (e.g. not using python, or mixing in other languages), submit an issue, and it can be a special case. +*Note*: You do not have to change the makefile. +However, if necessary (e.g. not using Python, or mixing in other languages), submit an issue, and it can be a special case. Do note that generators should be easy to maintain, lean, and based on the spec. @@ -167,5 +167,5 @@ Do note that generators should be easy to maintain, lean, and based on the spec. If a test generator is not needed anymore, undo the steps described above and make a new release: 1. Remove the generator directory. -2. Remove the generated tests in the [`eth2.0-tests`](https://github.com/ethereum/eth2.0-tests) repository by opening a PR there. +2. Remove the generated tests in the [`eth2.0-spec-tests`](https://github.com/ethereum/eth2.0-spec-tests) repository by opening a pull request there. 3. Make a new release. diff --git a/test_libs/pyspec/README.md b/test_libs/pyspec/README.md index df18342100..bb6991a930 100644 --- a/test_libs/pyspec/README.md +++ b/test_libs/pyspec/README.md @@ -1,11 +1,11 @@ -# ETH 2.0 PySpec +# Eth 2.0 Executable Python Spec (PySpec) -The Python executable spec is built from the ETH 2.0 specification, +The executable Python spec is built from the Eth 2.0 specification, complemented with the necessary helper functions for hashing, BLS, and more. With this executable spec, test-generators can easily create test-vectors for client implementations, - and the spec itself can be verified to be consistent and coherent, through sanity tests implemented with pytest. + and the spec itself can be verified to be consistent and coherent through sanity tests implemented with pytest. ## Building @@ -14,12 +14,12 @@ All the dynamic parts of the spec can be build at once with `make pyspec`. Alternatively, you can build a sub-set of the pyspec: `make phase0`. -Or, to build a single file, specify the path, e.g. `make test_libs/pyspec/eth2spec/phase0/spec.py` +Or, to build a single file, specify the path, e.g. `make test_libs/pyspec/eth2spec/phase0/spec.py`. ## Py-tests -After building, you can install the dependencies for running the `pyspec` tests with `make install_test` +After building, you can install the dependencies for running the `pyspec` tests with `make install_test`. These tests are not intended for client-consumption. These tests are sanity tests, to verify if the spec itself is consistent. @@ -28,7 +28,7 @@ These tests are sanity tests, to verify if the spec itself is consistent. #### Automated -Run `make test` from the root of the spec repository. +Run `make test` from the root of the specs repository. #### Manual @@ -40,7 +40,7 @@ python3 -m venv venv . venv/bin/activate pip3 install -r requirements-testing.txt ``` -Note: make sure to run `make -B pyspec` from the root of the specs repository, +*Note*: Make sure to run `make -B pyspec` from the root of the specs repository, to build the parts of the pyspec module derived from the markdown specs. The `-B` flag may be helpful to force-overwrite the `pyspec` output after you made a change to the markdown source files. @@ -58,4 +58,4 @@ The pyspec is not a replacement. ## License -Same as the spec itself, see [LICENSE](../../LICENSE) file in spec repository root. +Same as the spec itself; see [LICENSE](../../LICENSE) file in the specs repository root. From b15105e1cbb4a597fea233b16eac23ae911570a6 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Mon, 6 May 2019 17:34:03 +0100 Subject: [PATCH 048/308] Address Danny's comment --- specs/core/0_beacon-chain.md | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 156a2523d2..0bd85478ec 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1298,22 +1298,17 @@ def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestat ```python def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[Crosslink, List[ValidatorIndex]]: - shard_attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.crosslink.shard == shard] - candidate_crosslinks = [ - c for c in [a.data.crosslink for a in shard_attestations] - if hash_tree_root(state.current_crosslinks[shard]) in (c.previous_crosslink_root, hash_tree_root(c)) - ] - if len(candidate_crosslinks) == 0: - return Crosslink(), [] - - def get_attestations_for(crosslink: Crosslink) -> List[PendingAttestation]: - return [a for a in shard_attestations if a.data.crosslink == crosslink] - # Winning crosslink has the crosslink data root with the most balance voting for it (ties broken lexicographically) - winning_crosslink = max(candidate_crosslinks, key=lambda crosslink: ( - get_attesting_balance(state, get_attestations_for(crosslink)), crosslink.crosslink_data_root + attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.crosslink.shard == shard] + crosslinks = list(filter( + lambda c: hash_tree_root(state.current_crosslinks[shard]) in (c.previous_crosslink_root, hash_tree_root(c)), + [a.data.crosslink for a in attestations] )) - - return winning_crosslink, get_unslashed_attesting_indices(state, get_attestations_for(winning_crosslink)) + # Winning crosslink has the crosslink data root with the most balance voting for it (ties broken lexicographically) + winning_crosslink = max(crosslinks, key=lambda c: ( + get_attesting_balance(state, [a for a in attestations if a.data.crosslink == c]), c.crosslink_data_root + ), default=Crosslink()) + winning_attestations = [a for a in attestations if a.data.crosslink == winning_crosslink] + return winning_crosslink, get_unslashed_attesting_indices(state, winning_attestations) ``` #### Justification and finalization From 8b1a2edb7c650e0e0205e11c44f44a2a578ce392 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Mon, 6 May 2019 17:53:49 +0100 Subject: [PATCH 049/308] Fix genesis bug --- specs/core/0_beacon-chain.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9aed2001a8..04ab38e18f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -767,7 +767,7 @@ def get_epoch_start_shard(state: BeaconState, epoch: Epoch) -> Shard: ```python def get_attestation_data_slot(state: BeaconState, data: AttestationData) -> Slot: committee_count = get_epoch_committee_count(state, data.target_epoch) - offset = (data.shard + SHARD_COUNT - get_epoch_start_shard(state, data.target_epoch)) % SHARD_COUNT + offset = (data.crosslink.shard + SHARD_COUNT - get_epoch_start_shard(state, data.target_epoch)) % SHARD_COUNT return get_epoch_start_slot(data.target_epoch) + offset // (committee_count // SLOTS_PER_EPOCH) ``` @@ -1705,14 +1705,14 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ) assert data.target_epoch in (get_previous_epoch(state), get_current_epoch(state)) - if data.target_epoch == get_previous_epoch(state): - ffg_data = (state.previous_justified_epoch, state.previous_justified_root, get_previous_epoch(state)) - previous_crosslink = state.previous_crosslinks[data.crosslink.shard] - state.previous_epoch_attestations.append(pending_attestation) if data.target_epoch == get_current_epoch(state): ffg_data = (state.current_justified_epoch, state.current_justified_root, get_current_epoch(state)) previous_crosslink = state.current_crosslinks[data.crosslink.shard] state.current_epoch_attestations.append(pending_attestation) + else: + ffg_data = (state.previous_justified_epoch, state.previous_justified_root, get_previous_epoch(state)) + previous_crosslink = state.previous_crosslinks[data.crosslink.shard] + state.previous_epoch_attestations.append(pending_attestation) # Check FFG data, crosslink data, and signature assert ffg_data == (data.source_epoch, data.source_root, data.target_epoch) From f4db9ebae00338ba336943a08271012099bcef1d Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Mon, 6 May 2019 18:26:14 +0100 Subject: [PATCH 050/308] Renamings triggered by HW comment --- specs/core/0_beacon-chain.md | 26 +++++++++---------- specs/core/0_fork-choice.md | 2 +- specs/core/1_custody-game.md | 14 +++++----- specs/core/1_shard-data-chains.md | 16 ++++++------ specs/validator/0_beacon-chain-validator.md | 6 ++--- .../test_process_attestation.py | 2 +- .../test_process_block_header.py | 4 +-- test_libs/pyspec/tests/helpers.py | 12 ++++----- test_libs/pyspec/tests/test_sanity.py | 6 ++--- 9 files changed, 44 insertions(+), 44 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 04ab38e18f..afca9eb1ba 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -287,9 +287,9 @@ The types are defined topologically to aid in facilitating an executable version # Epoch number 'epoch': 'uint64', # Root of the previous crosslink - 'previous_crosslink_root': 'bytes32', + 'parent_crosslink_root': 'bytes32', # Root of the crosslinked shard data since the previous crosslink - 'crosslink_data_root': 'bytes32', + 'data_root': 'bytes32', } ``` @@ -369,7 +369,7 @@ The types are defined topologically to aid in facilitating an executable version ```python { 'slot': 'uint64', - 'previous_block_root': 'bytes32', + 'parent_block_root': 'bytes32', 'state_root': 'bytes32', 'block_body_root': 'bytes32', 'signature': 'bytes96', @@ -536,7 +536,7 @@ The types are defined topologically to aid in facilitating an executable version { # Header 'slot': 'uint64', - 'previous_block_root': 'bytes32', + 'parent_block_root': 'bytes32', 'state_root': 'bytes32', 'body': BeaconBlockBody, 'signature': 'bytes96', @@ -1306,12 +1306,12 @@ def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestat def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[Crosslink, List[ValidatorIndex]]: attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.crosslink.shard == shard] crosslinks = list(filter( - lambda c: hash_tree_root(state.current_crosslinks[shard]) in (c.previous_crosslink_root, hash_tree_root(c)), + lambda c: hash_tree_root(state.current_crosslinks[shard]) in (c.parent_crosslink_root, hash_tree_root(c)), [a.data.crosslink for a in attestations] )) # Winning crosslink has the crosslink data root with the most balance voting for it (ties broken lexicographically) winning_crosslink = max(crosslinks, key=lambda c: ( - get_attesting_balance(state, [a for a in attestations if a.data.crosslink == c]), c.crosslink_data_root + get_attesting_balance(state, [a for a in attestations if a.data.crosslink == c]), c.data_root ), default=Crosslink()) winning_attestations = [a for a in attestations if a.data.crosslink == winning_crosslink] return winning_crosslink, get_unslashed_attesting_indices(state, winning_attestations) @@ -1586,11 +1586,11 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: # Verify that the slots match assert block.slot == state.slot # Verify that the parent matches - assert block.previous_block_root == signing_root(state.latest_block_header) + assert block.parent_block_root == signing_root(state.latest_block_header) # Save current block as the new latest block state.latest_block_header = BeaconBlockHeader( slot=block.slot, - previous_block_root=block.previous_block_root, + parent_block_root=block.parent_block_root, block_body_root=hash_tree_root(block.body), ) # Verify proposer is not slashed @@ -1707,18 +1707,18 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.target_epoch in (get_previous_epoch(state), get_current_epoch(state)) if data.target_epoch == get_current_epoch(state): ffg_data = (state.current_justified_epoch, state.current_justified_root, get_current_epoch(state)) - previous_crosslink = state.current_crosslinks[data.crosslink.shard] + parent_crosslink = state.current_crosslinks[data.crosslink.shard] state.current_epoch_attestations.append(pending_attestation) else: ffg_data = (state.previous_justified_epoch, state.previous_justified_root, get_previous_epoch(state)) - previous_crosslink = state.previous_crosslinks[data.crosslink.shard] + parent_crosslink = state.previous_crosslinks[data.crosslink.shard] state.previous_epoch_attestations.append(pending_attestation) # Check FFG data, crosslink data, and signature assert ffg_data == (data.source_epoch, data.source_root, data.target_epoch) - assert data.crosslink.epoch == min(data.target_epoch, previous_crosslink.epoch + MAX_EPOCHS_PER_CROSSLINK) - assert data.crosslink.previous_crosslink_root == hash_tree_root(previous_crosslink) - assert data.crosslink.crosslink_data_root == ZERO_HASH # [to be removed in phase 1] + assert data.crosslink.epoch == min(data.target_epoch, parent_crosslink.epoch + MAX_EPOCHS_PER_CROSSLINK) + assert data.crosslink.parent_crosslink_root == hash_tree_root(parent_crosslink) + assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] assert verify_indexed_attestation(state, convert_to_indexed(state, attestation)) ``` diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 549e9e2077..891120de8e 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -38,7 +38,7 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph Processing the beacon chain is similar to processing the Ethereum 1.0 chain. Clients download and process blocks and maintain a view of what is the current "canonical chain", terminating at the current "head". For a beacon block, `block`, to be processed by a node, the following conditions must be met: -* The parent block with root `block.previous_block_root` has been processed and accepted. +* The parent block with root `block.parent_block_root` has been processed and accepted. * An Ethereum 1.0 block pointed to by the `state.latest_eth1_data.block_hash` has been processed and accepted. * The node's Unix time is greater than or equal to `state.genesis_time + block.slot * SECONDS_PER_SLOT`. diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index e259b82a55..a030c577bf 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -147,7 +147,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether 'challenger_index': ValidatorIndex, 'responder_index': ValidatorIndex, 'deadline': Epoch, - 'crosslink_data_root': Hash, + 'data_root': Hash, 'depth': 'uint64', 'chunk_index': 'uint64', } @@ -161,7 +161,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether 'challenger_index': ValidatorIndex, 'responder_index': ValidatorIndex, 'deadline': Epoch, - 'crosslink_data_root': Hash, + 'data_root': Hash, 'chunk_count': 'uint64', 'chunk_bits_merkle_root': Hash, 'responder_key': BLSSignature, @@ -474,7 +474,7 @@ def process_chunk_challenge(state: BeaconState, # Verify the challenge is not a duplicate for record in state.custody_chunk_challenge_records: assert ( - record.crosslink_data_root != challenge.attestation.data.crosslink.crosslink_data_root or + record.data_root != challenge.attestation.data.crosslink.data_root or record.chunk_index != challenge.chunk_index ) # Verify depth @@ -486,7 +486,7 @@ def process_chunk_challenge(state: BeaconState, challenger_index=get_beacon_proposer_index(state), responder_index=challenge.responder_index deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE, - crosslink_data_root=challenge.attestation.data.crosslink.crosslink_data_root, + data_root=challenge.attestation.data.crosslink.data_root, depth=depth, chunk_index=challenge.chunk_index, ) @@ -564,7 +564,7 @@ def process_bit_challenge(state: BeaconState, challenger_index=challenge.challenger_index, responder_index=challenge.responder_index, deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE, - crosslink_data_root=challenge.attestation.data.crosslink.crosslink_data_root, + data_root=challenge.attestation.data.crosslink.data_root, chunk_count=chunk_count, chunk_bits_merkle_root=merkle_root(pad_to_power_of_2((challenge.chunk_bits))), responder_key=challenge.responder_key, @@ -610,7 +610,7 @@ def process_chunk_challenge_response(state: BeaconState, branch=response.data_branch, depth=challenge.depth, index=response.chunk_index, - root=challenge.crosslink_data_root, + root=challenge.data_root, ) # Clear the challenge records = state.custody_chunk_challenge_records @@ -632,7 +632,7 @@ def process_bit_challenge_response(state: BeaconState, branch=response.data_branch, depth=math.log2(next_power_of_two(challenge.chunk_count)), index=response.chunk_index, - root=challenge.crosslink_data_root, + root=challenge.data_root, ) # Verify the chunk bit leaf matches the challenge data assert verify_merkle_branch( diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 9fbc696929..316bd0068f 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -78,7 +78,7 @@ This document describes the shard data layer and the shard fork choice rule in P 'slot': Slot, 'shard': Shard, 'beacon_chain_root': Hash, - 'previous_block_root': Hash, + 'parent_block_root': Hash, 'data': ShardBlockBody, 'state_root': Hash, 'attestations': [ShardAttestation], @@ -93,7 +93,7 @@ This document describes the shard data layer and the shard fork choice rule in P 'slot': Slot, 'shard': Shard, 'beacon_chain_root': Hash, - 'previous_block_root': Hash, + 'parent_block_root': Hash, 'body_root': Hash, 'state_root': Hash, 'attestations': [ShardAttestation], @@ -201,7 +201,7 @@ def get_shard_header(block: ShardBlock) -> ShardBlockHeader: slot: block.slot, shard: block.shard, beacon_chain_root: block.beacon_chain_root, - previous_block_root: block.previous_block_root, + parent_block_root: block.parent_block_root, body_root: hash_tree_root(block.body), state_root: block.state_root, attestations: block.attestations, @@ -296,11 +296,11 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], # Check parent block if block.slot == PHASE_1_GENESIS_SLOT: - assert candidate.previous_block_root == ZERO_HASH + assert candidate.parent_block_root == ZERO_HASH else: parent_block = next( block for block in valid_shard_blocks if - signing_root(block) == candidate.previous_block_root + signing_root(block) == candidate.parent_block_root , None) assert parent_block != None assert parent_block.shard == block.shard @@ -378,11 +378,11 @@ def is_valid_beacon_attestation(shard: Shard, # Check previous attestation if candidate.data.previous_crosslink.epoch <= PHASE_1_GENESIS_EPOCH: - assert candidate.data.previous_crosslink.crosslink_data_root == ZERO_HASH + assert candidate.data.previous_crosslink.data_root == ZERO_HASH else: previous_attestation = next( attestation for attestation in valid_attestations if - attestation.data.crosslink.crosslink_data_root == candidate.data.previous_crosslink.crosslink_data_root + attestation.data.crosslink.data_root == candidate.data.previous_crosslink.data_root , None) assert previous_attestation != None assert candidate.data.previous_attestation.epoch < slot_to_epoch(candidate.data.slot) @@ -393,7 +393,7 @@ def is_valid_beacon_attestation(shard: Shard, blocks = [] for slot in range(start_epoch * SLOTS_PER_EPOCH, end_epoch * SLOTS_PER_EPOCH): blocks.append(shard_blocks[slot]) - assert candidate.data.crosslink.crosslink_data_root == compute_crosslink_data_root(blocks) + assert candidate.data.crosslink.data_root == compute_crosslink_data_root(blocks) return True ``` diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index cfe6a8f1ff..dae9ac85f4 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -152,7 +152,7 @@ Set `block.slot = slot` where `slot` is the current slot at which the validator ##### Parent root -Set `block.previous_block_root = signing_root(parent)`. +Set `block.parent_block_root = signing_root(parent)`. ##### State root @@ -275,11 +275,11 @@ Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associ ##### Previous crosslink root -Set `attestation_data.previous_crosslink_root = hash_tree_root(head_state.current_crosslinks[shard])`. +Set `attestation_data.parent_crosslink_root = hash_tree_root(head_state.current_crosslinks[shard])`. ##### Crosslink data root -Set `attestation_data.crosslink.crosslink_data_root = ZERO_HASH`. +Set `attestation_data.crosslink.data_root = ZERO_HASH`. *Note*: This is a stub for Phase 0. diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index 24cafc2755..fcbc68fe9b 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -113,7 +113,7 @@ def test_non_zero_crosslink_data_root(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation.data.crosslink.crosslink_data_root = b'\x42' * 32 + attestation.data.crosslink.data_root = b'\x42' * 32 pre_state, post_state = run_attestation_processing(state, attestation, False) diff --git a/test_libs/pyspec/tests/block_processing/test_process_block_header.py b/test_libs/pyspec/tests/block_processing/test_process_block_header.py index b35b0a9c11..e4fa516b1e 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/tests/block_processing/test_process_block_header.py @@ -53,9 +53,9 @@ def test_invalid_slot(state): return pre_state, block, None -def test_invalid_previous_block_root(state): +def test_invalid_parent_block_root(state): block = build_empty_block_for_next_slot(state) - block.previous_block_root = b'\12' * 32 # invalid prev root + block.parent_block_root = b'\12' * 32 # invalid prev root pre_state, post_state = run_block_header_processing(state, block, valid=False) return pre_state, block, None diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 1ea299453d..e7ee826db4 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -128,7 +128,7 @@ def build_empty_block_for_next_slot(state): previous_block_header = deepcopy(state.latest_block_header) if previous_block_header.state_root == spec.ZERO_HASH: previous_block_header.state_root = state.hash_tree_root() - empty_block.previous_block_root = signing_root(previous_block_header) + empty_block.parent_block_root = signing_root(previous_block_header) return empty_block @@ -155,7 +155,7 @@ def build_attestation_data(state, slot, shard): assert state.slot >= slot if slot == state.slot: - block_root = build_empty_block_for_next_slot(state).previous_block_root + block_root = build_empty_block_for_next_slot(state).parent_block_root else: block_root = get_block_root_at_slot(state, slot) @@ -184,8 +184,8 @@ def build_attestation_data(state, slot, shard): crosslink=Crosslink( shard=shard, epoch=min(slot_to_epoch(slot), crosslinks[shard].epoch + MAX_EPOCHS_PER_CROSSLINK), - crosslink_data_root=spec.ZERO_HASH, - previous_crosslink_root=hash_tree_root(crosslinks[shard]), + data_root=spec.ZERO_HASH, + parent_crosslink_root=hash_tree_root(crosslinks[shard]), ), ) @@ -240,12 +240,12 @@ def get_valid_proposer_slashing(state): header_1 = BeaconBlockHeader( slot=slot, - previous_block_root=ZERO_HASH, + parent_block_root=ZERO_HASH, state_root=ZERO_HASH, block_body_root=ZERO_HASH, ) header_2 = deepcopy(header_1) - header_2.previous_block_root = b'\x02' * 32 + header_2.parent_block_root = b'\x02' * 32 header_2.slot = slot + 1 domain = get_domain( diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 1b4d20f4c5..83ba9cc113 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -68,7 +68,7 @@ def test_empty_block_transition(state): state_transition(test_state, block) assert len(test_state.eth1_data_votes) == len(state.eth1_data_votes) + 1 - assert get_block_root_at_slot(test_state, state.slot) == block.previous_block_root + assert get_block_root_at_slot(test_state, state.slot) == block.parent_block_root return state, [block], test_state @@ -82,7 +82,7 @@ def test_skipped_slots(state): assert test_state.slot == block.slot for slot in range(state.slot, test_state.slot): - assert get_block_root_at_slot(test_state, slot) == block.previous_block_root + assert get_block_root_at_slot(test_state, slot) == block.parent_block_root return state, [block], test_state @@ -96,7 +96,7 @@ def test_empty_epoch_transition(state): assert test_state.slot == block.slot for slot in range(state.slot, test_state.slot): - assert get_block_root_at_slot(test_state, slot) == block.previous_block_root + assert get_block_root_at_slot(test_state, slot) == block.parent_block_root return state, [block], test_state From ea60fb632c3a07d2dae29b6ca8d37cea204bc6e7 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Mon, 6 May 2019 20:49:46 +0100 Subject: [PATCH 051/308] More renaming --- specs/core/0_beacon-chain.md | 14 +++++++------- specs/core/0_fork-choice.md | 2 +- specs/core/1_shard-data-chains.md | 10 +++++----- specs/validator/0_beacon-chain-validator.md | 4 ++-- .../block_processing/test_process_block_header.py | 2 +- test_libs/pyspec/tests/helpers.py | 12 ++++++------ test_libs/pyspec/tests/test_sanity.py | 6 +++--- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index afca9eb1ba..ccba4c817d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -287,7 +287,7 @@ The types are defined topologically to aid in facilitating an executable version # Epoch number 'epoch': 'uint64', # Root of the previous crosslink - 'parent_crosslink_root': 'bytes32', + 'parent_root': 'bytes32', # Root of the crosslinked shard data since the previous crosslink 'data_root': 'bytes32', } @@ -369,7 +369,7 @@ The types are defined topologically to aid in facilitating an executable version ```python { 'slot': 'uint64', - 'parent_block_root': 'bytes32', + 'parent_root': 'bytes32', 'state_root': 'bytes32', 'block_body_root': 'bytes32', 'signature': 'bytes96', @@ -536,7 +536,7 @@ The types are defined topologically to aid in facilitating an executable version { # Header 'slot': 'uint64', - 'parent_block_root': 'bytes32', + 'parent_root': 'bytes32', 'state_root': 'bytes32', 'body': BeaconBlockBody, 'signature': 'bytes96', @@ -1306,7 +1306,7 @@ def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestat def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[Crosslink, List[ValidatorIndex]]: attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.crosslink.shard == shard] crosslinks = list(filter( - lambda c: hash_tree_root(state.current_crosslinks[shard]) in (c.parent_crosslink_root, hash_tree_root(c)), + lambda c: hash_tree_root(state.current_crosslinks[shard]) in (c.parent_root, hash_tree_root(c)), [a.data.crosslink for a in attestations] )) # Winning crosslink has the crosslink data root with the most balance voting for it (ties broken lexicographically) @@ -1586,11 +1586,11 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: # Verify that the slots match assert block.slot == state.slot # Verify that the parent matches - assert block.parent_block_root == signing_root(state.latest_block_header) + assert block.parent_root == signing_root(state.latest_block_header) # Save current block as the new latest block state.latest_block_header = BeaconBlockHeader( slot=block.slot, - parent_block_root=block.parent_block_root, + parent_root=block.parent_root, block_body_root=hash_tree_root(block.body), ) # Verify proposer is not slashed @@ -1717,7 +1717,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Check FFG data, crosslink data, and signature assert ffg_data == (data.source_epoch, data.source_root, data.target_epoch) assert data.crosslink.epoch == min(data.target_epoch, parent_crosslink.epoch + MAX_EPOCHS_PER_CROSSLINK) - assert data.crosslink.parent_crosslink_root == hash_tree_root(parent_crosslink) + assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] assert verify_indexed_attestation(state, convert_to_indexed(state, attestation)) ``` diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md index 891120de8e..91c3e27ee4 100644 --- a/specs/core/0_fork-choice.md +++ b/specs/core/0_fork-choice.md @@ -38,7 +38,7 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph Processing the beacon chain is similar to processing the Ethereum 1.0 chain. Clients download and process blocks and maintain a view of what is the current "canonical chain", terminating at the current "head". For a beacon block, `block`, to be processed by a node, the following conditions must be met: -* The parent block with root `block.parent_block_root` has been processed and accepted. +* The parent block with root `block.parent_root` has been processed and accepted. * An Ethereum 1.0 block pointed to by the `state.latest_eth1_data.block_hash` has been processed and accepted. * The node's Unix time is greater than or equal to `state.genesis_time + block.slot * SECONDS_PER_SLOT`. diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 316bd0068f..ca59c3ebc4 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -78,7 +78,7 @@ This document describes the shard data layer and the shard fork choice rule in P 'slot': Slot, 'shard': Shard, 'beacon_chain_root': Hash, - 'parent_block_root': Hash, + 'parent_root': Hash, 'data': ShardBlockBody, 'state_root': Hash, 'attestations': [ShardAttestation], @@ -93,7 +93,7 @@ This document describes the shard data layer and the shard fork choice rule in P 'slot': Slot, 'shard': Shard, 'beacon_chain_root': Hash, - 'parent_block_root': Hash, + 'parent_root': Hash, 'body_root': Hash, 'state_root': Hash, 'attestations': [ShardAttestation], @@ -201,7 +201,7 @@ def get_shard_header(block: ShardBlock) -> ShardBlockHeader: slot: block.slot, shard: block.shard, beacon_chain_root: block.beacon_chain_root, - parent_block_root: block.parent_block_root, + parent_root: block.parent_root, body_root: hash_tree_root(block.body), state_root: block.state_root, attestations: block.attestations, @@ -296,11 +296,11 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], # Check parent block if block.slot == PHASE_1_GENESIS_SLOT: - assert candidate.parent_block_root == ZERO_HASH + assert candidate.parent_root == ZERO_HASH else: parent_block = next( block for block in valid_shard_blocks if - signing_root(block) == candidate.parent_block_root + signing_root(block) == candidate.parent_root , None) assert parent_block != None assert parent_block.shard == block.shard diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index dae9ac85f4..a0173ebe92 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -152,7 +152,7 @@ Set `block.slot = slot` where `slot` is the current slot at which the validator ##### Parent root -Set `block.parent_block_root = signing_root(parent)`. +Set `block.parent_root = signing_root(parent)`. ##### State root @@ -275,7 +275,7 @@ Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associ ##### Previous crosslink root -Set `attestation_data.parent_crosslink_root = hash_tree_root(head_state.current_crosslinks[shard])`. +Set `attestation_data.parent_root = hash_tree_root(head_state.current_crosslinks[shard])`. ##### Crosslink data root diff --git a/test_libs/pyspec/tests/block_processing/test_process_block_header.py b/test_libs/pyspec/tests/block_processing/test_process_block_header.py index e4fa516b1e..36c7298210 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/tests/block_processing/test_process_block_header.py @@ -55,7 +55,7 @@ def test_invalid_slot(state): def test_invalid_parent_block_root(state): block = build_empty_block_for_next_slot(state) - block.parent_block_root = b'\12' * 32 # invalid prev root + block.parent_root = b'\12' * 32 # invalid prev root pre_state, post_state = run_block_header_processing(state, block, valid=False) return pre_state, block, None diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index e7ee826db4..2024885763 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -128,7 +128,7 @@ def build_empty_block_for_next_slot(state): previous_block_header = deepcopy(state.latest_block_header) if previous_block_header.state_root == spec.ZERO_HASH: previous_block_header.state_root = state.hash_tree_root() - empty_block.parent_block_root = signing_root(previous_block_header) + empty_block.parent_root = signing_root(previous_block_header) return empty_block @@ -155,7 +155,7 @@ def build_attestation_data(state, slot, shard): assert state.slot >= slot if slot == state.slot: - block_root = build_empty_block_for_next_slot(state).parent_block_root + block_root = build_empty_block_for_next_slot(state).parent_root else: block_root = get_block_root_at_slot(state, slot) @@ -185,7 +185,7 @@ def build_attestation_data(state, slot, shard): shard=shard, epoch=min(slot_to_epoch(slot), crosslinks[shard].epoch + MAX_EPOCHS_PER_CROSSLINK), data_root=spec.ZERO_HASH, - parent_crosslink_root=hash_tree_root(crosslinks[shard]), + parent_root=hash_tree_root(crosslinks[shard]), ), ) @@ -240,12 +240,12 @@ def get_valid_proposer_slashing(state): header_1 = BeaconBlockHeader( slot=slot, - parent_block_root=ZERO_HASH, + parent_root=ZERO_HASH, state_root=ZERO_HASH, - block_body_root=ZERO_HASH, + body_root=ZERO_HASH, ) header_2 = deepcopy(header_1) - header_2.parent_block_root = b'\x02' * 32 + header_2.parent_root = b'\x02' * 32 header_2.slot = slot + 1 domain = get_domain( diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 83ba9cc113..0cb1b9be3b 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -68,7 +68,7 @@ def test_empty_block_transition(state): state_transition(test_state, block) assert len(test_state.eth1_data_votes) == len(state.eth1_data_votes) + 1 - assert get_block_root_at_slot(test_state, state.slot) == block.parent_block_root + assert get_block_root_at_slot(test_state, state.slot) == block.parent_root return state, [block], test_state @@ -82,7 +82,7 @@ def test_skipped_slots(state): assert test_state.slot == block.slot for slot in range(state.slot, test_state.slot): - assert get_block_root_at_slot(test_state, slot) == block.parent_block_root + assert get_block_root_at_slot(test_state, slot) == block.parent_root return state, [block], test_state @@ -96,7 +96,7 @@ def test_empty_epoch_transition(state): assert test_state.slot == block.slot for slot in range(state.slot, test_state.slot): - assert get_block_root_at_slot(test_state, slot) == block.parent_block_root + assert get_block_root_at_slot(test_state, slot) == block.parent_root return state, [block], test_state From 3a309155aa51288103b6d4f09343b68389b9dca7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 6 May 2019 22:06:00 +0200 Subject: [PATCH 052/308] fix deposit domain: forks are ignored for deposit validity, deposits are always accepted, if coming from the correct contract(s). --- specs/core/0_beacon-chain.md | 6 ++++-- specs/validator/0_beacon-chain-validator.md | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index dd2d3d1a64..cee97f9f50 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -966,7 +966,8 @@ def get_domain(state: BeaconState, """ epoch = get_current_epoch(state) if message_epoch is None else message_epoch fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version - return bytes_to_int(fork_version + int_to_bytes4(domain_type)) + # fork version is on the big-endian side: when signing using only the type (e.g. deposits), the type can be passed directly. + return bytes_to_int(int_to_bytes4(domain_type) + fork_version) ``` ### `get_bitfield_bit` @@ -1766,7 +1767,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: validator_pubkeys = [v.pubkey for v in state.validator_registry] if pubkey not in validator_pubkeys: # Verify the deposit signature (proof of possession) - if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, get_domain(state, DOMAIN_DEPOSIT)): + # Note: deposits are valid regardless of fork version, hence the type is passed directly as domain. + if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, DOMAIN_DEPOSIT): return # Add validator and balance entries diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 600ed08391..a74fbb80f5 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -103,7 +103,7 @@ To submit a deposit: * Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](../core/0_beacon-chain.md#depositdata) SSZ object. * Let `amount` be the amount in Gwei to be deposited by the validator where `MIN_DEPOSIT_AMOUNT <= amount <= MAX_EFFECTIVE_BALANCE`. * Set `deposit_data.amount = amount`. -* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=DOMAIN_DEPOSIT`. +* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=DOMAIN_DEPOSIT`. (Deposits are valid regardless of fork version, hence the type is passed directly as domain.) * Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])` along with a deposit of `amount` Gwei. *Note*: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validator_registry` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. From 54fd822b55aa6fd9426253716c0e6da70d43b367 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Mon, 6 May 2019 17:36:55 -0700 Subject: [PATCH 053/308] Update simple-serialize.md typo --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 343c967d4a..834f566458 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -85,7 +85,7 @@ return b"\x01" if value is True else b"\x00" ### Containers, vectors, lists ```python -# Reccursively serialize +# Recursively serialize fixed_parts = [serialize(element) if not is_variable_size(element) else None for element in value] variable_parts = [serialize(element) if is_variable_size(element) else b"" for element in value] From 24966d71be58066dc9586d355ec219220e5bd94a Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 6 May 2019 17:54:40 -0700 Subject: [PATCH 054/308] Update 0_beacon-chain-validator.md --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 600ed08391..0a19977888 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -134,7 +134,7 @@ Once a validator is activated, the validator is assigned [responsibilities](#bea ## Beacon chain responsibilities -A validator has two primary responsibilities to the beacon chain: [proposing blocks](block-proposal) and [creating attestations](attestations-1). Proposals happen infrequently, whereas attestations should be created once per epoch. +A validator has two primary responsibilities to the beacon chain: [proposing blocks](#block-proposal) and [creating attestations](#attestations-1). Proposals happen infrequently, whereas attestations should be created once per epoch. ### Block proposal From fc1239c0ff8fda299d4a5e5a532bb2203e130446 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Mon, 6 May 2019 18:50:20 -0700 Subject: [PATCH 055/308] Add clarity around merkleize on a single chunk --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 834f566458..c02d7fbedf 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -111,7 +111,7 @@ Because serialization is an injective function (i.e. two distinct objects of the We first define helper functions: * `pack`: Given ordered objects of the same basic type, serialize them, pack them into `BYTES_PER_CHUNK`-byte chunks, right-pad the last chunk with zero bytes, and return the chunks. -* `merkleize`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. +* `merkleize`: Given ordered `BYTES_PER_CHUNK`-byte chunks, if necessary append zero chunks so that the number of chunks is a power of two, Merkleize the chunks, and return the root. Note that `merkleize` on a single chunk is simply that chunk, i.e. the identity when the number of chunks is one. * `mix_in_length`: Given a Merkle root `root` and a length `length` (`"uint256"` little-endian serialization) return `hash(root + length)`. We now define Merkleization `hash_tree_root(value)` of an object `value` recursively: From 964e55cd4a8b8bfecbd70e279c13e69c6c977e22 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Tue, 7 May 2019 08:52:56 +0100 Subject: [PATCH 056/308] block_body_root => body_root --- specs/core/0_beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ccba4c817d..9f80fe1736 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -371,7 +371,7 @@ The types are defined topologically to aid in facilitating an executable version 'slot': 'uint64', 'parent_root': 'bytes32', 'state_root': 'bytes32', - 'block_body_root': 'bytes32', + 'body_root': 'bytes32', 'signature': 'bytes96', } ``` @@ -1591,7 +1591,7 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: state.latest_block_header = BeaconBlockHeader( slot=block.slot, parent_root=block.parent_root, - block_body_root=hash_tree_root(block.body), + body_root=hash_tree_root(block.body), ) # Verify proposer is not slashed proposer = state.validator_registry[get_beacon_proposer_index(state)] From f371daeb204c1ede12fcb1bf6e5b15263f1bc504 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 7 May 2019 09:01:07 +0100 Subject: [PATCH 057/308] Update specs/core/0_beacon-chain.md Co-Authored-By: JustinDrake --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8d999c65cc..aaa85e425c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1265,7 +1265,7 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. ```python -def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconBlock: +def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: # Process slots (including those with no blocks) since block process_slots(state, block.slot) # Process block From 50009ea85bd1fd96b81c3f3b6fb3f39b58662050 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Tue, 7 May 2019 10:12:33 +0100 Subject: [PATCH 058/308] Implement HW's exception-handling suggestion --- scripts/phase0/build_spec.py | 5 +++-- specs/core/0_beacon-chain.md | 17 +++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index 54adfdde72..a1f0209bbb 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -5,7 +5,8 @@ def build_phase0_spec(sourcefile, outfile): code_lines = [] code_lines.append(""" - + +import copy from typing import ( Any, Callable, @@ -88,7 +89,7 @@ def apply_constants_preset(preset: Dict[str, Any]): # Deal with derived constants global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT) - + # Initialize SSZ types again, to account for changed lengths init_SSZ_types() """) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index aaa85e425c..84d780ad6f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1262,16 +1262,21 @@ def get_genesis_beacon_state(genesis_validator_deposits: List[Deposit], ## Beacon chain state transition function -The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. +The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. ```python def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: - # Process slots (including those with no blocks) since block - process_slots(state, block.slot) - # Process block - process_block(state, block) - # Return post-state + pre_state = copy.deepcopy(state) + try: + # Process slots (including those with no blocks) since block + process_slots(state, block.slot) + # Process block + process_block(state, block) + except Exception: + # State is not advanced on exceptions (e.g. a failed `assert` or an out-of-range list access) + return pre_state return state + ``` ```python From b1520ea96766e34397c161cea3518e7e7f0104be Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 7 May 2019 10:33:51 +0100 Subject: [PATCH 059/308] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index dd2d3d1a64..f0798f5877 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -75,7 +75,7 @@ - [`compute_committee`](#compute_committee) - [`get_crosslink_committee`](#get_crosslink_committee) - [`get_attesting_indices`](#get_attesting_indices) - - [`int_to_bytes1`, `int_to_bytes2`, ...](#int_to_bytes1-int_to_bytes2-) + - [`int_to_bytes`](#int_to_bytes) - [`bytes_to_int`](#bytes_to_int) - [`get_total_balance`](#get_total_balance) - [`get_domain`](#get_domain) @@ -194,8 +194,8 @@ These configurations are updated for releases, but may be out of sync during `de | `GENESIS_SLOT` | `0` | | `GENESIS_EPOCH` | `0` | | `FAR_FUTURE_EPOCH` | `2**64 - 1` | -| `ZERO_HASH` | `int_to_bytes32(0)` | -| `BLS_WITHDRAWAL_PREFIX_BYTE` | `int_to_bytes1(0)` | +| `ZERO_HASH` | `int_to_bytes(0, length=32)` | +| `BLS_WITHDRAWAL_PREFIX_BYTE` | `int_to_bytes(0, length=1)` | ### Time parameters @@ -830,7 +830,7 @@ def generate_seed(state: BeaconState, return hash( get_randao_mix(state, epoch + LATEST_RANDAO_MIXES_LENGTH - MIN_SEED_LOOKAHEAD) + get_active_index_root(state, epoch) + - int_to_bytes32(epoch) + int_to_bytes(epoch, length=32) ) ``` @@ -851,7 +851,7 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: i = 0 while True: candidate_index = first_committee[(epoch + i) % len(first_committee)] - random_byte = hash(seed + int_to_bytes8(i // 32))[i % 32] + random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] effective_balance = state.validator_registry[candidate_index].effective_balance if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: return candidate_index @@ -888,10 +888,10 @@ def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Bytes32) - # Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf) # See the 'generalized domain' algorithm on page 3 for round in range(SHUFFLE_ROUND_COUNT): - pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % index_count + pivot = bytes_to_int(hash(seed + int_to_bytes(round, length=1))[0:8]) % index_count flip = (pivot - index) % index_count position = max(index, flip) - source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256)) + source = hash(seed + int_to_bytes(round, length=1) + int_to_bytes(position // 256, length=4)) byte = source[(position % 256) // 8] bit = (byte >> (position % 8)) % 2 index = flip if bit else index @@ -934,9 +934,12 @@ def get_attesting_indices(state: BeaconState, return sorted([index for i, index in enumerate(committee) if get_bitfield_bit(bitfield, i) == 0b1]) ``` -### `int_to_bytes1`, `int_to_bytes2`, ... +### `int_to_bytes` -`int_to_bytes1(x): return x.to_bytes(1, 'little')`, `int_to_bytes2(x): return x.to_bytes(2, 'little')`, and so on for all integers, particularly 1, 2, 3, 4, 8, 32, 48, 96. +```python +def int_to_bytes(integer: int, length: int) -> bytes: + return integer.to_bytes(length, 'little') +``` ### `bytes_to_int` @@ -966,7 +969,7 @@ def get_domain(state: BeaconState, """ epoch = get_current_epoch(state) if message_epoch is None else message_epoch fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version - return bytes_to_int(fork_version + int_to_bytes4(domain_type)) + return bytes_to_int(fork_version + int_to_bytes(domain_type, length=4)) ``` ### `get_bitfield_bit` From 22b06d581dcdc84ca3cfedd9e21af529557d14d6 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Tue, 7 May 2019 10:57:41 +0100 Subject: [PATCH 060/308] Update instances of int_to_bytes --- scripts/phase0/build_spec.py | 8 ++------ specs/core/0_beacon-chain.md | 4 ++-- specs/core/1_shard-data-chains.md | 2 +- specs/light_client/sync_protocol.md | 2 +- test_generators/shuffling/main.py | 2 +- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index da5845951d..28e5049ae4 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -15,15 +15,11 @@ def build_phase0_spec(sourcefile, outfile): from eth2spec.utils.minimal_ssz import * from eth2spec.utils.bls_stub import * -""") - for i in (1, 2, 3, 4, 8, 32, 48, 96): - code_lines.append("def int_to_bytes%d(x): return x.to_bytes(%d, 'little')" % (i, i)) - - code_lines.append(""" - # stub, will get overwritten by real var SLOTS_PER_EPOCH = 64 +def int_to_bytes(integer: int, length: int) -> bytes: + return integer.to_bytes(length, 'little') Slot = NewType('Slot', int) # uint64 Epoch = NewType('Epoch', int) # uint64 diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f0798f5877..1e81700365 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -195,7 +195,7 @@ These configurations are updated for releases, but may be out of sync during `de | `GENESIS_EPOCH` | `0` | | `FAR_FUTURE_EPOCH` | `2**64 - 1` | | `ZERO_HASH` | `int_to_bytes(0, length=32)` | -| `BLS_WITHDRAWAL_PREFIX_BYTE` | `int_to_bytes(0, length=1)` | +| `BLS_WITHDRAWAL_PREFIX` | `0` | ### Time parameters @@ -1840,7 +1840,7 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: # Verify that the pubkey is valid assert ( state.validator_registry[transfer.sender].withdrawal_credentials == - BLS_WITHDRAWAL_PREFIX_BYTE + hash(transfer.pubkey)[1:] + int_to_bytes(BLS_WITHDRAWAL_PREFIX, length=1) + hash(transfer.pubkey)[1:] ) # Verify that the signature is valid assert bls_verify(transfer.pubkey, signing_root(transfer), transfer.signature, get_domain(state, DOMAIN_TRANSFER)) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 14eb511931..90ea308413 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -180,7 +180,7 @@ def get_shard_proposer_index(state: BeaconState, slot: Slot) -> ValidatorIndex: # Randomly shift persistent committee persistent_committee = get_persistent_committee(state, shard, slot) - seed = hash(state.current_shuffling_seed + int_to_bytes8(shard) + int_to_bytes8(slot)) + seed = hash(state.current_shuffling_seed + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) random_index = bytes_to_int(seed[0:8]) % len(persistent_committee) persistent_committee = persistent_committee[random_index:] + persistent_committee[:random_index] diff --git a/specs/light_client/sync_protocol.md b/specs/light_client/sync_protocol.md index f6e3d22657..8501c58690 100644 --- a/specs/light_client/sync_protocol.md +++ b/specs/light_client/sync_protocol.md @@ -146,7 +146,7 @@ def compute_committee(header: BeaconBlockHeader, ] def get_switchover_epoch(index): return ( - bytes_to_int(hash(validator_memory.earlier_period_data.seed + int_to_bytes3(index))[0:8]) % + bytes_to_int(hash(validator_memory.earlier_period_data.seed + int_to_bytes(index, length=3))[0:8]) % PERSISTENT_COMMITTEE_PERIOD ) diff --git a/test_generators/shuffling/main.py b/test_generators/shuffling/main.py index 2c4faeb8fb..9ca3e2d36f 100644 --- a/test_generators/shuffling/main.py +++ b/test_generators/shuffling/main.py @@ -15,7 +15,7 @@ def shuffling_case(seed: spec.Bytes32, count: int): @to_tuple def shuffling_test_cases(): - for seed in [spec.hash(spec.int_to_bytes4(seed_init_value)) for seed_init_value in range(30)]: + for seed in [spec.hash(spec.int_to_bytes(seed_init_value, length=4)) for seed_init_value in range(30)]: for count in [0, 1, 2, 3, 5, 10, 33, 100, 1000]: yield shuffling_case(seed, count) From a604d03dff66aed47a0790525651267008511fae Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Tue, 7 May 2019 12:13:22 +0100 Subject: [PATCH 061/308] Basic phase 1 pulling + correcting syntax errors in phase 1 --- Makefile | 10 +++++ scripts/phase0/build_spec.py | 35 +++++++++++++-- scripts/phase0/function_puller.py | 19 ++++++-- specs/core/1_custody-game.md | 19 +++++--- specs/core/1_shard-data-chains.md | 44 +++++++++---------- .../pyspec/eth2spec/utils/minimal_ssz.py | 4 ++ 6 files changed, 96 insertions(+), 35 deletions(-) diff --git a/Makefile b/Makefile index 73d8adea89..420fc32769 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERAT GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS)) PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py +PY_SPEC_PHASE_1_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase1/custody-game.py $(PY_SPEC_DIR)/eth2spec/phase1/shard-data-chains.py PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) @@ -25,6 +26,7 @@ clean: rm -rf $(GENERATOR_VENVS) rm -rf $(PY_SPEC_DIR)/venv $(PY_SPEC_DIR)/.pytest_cache rm -rf $(PY_SPEC_ALL_TARGETS) + rm -rf $(PY_SPEC_PHASE_1_TARGETS) # "make gen_yaml_tests" to run generators gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS) @@ -45,10 +47,18 @@ pyspec: $(PY_SPEC_ALL_TARGETS) # "make phase0" to create pyspec for phase0 phase0: $(PY_SPEC_PHASE_0_TARGETS) +# "make phase1" to create pyspec for phase1 +phase1: $(PY_SPEC_PHASE_1_TARGETS) $(PY_SPEC_DIR)/eth2spec/phase0/spec.py: python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@ +$(PY_SPEC_DIR)/eth2spec/phase1/custody-game.py: + python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_custody-game.md $@ + +$(PY_SPEC_DIR)/eth2spec/phase1/shard-data-chains.py: + python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_shard-data-chains.md $@ + CURRENT_DIR = ${CURDIR} diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index da5845951d..80b378fbb9 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -1,6 +1,6 @@ import sys import function_puller - +from optparse import OptionParser def build_phase0_spec(sourcefile, outfile): code_lines = [] @@ -85,8 +85,35 @@ def apply_constants_preset(preset: Dict[str, Any]): out.write("\n".join(code_lines)) +def build_phase1_spec(sourcefile, outfile): + code_lines = [] + code_lines.append(""" +from eth2spec.phase0.spec import * + +""") + + code_lines += function_puller.get_spec(sourcefile) + + with open(outfile, 'w') as out: + out.write("\n".join(code_lines)) + + if __name__ == '__main__': - if len(sys.argv) < 3: - print("Usage: ") - build_phase0_spec(sys.argv[1], sys.argv[2]) + parser = OptionParser() + parser.add_option("-p", "--phase", dest="phase", type="int", default=0, + help="Build for phase #") + + (options, args) = parser.parse_args() + + if len(args) < 2: + parser.print_help() + + if options.phase == 0: + build_phase0_spec(args[0], args[1]) + print(args[0]) + print(args[1]) + elif options.phase == 1: + build_phase1_spec(args[0], args[1]) + else: + print("Invalid phase: {0}".format(options["phase"])) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 1fad41fa9d..393f821d56 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -2,16 +2,26 @@ from typing import List -def get_spec(file_name: str) -> List[str]: +def get_spec(file_name: str, phase:int = 0) -> List[str]: code_lines = [] pulling_from = None current_name = None current_typedef = None + is_update_section = False + update_section_depth = None type_defs = [] - for linenum, line in enumerate(open(sys.argv[1]).readlines()): + for linenum, line in enumerate(open(file_name).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': current_name = line[line[:-1].rfind('`') + 1: -1] + if pulling_from is None and len(line) > 0 and line[0] == '#' and line.endswith('updates'): + is_update_section = True + update_section_depth = max(i for i in range(10) if line.startswith('#' * i)) + elif pulling_from is None and len(line) > 0 and line[0] == '#' and is_update_section: + section_depth = max(i for i in range(10) if line.startswith('#' * i)) + if section_depth <= update_section_depth: + is_update_section = False + update_section_depth = None if line[:9] == '```python': assert pulling_from is None pulling_from = linenum + 1 @@ -28,7 +38,10 @@ def get_spec(file_name: str) -> List[str]: current_typedef = None else: if pulling_from == linenum and line == '{': - code_lines.append('%s = SSZType({' % current_name) + if is_update_section: + code_lines.append('%s = SSZTypeExtension({' % current_name) + else: + code_lines.append('%s = SSZType({' % current_name) current_typedef = ['global_vars["%s"] = SSZType({' % current_name] elif pulling_from is not None: # Add some whitespace between functions diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index d565266110..09e876ec35 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -220,16 +220,19 @@ Add the following fields to the end of the specified container objects. Fields w #### `Validator` ```python +{ # next_custody_reveal_period is initialised to the custody period # (of the particular validator) in which the validator is activated # = get_validators_custody_reveal_period(...) 'next_custody_reveal_period': 'uint64', 'max_reveal_lateness': 'uint64', +} ``` #### `BeaconState` ```python +{ 'custody_chunk_challenge_records': [CustodyChunkChallengeRecord], 'custody_bit_challenge_records': [CustodyBitChallengeRecord], 'custody_challenge_index': 'uint64', @@ -237,16 +240,19 @@ Add the following fields to the end of the specified container objects. Fields w # Future derived secrets already exposed; contains the indices of the exposed validator # at RANDAO reveal period % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS 'exposed_derived_secrets': [['uint64'], EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS], +} ``` #### `BeaconBlockBody` ```python +{ 'custody_chunk_challenges': [CustodyChunkChallenge], 'custody_bit_challenges': [CustodyBitChallenge], 'custody_responses': [CustodyResponse], 'custody_key_reveals': [CustodyKeyReveal], 'early_derived_secret_reveals': [EarlyDerivedSecretReveal], +} ``` ## Helpers @@ -299,7 +305,7 @@ def get_randao_epoch_for_custody_period(period: int, validator_index: ValidatorI ### `get_validators_custody_reveal_period` - ```python +```python def get_validators_custody_reveal_period(state: BeaconState, validator_index: ValidatorIndex, epoch: Epoch=None) -> int: @@ -484,7 +490,7 @@ def process_chunk_challenge(state: BeaconState, new_record = CustodyChunkChallengeRecord( challenge_index=state.custody_challenge_index, challenger_index=get_beacon_proposer_index(state), - responder_index=challenge.responder_index + responder_index=challenge.responder_index, deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE, crosslink_data_root=challenge.attestation.data.crosslink_data_root, depth=depth, @@ -538,7 +544,7 @@ def process_bit_challenge(state: BeaconState, get_validators_custody_reveal_period( state=state, index=challenge.responder_index, - epoch=slot_to_epoch(attestation.data.slot), + epoch=slot_to_epoch(attestation.data.slot)), challenge.responder_index ) assert bls_verify( @@ -585,11 +591,11 @@ For each `response` in `block.body.custody_responses`, run the following functio ```python def process_custody_response(state: BeaconState, response: CustodyResponse) -> None: - chunk_challenge = next(record for record in state.custody_chunk_challenge_records if record.challenge_index == response.challenge_index, None) + chunk_challenge = next((record for record in state.custody_chunk_challenge_records if record.challenge_index == response.challenge_index), None) if chunk_challenge is not None: return process_chunk_challenge_response(state, response, chunk_challenge) - bit_challenge = next(record for record in state.custody_bit_challenge_records if record.challenge_index == response.challenge_index, None) + bit_challenge = next((record for record in state.custody_bit_challenge_records if record.challenge_index == response.challenge_index), None) if bit_challenge is not None: return process_bit_challenge_response(state, response, bit_challenge) @@ -657,7 +663,7 @@ def process_bit_challenge_response(state: BeaconState, Run `process_reveal_deadlines(state)` immediately after `process_ejections(state)`: - ```python +```python def process_reveal_deadlines(state: BeaconState) -> None: for index, validator in enumerate(state.validator_registry): if (validator.latest_custody_reveal_period + @@ -686,6 +692,7 @@ def process_challenge_deadlines(state: BeaconState) -> None: Append this to `process_final_updates(state)`: ```python +def after_process_final_updates(state: BeaconState) -> None: # Clean up exposed RANDAO key reveals state.exposed_derived_secrets[current_epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] ``` diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 33ef8632b0..ffb7926f83 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -53,8 +53,8 @@ This document describes the shard data layer and the shard fork choice rule in P | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `CROSSLINK_LOOKBACK` | 2**0 (= 1) | epochs | 6.2 minutes | -| `PERSISTENT_COMMITTEE_PERIOD` | 2**11 (= 2,048) | epochs | ~9 days | +| `CROSSLINK_LOOKBACK` | `2**0` (= 1) | epochs | 6.2 minutes | +| `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | ### Signature domains @@ -198,14 +198,14 @@ def get_shard_proposer_index(state: BeaconState, ```python def get_shard_header(block: ShardBlock) -> ShardBlockHeader: return ShardBlockHeader( - slot: block.slot, - shard: block.shard, - beacon_chain_root: block.beacon_chain_root, - previous_block_root: block.previous_block_root, - body_root: hash_tree_root(block.body), - state_root: block.state_root, - attestations: block.attestations, - signature: block.signature, + slot=block.slot, + shard=block.shard, + beacon_chain_root=block.beacon_chain_root, + previous_block_root=block.previous_block_root, + body_root=hash_tree_root(block.body), + state_root=block.state_root, + attestations=block.attestations, + signature=block.signature, ) ``` @@ -219,7 +219,7 @@ def verify_shard_attestation_signature(state: BeaconState, assert verify_bitfield(attestation.aggregation_bitfield, len(persistent_committee)) pubkeys = [] for i, index in enumerate(persistent_committee): - if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b1 + if get_bitfield_bit(attestation.aggregation_bitfield, i) == 0b1: validator = state.validator_registry[index] assert is_active_validator(validator, get_current_epoch(state)) pubkeys.append(validator.pubkey) @@ -273,7 +273,7 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], beacon_state: BeaconState, valid_shard_blocks: List[ShardBlock], unix_time: uint64, - candidate: ShardBlock) -> bool + candidate: ShardBlock) -> bool: # Check if block is already determined valid for _, block in enumerate(valid_shard_blocks): if candidate == block: @@ -289,7 +289,7 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], # Check beacon block beacon_block = beacon_blocks[block.slot] assert block.beacon_block_root == signing_root(beacon_block) - assert beacon_block.slot <= block.slot: + assert beacon_block.slot <= block.slot # Check state root assert block.state_root == ZERO_HASH # [to be removed in phase 2] @@ -299,9 +299,9 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], assert candidate.previous_block_root == ZERO_HASH else: parent_block = next( - block for block in valid_shard_blocks if - signing_root(block) == candidate.previous_block_root - , None) + (block for block in valid_shard_blocks if + signing_root(block) == candidate.previous_block_root), + None) assert parent_block != None assert parent_block.shard == block.shard assert parent_block.slot < block.slot @@ -342,9 +342,9 @@ def is_valid_shard_attestation(valid_shard_blocks: List[ShardBlock], candidate: Attestation) -> bool: # Check shard block shard_block = next( - block for block in valid_shard_blocks if - signing_root(block) == candidate.attestation.data.shard_block_root - , None) + (block for block in valid_shard_blocks if + signing_root(block) == candidate.attestation.data.shard_block_root), + None) assert shard_block != None assert shard_block.slot == attestation.data.slot assert shard_block.shard == attestation.data.shard @@ -381,9 +381,9 @@ def is_valid_beacon_attestation(shard: Shard, assert candidate.data.previous_crosslink.crosslink_data_root == ZERO_HASH else: previous_attestation = next( - attestation for attestation in valid_attestations if - attestation.data.crosslink_data_root == candidate.data.previous_crosslink.crosslink_data_root - , None) + (attestation for attestation in valid_attestations if + attestation.data.crosslink_data_root == candidate.data.previous_crosslink.crosslink_data_root), + None) assert previous_attestation != None assert candidate.data.previous_attestation.epoch < slot_to_epoch(candidate.data.slot) diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py index 9cc2baebb3..cfd2e56ac4 100644 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py @@ -37,6 +37,10 @@ def hash_tree_root(self): SSZObject.fields = fields return SSZObject +def SSZTypeExtension(original_type, new_fields): + fields = original_type.fields.copy() + fields.update(original_type.fields) + return SSZType(fields) class Vector(): def __init__(self, items): From 4ad92a0989bec522b5ccdf2d7969a76bfeaac715 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Tue, 7 May 2019 13:23:28 +0100 Subject: [PATCH 062/308] Fix typing --- scripts/phase0/function_puller.py | 2 +- specs/core/1_custody-game.md | 16 +++++----- specs/core/1_shard-data-chains.md | 52 ++++++++++++++++--------------- 3 files changed, 36 insertions(+), 34 deletions(-) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 393f821d56..830e52a32d 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -39,7 +39,7 @@ def get_spec(file_name: str, phase:int = 0) -> List[str]: else: if pulling_from == linenum and line == '{': if is_update_section: - code_lines.append('%s = SSZTypeExtension({' % current_name) + code_lines.append('%s = SSZTypeExtension(%s, {' % (current_name, current_name)) else: code_lines.append('%s = SSZType({' % current_name) current_typedef = ['global_vars["%s"] = SSZType({' % current_name] diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index c510abf6c0..5de52a4ea3 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -134,7 +134,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether 'attestation': Attestation, 'challenger_index': ValidatorIndex, 'responder_key': BLSSignature, - 'chunk_bits': Bitfield, + 'chunk_bits': "bytes", 'signature': BLSSignature, } ``` @@ -147,7 +147,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether 'challenger_index': ValidatorIndex, 'responder_index': ValidatorIndex, 'deadline': Epoch, - 'crosslink_data_root': Hash, + 'crosslink_data_root': 'bytes32', 'depth': 'uint64', 'chunk_index': 'uint64', } @@ -161,9 +161,9 @@ This document details the beacon chain additions and changes in Phase 1 of Ether 'challenger_index': ValidatorIndex, 'responder_index': ValidatorIndex, 'deadline': Epoch, - 'crosslink_data_root': Hash, + 'crosslink_data_root': 'bytes32', 'chunk_count': 'uint64', - 'chunk_bits_merkle_root': Hash, + 'chunk_bits_merkle_root': 'bytes32', 'responder_key': BLSSignature, } ``` @@ -175,9 +175,9 @@ This document details the beacon chain additions and changes in Phase 1 of Ether 'challenge_index': 'uint64', 'chunk_index': 'uint64', 'chunk': ['byte', BYTES_PER_CUSTODY_CHUNK], - 'data_branch': [Hash], - 'chunk_bits_branch': [Hash], - 'chunk_bits_leaf': Hash, + 'data_branch': ['bytes32'], + 'chunk_bits_branch': ['bytes32'], + 'chunk_bits_leaf': 'bytes32', } ``` @@ -287,7 +287,7 @@ def get_custody_chunk_bit(key: BLSSignature, chunk: bytes) -> bool: ### `get_chunk_bits_root` ```python -def get_chunk_bits_root(chunk_bitfield: Bitfield) -> Bytes32: +def get_chunk_bits_root(chunk_bitfield: bytes) -> Bytes32: aggregated_bits = bytearray([0] * 32) for i in range(0, len(chunk_bitfield), 32): for j in range(32): diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index c277dbdc33..691e38c048 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -15,9 +15,9 @@ - [Signature domains](#signature-domains) - [Data structures](#data-structures) - [`ShardBlockBody`](#shardblockbody) + - [`ShardAttestation`](#shardattestation) - [`ShardBlock`](#shardblock) - [`ShardBlockHeader`](#shardblockheader) - - [`ShardAttestation`](#shardattestation) - [Helper functions](#helper-functions) - [`get_period_committee`](#get_period_committee) - [`get_switchover_epoch`](#get_switchover_epoch) @@ -68,7 +68,23 @@ This document describes the shard data layer and the shard fork choice rule in P ### `ShardBlockBody` ```python -['byte', BYTES_PER_SHARD_BLOCK_BODY] +{ + 'data': ['byte', BYTES_PER_SHARD_BLOCK_BODY], +} +``` + +### `ShardAttestation` + +```python +{ + 'data': { + 'slot': Slot, + 'shard': Shard, + 'shard_block_root': 'bytes32', + }, + 'aggregation_bitfield': 'bytes', + 'aggregate_signature': BLSSignature, +} ``` ### `ShardBlock` @@ -77,10 +93,10 @@ This document describes the shard data layer and the shard fork choice rule in P { 'slot': Slot, 'shard': Shard, - 'beacon_chain_root': Hash, - 'previous_block_root': Hash, + 'beacon_chain_root': 'bytes32', + 'previous_block_root': 'bytes32', 'data': ShardBlockBody, - 'state_root': Hash, + 'state_root': 'bytes32', 'attestations': [ShardAttestation], 'signature': BLSSignature, } @@ -92,29 +108,15 @@ This document describes the shard data layer and the shard fork choice rule in P { 'slot': Slot, 'shard': Shard, - 'beacon_chain_root': Hash, - 'previous_block_root': Hash, - 'body_root': Hash, - 'state_root': Hash, + 'beacon_chain_root': 'bytes32', + 'previous_block_root': 'bytes32', + 'body_root': 'bytes32', + 'state_root': 'bytes32', 'attestations': [ShardAttestation], 'signature': BLSSignature, } ``` -### `ShardAttestation` - -```python -{ - 'data': { - 'slot': Slot, - 'shard': Shard, - 'shard_block_root': Hash, - }, - 'aggregation_bitfield': Bitfield, - 'aggregate_signature': BLSSignature, -} -``` - ## Helper functions ### `get_period_committee` @@ -234,7 +236,7 @@ def verify_shard_attestation_signature(state: BeaconState, ### `compute_crosslink_data_root` ```python -def compute_crosslink_data_root(blocks: List[ShardBlock]) -> Hash: +def compute_crosslink_data_root(blocks: List[ShardBlock]) -> 'bytes32': def is_power_of_two(value: int) -> bool: return (value > 0) and (value & (value - 1) == 0) @@ -272,7 +274,7 @@ Let: def is_valid_shard_block(beacon_blocks: List[BeaconBlock], beacon_state: BeaconState, valid_shard_blocks: List[ShardBlock], - unix_time: uint64, + unix_time: int, candidate: ShardBlock) -> bool: # Check if block is already determined valid for _, block in enumerate(valid_shard_blocks): From ecc6429b9eaef80b13ee8f9f72daa76930f4ad25 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Tue, 7 May 2019 15:01:23 +0100 Subject: [PATCH 063/308] Address Danny's comment --- scripts/phase0/build_spec.py | 3 --- specs/core/0_beacon-chain.md | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index 28e5049ae4..e3c431a4aa 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -18,9 +18,6 @@ def build_phase0_spec(sourcefile, outfile): # stub, will get overwritten by real var SLOTS_PER_EPOCH = 64 -def int_to_bytes(integer: int, length: int) -> bytes: - return integer.to_bytes(length, 'little') - Slot = NewType('Slot', int) # uint64 Epoch = NewType('Epoch', int) # uint64 Shard = NewType('Shard', int) # uint64 diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 1e81700365..a26a4e9793 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -194,7 +194,7 @@ These configurations are updated for releases, but may be out of sync during `de | `GENESIS_SLOT` | `0` | | `GENESIS_EPOCH` | `0` | | `FAR_FUTURE_EPOCH` | `2**64 - 1` | -| `ZERO_HASH` | `int_to_bytes(0, length=32)` | +| `ZERO_HASH` | `b'\x00' * 32` | | `BLS_WITHDRAWAL_PREFIX` | `0` | ### Time parameters From 62c44ffce3068b0e9d9146c3e4ca30deb68583eb Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Tue, 7 May 2019 17:34:19 +0100 Subject: [PATCH 064/308] Refactor to validate_indexed_attestation --- specs/core/0_beacon-chain.md | 54 +++++++++++++++++------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 963c2ac444..27c7fd4619 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -82,7 +82,7 @@ - [`get_bitfield_bit`](#get_bitfield_bit) - [`verify_bitfield`](#verify_bitfield) - [`convert_to_indexed`](#convert_to_indexed) - - [`verify_indexed_attestation`](#verify_indexed_attestation) + - [`validate_indexed_attestation`](#validate_indexed_attestation) - [`is_slashable_attestation_data`](#is_slashable_attestation_data) - [`integer_squareroot`](#integer_squareroot) - [`get_delayed_activation_exit_epoch`](#get_delayed_activation_exit_epoch) @@ -1013,38 +1013,36 @@ def convert_to_indexed(state: BeaconState, attestation: Attestation) -> IndexedA ) ``` -### `verify_indexed_attestation` +### `validate_indexed_attestation` ```python -def verify_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: +def validate_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> None: """ Verify validity of ``indexed_attestation``. """ bit_0_indices = indexed_attestation.custody_bit_0_indices bit_1_indices = indexed_attestation.custody_bit_1_indices - return ( - # Verify no index has custody bit equal to 1 [to be removed in phase 1] - len(bit_1_indices) == 0 and - # Verify max number of indices - len(bit_0_indices) + len(bit_1_indices) <= MAX_INDICES_PER_ATTESTATION and - # Verify index sets are disjoint - len(set(bit_0_indices).intersection(bit_1_indices)) == 0 and - # Verify indices are sorted - bit_0_indices == sorted(bit_0_indices) and bit_1_indices == sorted(bit_1_indices) and - # Verify aggregate signature - bls_verify_multiple( - pubkeys=[ - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in bit_0_indices]), - bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in bit_1_indices]), - ], - message_hashes=[ - hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b0)), - hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), - ], - signature=indexed_attestation.signature, - domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target_epoch), - ) + # Verify no index has custody bit equal to 1 [to be removed in phase 1] + assert len(bit_1_indices) == 0 + # Verify max number of indices + assert len(bit_0_indices) + len(bit_1_indices) <= MAX_INDICES_PER_ATTESTATION + # Verify index sets are disjoint + assert len(set(bit_0_indices).intersection(bit_1_indices)) == 0 + # Verify indices are sorted + assert bit_0_indices == sorted(bit_0_indices) and bit_1_indices == sorted(bit_1_indices) + # Verify aggregate signature + assert bls_verify_multiple( + pubkeys=[ + bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in bit_0_indices]), + bls_aggregate_pubkeys([state.validator_registry[i].pubkey for i in bit_1_indices]), + ], + message_hashes=[ + hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b0)), + hash_tree_root(AttestationDataAndCustodyBit(data=indexed_attestation.data, custody_bit=0b1)), + ], + signature=indexed_attestation.signature, + domain=get_domain(state, DOMAIN_ATTESTATION, indexed_attestation.data.target_epoch), ) ``` @@ -1669,8 +1667,8 @@ def process_attester_slashing(state: BeaconState, attestation_1 = attester_slashing.attestation_1 attestation_2 = attester_slashing.attestation_2 assert is_slashable_attestation_data(attestation_1.data, attestation_2.data) - assert verify_indexed_attestation(state, attestation_1) - assert verify_indexed_attestation(state, attestation_2) + validate_indexed_attestation(state, attestation_1) + validate_indexed_attestation(state, attestation_2) slashed_any = False attesting_indices_1 = attestation_1.custody_bit_0_indices + attestation_1.custody_bit_1_indices @@ -1707,7 +1705,7 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: assert data.crosslink_data_root == ZERO_HASH # [to be removed in phase 1] # Check signature and bitfields - assert verify_indexed_attestation(state, convert_to_indexed(state, attestation)) + validate_indexed_attestation(state, convert_to_indexed(state, attestation)) # Cache pending attestation pending_attestation = PendingAttestation( From 79c193ff60e6941781bc3de57de74265633edb52 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Tue, 7 May 2019 18:07:51 +0100 Subject: [PATCH 065/308] Underscores for python modules --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 420fc32769..1c7f84ec60 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERAT GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS)) PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py -PY_SPEC_PHASE_1_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase1/custody-game.py $(PY_SPEC_DIR)/eth2spec/phase1/shard-data-chains.py +PY_SPEC_PHASE_1_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase1/custody_game.py $(PY_SPEC_DIR)/eth2spec/phase1/shard_data_chains.py PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) @@ -53,10 +53,10 @@ phase1: $(PY_SPEC_PHASE_1_TARGETS) $(PY_SPEC_DIR)/eth2spec/phase0/spec.py: python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@ -$(PY_SPEC_DIR)/eth2spec/phase1/custody-game.py: +$(PY_SPEC_DIR)/eth2spec/phase1/custody_game.py: python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_custody-game.md $@ -$(PY_SPEC_DIR)/eth2spec/phase1/shard-data-chains.py: +$(PY_SPEC_DIR)/eth2spec/phase1/shard_data_chains.py: python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_shard-data-chains.md $@ From bff71b6e90dc87a1e2e85b631ec5e3593a922722 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 7 May 2019 11:08:14 -0600 Subject: [PATCH 066/308] change some language to be more declarative rather than about the future --- specs/networking/libp2p-standardization.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index 67bf74d1df..0c65734a5b 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -77,7 +77,7 @@ optionally [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md) study* The [Gossipsub](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) -protocol will be used for block and attestation propagation across the +protocol is used for block and attestation propagation across the network. ### Configuration Parameters @@ -111,7 +111,7 @@ propagation times and message duplication. Current network-related defaults are: *The Go and Js implementations use string topics - This is likely to be updated to topic hashes in later versions - https://github.com/libp2p/rust-libp2p/issues/473* -For Eth2.0 clients, topics will be sent as `SHA2-256` hashes of the topic string. +For Eth2.0 clients, topics are sent as `SHA2-256` hashes of the topic string. There are two main topics used to propagate attestations and beacon blocks to all nodes on the network. @@ -120,12 +120,12 @@ all nodes on the network. beacon blocks to all nodes on the networks. - The `beacon_attestation` topic - This topic is used to propagate aggregated attestations to subscribing nodes (typically block proposers) to - be included into future blocks. Attestations will be aggregated in their + be included into future blocks. Attestations are aggregated in their respective subnets before publishing on this topic. -Shards will be grouped into their own subnets (defined by a shard topic). The -number of shard subnets will be defined via `SHARD_SUBNET_COUNT` and the shard -`shard_number % SHARD_SUBNET_COUNT` will be assigned to the topic: +Shards are grouped into their own subnets (defined by a shard topic). The +number of shard subnets is defined via `SHARD_SUBNET_COUNT` and the shard +`shard_number % SHARD_SUBNET_COUNT` is assigned to the topic: `shard{shard_number % SHARD_SUBNET_COUNT}`. ### Messages @@ -138,7 +138,7 @@ has a maximum size of 512KB (estimated from expected largest uncompressed block size). The `data` field of a Gossipsub `Message` is an SSZ-encoded object. For the `beacon_block` topic, -this will be a `beacon_block`. For the `beacon_attestation` topic, this will be +this is a `beacon_block`. For the `beacon_attestation` topic, this is an `attestation`. ## Eth-2 RPC From 13d2ee696939e192534c0278eb271fe53510caf7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 7 May 2019 11:49:45 -0600 Subject: [PATCH 067/308] cleanup validator guide for crosslinks --- specs/validator/0_beacon-chain-validator.md | 53 +++++++-------------- 1 file changed, 17 insertions(+), 36 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index a0173ebe92..a95052b860 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -37,14 +37,9 @@ - [Voluntary exits](#voluntary-exits) - [Attestations](#attestations-1) - [Attestation data](#attestation-data) - - [Slot](#slot-1) - - [Beacon block root](#beacon-block-root) - - [Source epoch](#source-epoch) - - [Source root](#source-root) - - [Target root](#target-root) - - [Shard](#shard) - - [Previous crosslink root](#previous-crosslink-root) - - [Crosslink data root](#crosslink-data-root) + - [LMD GHOST vote](#lmd-ghost-vote) + - [FFG vote](#ffg-vote) + - [Crosslink vote](#crosslink-vote) - [Construct attestation](#construct-attestation) - [Data](#data) - [Aggregation bitfield](#aggregation-bitfield) @@ -245,43 +240,29 @@ First the validator should construct `attestation_data`, an [`AttestationData`]( * Let `head_block` be the result of running the fork choice during the assigned slot. * Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot. -##### Slot - -Set `attestation_data.slot = head_state.slot`. - -##### Beacon block root +##### LMD GHOST vote Set `attestation_data.beacon_block_root = signing_root(head_block)`. -##### Source epoch - -Set `attestation_data.source_epoch = head_state.justified_epoch`. - -##### Source root - -Set `attestation_data.source_root = head_state.current_justified_root`. +##### FFG vote -##### Target root +* Set `attestation_data.source_epoch = head_state.justified_epoch`. +* Set `attestation_data.source_root = head_state.current_justified_root`. +* Set `attestation_data.target_epoch = get_current_epoch(head_state)` +* Set `attestation_data.target_root = signing_root(epoch_boundary_block)` where `epoch_boundary_block` is the block at the most recent epoch boundary. -Set `attestation_data.target_root = signing_root(epoch_boundary)` where `epoch_boundary` is the block at the most recent epoch boundary. - -*Note*: This can be looked up in the state using: +*Note*: `epoch_boundary_block` can be looked up in the state using: * Let `epoch_start_slot = get_epoch_start_slot(get_current_epoch(head_state))`. -* Set `epoch_boundary = head if epoch_start_slot == head_state.slot else get_block_root(state, epoch_start_slot)`. - -##### Shard - -Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associated with the validator's committee defined by `get_crosslink_committees_at_slot`. +* Let `epoch_boundary_block = head if epoch_start_slot == head_state.slot else get_block_root(state, epoch_start_slot)`. -##### Previous crosslink root +##### Crosslink vote -Set `attestation_data.parent_root = hash_tree_root(head_state.current_crosslinks[shard])`. +Construct `attestation_data.crosslink` via the following -##### Crosslink data root - -Set `attestation_data.crosslink.data_root = ZERO_HASH`. - -*Note*: This is a stub for Phase 0. +* Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associated with the validator's committee defined by `get_crosslink_committees_at_slot`. +* Set `attestation_data.crosslink.epoch = min(attestation_data.target_epoch, head_state.current_crosslinks[shard].epoch + MAX_EPOCHS_PER_CROSSLINK)`. +* Set `attestation_data.crosslink.parent_root = hash_tree_root(head_state.current_crosslinks[shard])`. +* Set `attestation_data.crosslink.data_root = ZERO_HASH`. *Note*: This is a stub for Phase 0. #### Construct attestation From 3c87754deee2bd21521579c7abb90d37484c0ebc Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 8 May 2019 10:28:08 +1000 Subject: [PATCH 068/308] Rename shard topics to explicitly state --- specs/networking/libp2p-standardization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index 0c65734a5b..1f728fa544 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -126,7 +126,7 @@ all nodes on the network. Shards are grouped into their own subnets (defined by a shard topic). The number of shard subnets is defined via `SHARD_SUBNET_COUNT` and the shard `shard_number % SHARD_SUBNET_COUNT` is assigned to the topic: -`shard{shard_number % SHARD_SUBNET_COUNT}`. +`shard{shard_number % SHARD_SUBNET_COUNT}_attestation`. ### Messages From 0d06f6bcc1bb16dc77d85067e480ce3623b94934 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 8 May 2019 15:04:27 +1000 Subject: [PATCH 069/308] Added the first draft of the BN-VC API RFC, as it was listed on the issue #1011. --- .../0_beacon-node-validator-client-api.md | 235 ++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 specs/validator/0_beacon-node-validator-client-api.md diff --git a/specs/validator/0_beacon-node-validator-client-api.md b/specs/validator/0_beacon-node-validator-client-api.md new file mode 100644 index 0000000000..fff4107caa --- /dev/null +++ b/specs/validator/0_beacon-node-validator-client-api.md @@ -0,0 +1,235 @@ +# ETH2.0 Beacon Node & Validator Client RPC Proposal + +## Outline + +This issue proposes a minimal communications specification between a beacon node and validator client, targeting _phase 0_ of the Eth2.0 specification. The protocol is designed to be a simple local communications interface, permitting two separate binaries to communicate. + +This specification is intended to describe communication abstractly, without choosing any particular protocol. The protocol used (e.g. gRPC/JSON-RPC) is discussed in a separate issue: #1012. + +This issue follows on from a discussion during the [Client Architecture](https://notes.ethereum.org/iuPB2YjKQMGua0nwCZJVAQ) session at the [Sydney Implementers meeting](https://notes.ethereum.org/7w7diW1-St2_Yu-YHjg6NA). This also follow from the [Client Architecture Roundtable](https://hackmd.io/s/rJl05Lk6X) in Prague. +There is an editable version of this document, here: https://hackmd.io/r7D61hs4RWKm8nz_O2iinQ + +### Background +The beacon node maintains the state of the beacon chain by communicating with other beacon nodes in the Ethereum Serenity network. Conceptually, it does not maintain keypairs that participate with the beacon chain. + +The validator client is conceptually a separate entity which utilises private keys to perform validator related tasks on the beacon chain, which we call validator "duties". This includes the production of beacon blocks and signing of attestations. + +Since it is recommended to separate these concerns in the client implementations, it is necessary for us to clearly define the communication between them. + +The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects. For example, the validator client from Lighthouse, could communicate with a running instance of the beacon node from Prysm. + +This proposal has been adapted from the [Lighthouse gRPC protocol specification](https://github.com/sigp/lighthouse/blob/master/protos/src/services.proto). + +There is also another [WIP proposal for a Minimum Validator Interface](https://notes.ethereum.org/Ia2kvjy0RX2J-GxrWfoCAQ), which describes additional functions possibly necessary for phase 1 and beyond. + +## Specification + +### Entities +The following are the two entities that participate in this protocol: + - **`BeaconNode`**: + A beacon node instance, run with a `--rpc` flag to enable the RPC interface. Runs stand-alone. + + - **`ValidatorClient`**: +A validator client instance, which must connect to at least one instance of `BeaconNode`. + + + +### Endpoints +This section summarises API endpoints which are published by an instance of `BeaconNode`, for the exclusive use of `ValidatorClient` implementations. + +This proposal is a minimum set of messages necessary to enable effective communication, without any extra features. Anything extra is beyond the scope of this document. + +#### Summary Table +| Name | Type | Parameters | Returns | +| -------- | --- | ----- | ----- | +| [`get_client_version`](#get_client_version) | GET | N/A | `client_version` | +| [`get_genesis_time`](#get_genesis_time) | GET | N/A | `genesis_time` | +| [`get_syncing_status`](#get_syncing_status) | GET | N/A | `syncing_status` | +| [`get_duties`](#get_duties) | GET | `validator_pubkeys` | `syncing_status`, `current_version`, [`ValidatorDuty`]| +| [`produce_block`](#produce_block) | GET | `slot`, `randao_reveal` | `beacon_block` | +| [`publish_block`](#publish_block) | POST | `beacon_block` | N/A | +| [`produce_attestation`](#produce_attestation) | GET | `slot`, `shard` | `indexed_attestation` | +| [`publish_attestation`](#publish_attestation) | POST | `indexed_attestation` | N/A | Publishes the IndexedAttestation after having been signed by the ValidatorClient | + +#### Status Codes +For each of these endpoints the underlying transport protocol should provide status codes. Assuming this will be based on HTTP, one of the following standard status codes will always be included as part of a response: + +| Code | Meaning | +| --- | --- | +| `200` | The API call succeeded. | +| `40X` | The request was malformed. | +| `500` | The `BeaconNode` cannot complete the request due to an internal error. | +| `503` | The `BeaconNode` is currently syncing, try again later. _A call can be made to `get_syncing_status` to in order to find out how much has been synchronised._ | + +#### `get_client_version` +Requests that the `BeaconNode` identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field. + + - **Parameters**: N/A + - **Returns**: + + | Name | Type | Description | + | --- | --- | --- | + | `client_version` | bytes32 | An ASCII-encoded hex string which uniquely defines the implementation of the `BeaconNode` and its current software version. | + + **Note**: _Unlike most other endpoints, `get_client_version` does not return an error `503` while the `BeaconNode` is syncing, but instead returns status code `200`._ + + +#### `get_genesis_time` + Requests the `genesis_time` parameter from the `BeaconNode`, which should be consistent across all `BeaconNodes` that follow the same beacon chain. + + - **Parameters**: N/A + - **Returns**: + + | Name | Type | Description | + | --- | --- | --- | + | `genesis_time` | uint64 | The [`genesis_time`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#on-genesis), which is a fairly static configuration option for the `BeaconNode`. | + + **Note**: _Unlike most other endpoints, `get_genesis_time` does not return an error `503` while the `BeaconNode` is syncing, but instead returns status code `200`._ + + +#### `get_syncing_status` + Requests the `BeaconNode` to describe if it's currently syncing or not, and if it is, what block it is up to. This is modelled after the Eth1.0 JSON-RPC [`eth_syncing`](https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_syncing) call. + - **Parameters**: N/A + - **Returns**: + + | Name | Type | Description | + | --- | --- | --- | + | `syncing` | `false` OR `SyncingStatus` | Either `false` if the node is not syncing, or a [`SyncingStatus`](#SyncingStatus) object if it is. | + + **Note**: _Unlike most other endpoints, `get_syncing_status` does not return an error `503` while the `BeaconNode` is syncing, but instead returns status code `200` with the `SyncingStatus` object._ + + +#### `get_duties` + Requests the BeaconNode to provide a set of “duties”, which are actions that should be performed by ValidatorClients. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected `BeaconNode` is properly synchronised. + + - **Parameters**: + + | Name | Type | Description | + | --- | --- | --- | + | `validator_pubkeys` | [bytes48] | A list of unique validator public keys, where each item is a `0x` encoded hex string. | + + - **Returns**: + + | Name | Type | Description | + | --- | --- | --- | + | `current_version` | bytes4 | The `current_version`, as described by the current [`Fork`](#Fork). | + | `validator_duties` | [`ValidatorDuty`] | A list where each item is a custom [`ValidatorDuty`](#ValidatorDuty) object. | + + + #### `produce_block` + Requests a `BeaconNode` to produce a valid block, which can then be signed by a ValidatorClient. + + - **Parameters**: + + | Name | Type | Description | + | --- | --- | --- | + | `slot` | uint64 | The slot for which the block should be proposed. | + | `randao_reveal` | bytes | The ValidatorClient's randao reveal value. | + + - **Returns**: + + | Name | Type | Description | + | --- | --- | --- | + | `beacon_block` | `BeaconBlock` | A proposed [`BeaconBlock`](#BeaconBlock) object, but with the `signature` field left blank. + + + #### `publish_block` + Instructs the `BeaconNode` to publish a newly signed beacon block to the beacon network, to be included in the beacon chain. + - **Parameters**: + + | Name | Type | Description | + | --- | --- | --- | + | `beacon_block` | `BeaconBlock` | The [`BeaconBlock`](#BeaconBlock) object, as sent from the `BeaconNode` originally, but now with the `signature` field completed. + + - **Returns**: N/A + + + #### `produce_attestation` + Requests that the `BeaconNode` produce an `IndexedAttestation`, with a blank `signature` field, which the `ValidatorClient` will then sign. + + - **Parameters**: + + | Name | Type | Description | + | --- | --- | --- | + | `slot` | uint64 | The slot for which the attestation should be proposed. | + | `shard` | uint64 | The shard number for which the attestation is to be proposed. | + + - **Returns**: + + | Name | Type | Description | + | --- | --- | --- | + | `indexed_attestation` | `IndexedAttestation` | An [`IndexedAttestation`](#IndexedAttestation) structure with the `signature` field left blank. | + + #### `publish_attestation` + Instructs the `BeaconNode` to publish a newly signed `IndexedAttestation` object, to be incorporated into the beacon chain. + + - **Parameters**: + + | Name | Type | Description | + | --- | --- | --- | + | `indexed_attestation` | `IndexedAttestation` | An [`IndexedAttestation`](#IndexedAttestation) structure, as originally provided by the `BeaconNode`, but now with the `signature` field completed. | + - **Returns**: N/A + + + + ----- + +### Data Structures +Two new data objects are proposed for the sake of implementation, and several other data objects from the Eth2.0 specs are referenced. + +The `bytes` data types are encoded hex strings, with `0x` preceeding them. `uint64` are decimal encoded integers, and `None` may be `null`, which is distinct from `0`. + +#### `ValidatorDuty` +```asm +{ + + # The validator's public key, uniquely identifying them + 'validator_pubkey': 'bytes48', + # The index of the validator in the committee + 'committee_index': 'uint64', + # The slot at which the validator must attest. + 'attestation_slot': 'uint64', + # The shard in which the validator must attest + 'attestation_shard': 'uint64', + # The slot in which a validator must propose a block. This field can also be None. + 'block_production_slot': 'uint64' or None +} +``` + +#### `SyncingStatus` +As described by the [Eth1.0 JSON-RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_syncing).: +```asm +{ + # The block at which syncing started (will only be reset, after the sync reached his head) + 'startingBlock': 'uint64', + # The current block + 'currentBlock': 'uint64', + # The estimated highest block, or current target block number + 'highestBlock': 'uint64' +} +``` + +#### `Fork` +As described by [Fork](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#Fork) in the Eth2.0 specs. + +#### `BeaconBlock` +As described by [BeaconBlock](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#BeaconBlock) in the Eth2.0 specs. + +#### `IndexedAttestation` +As described by [IndexedAttestation](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#IndexedAttestation) in the Eth2.0 specs. + + + + +## Optional Extras + +#### Endpoint: `get_fork` + Requests the `BeaconNode` to provide which fork version it is currently on. + - **Parameters**: N/A + - **Returns**: + + | Name | Type | Description | + | --- | --- | --- | + | `fork` | [`Fork`](#Fork) | Provides the current version information for the fork which the `BeaconNode` is currently following. | + | `chain_id` | uint64 | Sometimes called the network id, this number discerns the active chain for the `BeaconNode`. Analagous to Eth1.0 JSON-RPC [`net_version`](https://github.com/ethereum/wiki/wiki/JSON-RPC#net_version). | + From 4d2e752bb90437afb73ce58253cb68122fac16e6 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Wed, 8 May 2019 23:51:53 +1000 Subject: [PATCH 070/308] Started updating the markdown description of the BNVC REST API, removing stuff specific to the issue and conforming to standard text layout. --- ...-api.md => 0_beacon-node-validator-api.md} | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) rename specs/validator/{0_beacon-node-validator-client-api.md => 0_beacon-node-validator-api.md} (87%) diff --git a/specs/validator/0_beacon-node-validator-client-api.md b/specs/validator/0_beacon-node-validator-api.md similarity index 87% rename from specs/validator/0_beacon-node-validator-client-api.md rename to specs/validator/0_beacon-node-validator-api.md index fff4107caa..342e7f934a 100644 --- a/specs/validator/0_beacon-node-validator-client-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -1,26 +1,30 @@ -# ETH2.0 Beacon Node & Validator Client RPC Proposal +# Ethereum 2.0 Phase 0 -- Beacon Node API for Validator -## Outline +__NOTICE__: This document is a work-in-progress for researchers and implementers. This is an accompanying document to [Ethereum 2.0 Phase 0 -- Honest Validator](0_beacon-chain-validator.md) that describes an API exposed by the beacon node, which enables the validator client to participate in the Ethereum 2.0 protocol. + +## Table of Contents + + -This issue proposes a minimal communications specification between a beacon node and validator client, targeting _phase 0_ of the Eth2.0 specification. The protocol is designed to be a simple local communications interface, permitting two separate binaries to communicate. + -This specification is intended to describe communication abstractly, without choosing any particular protocol. The protocol used (e.g. gRPC/JSON-RPC) is discussed in a separate issue: #1012. +## Outline + +This document outlines a minimal application programming interface (API) which is exposed by a `BeaconNode` for use by a `ValidatorClient` which aims to facilitate [_phase 0_](../../README.md#phase-0) of Ethereum 2.0. -This issue follows on from a discussion during the [Client Architecture](https://notes.ethereum.org/iuPB2YjKQMGua0nwCZJVAQ) session at the [Sydney Implementers meeting](https://notes.ethereum.org/7w7diW1-St2_Yu-YHjg6NA). This also follow from the [Client Architecture Roundtable](https://hackmd.io/s/rJl05Lk6X) in Prague. -There is an editable version of this document, here: https://hackmd.io/r7D61hs4RWKm8nz_O2iinQ +The API is a REST interface, accessed via HTTP, designed for use as a local communications protocol between binaries. ### Background The beacon node maintains the state of the beacon chain by communicating with other beacon nodes in the Ethereum Serenity network. Conceptually, it does not maintain keypairs that participate with the beacon chain. -The validator client is conceptually a separate entity which utilises private keys to perform validator related tasks on the beacon chain, which we call validator "duties". This includes the production of beacon blocks and signing of attestations. +The validator client is a conceptually separate entity which utilises private keys to perform validator related tasks on the beacon chain, which we call validator "duties". These duties includes the production of beacon blocks and signing of attestations. -Since it is recommended to separate these concerns in the client implementations, it is necessary for us to clearly define the communication between them. +Since it is recommended to separate these concerns in the client implementations, we must clearly define the communication between them. The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects. For example, the validator client from Lighthouse, could communicate with a running instance of the beacon node from Prysm. -This proposal has been adapted from the [Lighthouse gRPC protocol specification](https://github.com/sigp/lighthouse/blob/master/protos/src/services.proto). +This specification is derived from a proposal and discussion on Issues [#1011](https://github.com/ethereum/eth2.0-specs/issues/1011) and [#1012](https://github.com/ethereum/eth2.0-specs/issues/1012) -There is also another [WIP proposal for a Minimum Validator Interface](https://notes.ethereum.org/Ia2kvjy0RX2J-GxrWfoCAQ), which describes additional functions possibly necessary for phase 1 and beyond. ## Specification @@ -33,7 +37,6 @@ The following are the two entities that participate in this protocol: A validator client instance, which must connect to at least one instance of `BeaconNode`. - ### Endpoints This section summarises API endpoints which are published by an instance of `BeaconNode`, for the exclusive use of `ValidatorClient` implementations. From 513c44bd3d19c70da61359376dc440fc86bddc23 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 8 May 2019 08:38:14 -0600 Subject: [PATCH 071/308] add back in empty attestation test --- .../block_processing/test_process_attestation.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index 165f0c84a9..c986cc4c87 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -142,3 +142,14 @@ def test_non_empty_custody_bitfield(state): pre_state, post_state = run_attestation_processing(state, attestation, False) return pre_state, attestation, post_state + + +def test_empty_aggregation_bitfield(state): + attestation = get_valid_attestation(state) + state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY + + attestation.aggregation_bitfield = b'\x00' * len(attestation.aggregation_bitfield) + + pre_state, post_state = run_attestation_processing(state, attestation) + + return pre_state, attestation, post_state From c37157ead1a97fd3fb2fccdda4f0b67ea258a1ea Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 8 May 2019 19:15:23 +0100 Subject: [PATCH 072/308] Revert exception handling --- specs/core/0_beacon-chain.md | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 3470459e10..bf210c60ce 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1216,19 +1216,15 @@ Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. ## Beacon chain state transition function -The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. +The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled excpetion (e.g. a failed `assert` or an out-of-range list access) are considered invalid. ```python def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: - pre_state = copy.deepcopy(state) - try: - # Process slots (including those with no blocks) since block - process_slots(state, block.slot) - # Process block - process_block(state, block) - except Exception: - # State is not advanced on exceptions (e.g. a failed `assert` or an out-of-range list access) - return pre_state + # Process slots (including those with no blocks) since block + process_slots(state, block.slot) + # Process block + process_block(state, block) + # Return post-state return state ``` From 8da4b8173e38bec3f46820beab731e173384c689 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 8 May 2019 14:49:53 -0600 Subject: [PATCH 073/308] remove unnecessary import of copy --- scripts/phase0/build_spec.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index ef7c055d21..f7587bad5a 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -6,7 +6,6 @@ def build_phase0_spec(sourcefile, outfile): code_lines = [] code_lines.append(""" -import copy from typing import ( Any, Dict, From 6ab55efd3d31fa0b5711a2f6843a70346a3e7704 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Wed, 8 May 2019 23:30:08 +0100 Subject: [PATCH 074/308] Phase 0 tests work on phase 1, yay :) --- Makefile | 10 ++- scripts/phase0/build_spec.py | 18 +++++ scripts/phase0/function_puller.py | 7 +- specs/core/1_custody-game.md | 5 +- test_libs/pyspec/eth2spec/phase1/__init__.py | 0 .../eth2spec/phase1/state_transition.py | 77 +++++++++++++++++++ .../pyspec/eth2spec/utils/minimal_ssz.py | 24 ++++-- .../test_process_attestation.py | 6 +- .../test_process_attester_slashing.py | 4 +- .../test_process_block_header.py | 2 +- .../block_processing/test_process_deposit.py | 4 +- .../test_process_proposer_slashing.py | 4 +- .../block_processing/test_process_transfer.py | 4 +- .../block_processing/test_voluntary_exit.py | 4 +- test_libs/pyspec/tests/conftest.py | 3 +- .../test_process_crosslinks.py | 6 +- .../test_process_registry_updates.py | 4 +- test_libs/pyspec/tests/helpers.py | 9 ++- test_libs/pyspec/tests/test_finality.py | 4 +- test_libs/pyspec/tests/test_sanity.py | 8 +- 20 files changed, 159 insertions(+), 44 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/phase1/__init__.py create mode 100644 test_libs/pyspec/eth2spec/phase1/state_transition.py diff --git a/Makefile b/Makefile index 1c7f84ec60..65c8bc33dc 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,11 @@ YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERAT GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS)) PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py +PY_SPEC_PHASE_0_DEPS = $(SPEC_DIR)/core/0_*.md + PY_SPEC_PHASE_1_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase1/custody_game.py $(PY_SPEC_DIR)/eth2spec/phase1/shard_data_chains.py +PY_SPEC_PHASE_1_DEPS = $(SPEC_DIR)/core/1_*.md + PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) @@ -50,13 +54,13 @@ phase0: $(PY_SPEC_PHASE_0_TARGETS) # "make phase1" to create pyspec for phase1 phase1: $(PY_SPEC_PHASE_1_TARGETS) -$(PY_SPEC_DIR)/eth2spec/phase0/spec.py: +$(PY_SPEC_DIR)/eth2spec/phase0/spec.py: $(PY_SPEC_PHASE_0_DEPS) python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@ -$(PY_SPEC_DIR)/eth2spec/phase1/custody_game.py: +$(PY_SPEC_DIR)/eth2spec/phase1/custody_game.py: $(PY_SPEC_PHASE_1_DEPS) python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_custody-game.md $@ -$(PY_SPEC_DIR)/eth2spec/phase1/shard_data_chains.py: +$(PY_SPEC_DIR)/eth2spec/phase1/shard_data_chains.py: $(PY_SPEC_PHASE_0_DEPS) python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_shard-data-chains.md $@ diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index 80b378fbb9..e3009004f7 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -89,11 +89,29 @@ def build_phase1_spec(sourcefile, outfile): code_lines = [] code_lines.append(""" from eth2spec.phase0.spec import * +from eth2spec.phase0.spec import apply_constants_preset as apply_constants_preset_phase0 """) code_lines += function_puller.get_spec(sourcefile) + code_lines.append(""" +# Access to overwrite spec constants based on configuration +def apply_constants_preset(preset: Dict[str, Any]): + + apply_constants_preset_phase0(preset) + + global_vars = globals() + for k, v in preset.items(): + global_vars[k] = v + + # Deal with derived constants + global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT) + + # Initialize SSZ types again, to account for changed lengths + init_SSZ_types() +""") + with open(outfile, 'w') as out: out.write("\n".join(code_lines)) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 830e52a32d..26ac2f17f8 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -39,10 +39,11 @@ def get_spec(file_name: str, phase:int = 0) -> List[str]: else: if pulling_from == linenum and line == '{': if is_update_section: - code_lines.append('%s = SSZTypeExtension(%s, {' % (current_name, current_name)) + code_lines.append('%s = SSZTypeExtension("%s", {' % (current_name, current_name)) + current_typedef = ['global_vars["%s"] = SSZTypeExtension("%s", {' % (current_name, current_name)] else: - code_lines.append('%s = SSZType({' % current_name) - current_typedef = ['global_vars["%s"] = SSZType({' % current_name] + code_lines.append('%s = SSZType("%s", {' % (current_name, current_name)) + current_typedef = ['global_vars["%s"] = SSZType("%s", {' % (current_name, current_name)] elif pulling_from is not None: # Add some whitespace between functions if line[:3] == 'def': diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 5de52a4ea3..501bfce466 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -661,12 +661,12 @@ def process_bit_challenge_response(state: BeaconState, ### Handling of custody-related deadlines -Run `process_reveal_deadlines(state)` immediately after `process_ejections(state)`: +Run `process_reveal_deadlines(state)` immediately after `process_registry_updates(state)`: ```python def process_reveal_deadlines(state: BeaconState) -> None: for index, validator in enumerate(state.validator_registry): - if (validator.latest_custody_reveal_period + + if (validator.next_custody_reveal_period + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) < get_validators_custody_reveal_period(state, index)): slash_validator(state, index) @@ -693,6 +693,7 @@ Append this to `process_final_updates(state)`: ```python def after_process_final_updates(state: BeaconState) -> None: + current_epoch = get_current_epoch(state) # Clean up exposed RANDAO key reveals state.exposed_derived_secrets[current_epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] ``` diff --git a/test_libs/pyspec/eth2spec/phase1/__init__.py b/test_libs/pyspec/eth2spec/phase1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_libs/pyspec/eth2spec/phase1/state_transition.py b/test_libs/pyspec/eth2spec/phase1/state_transition.py new file mode 100644 index 0000000000..7e6411287d --- /dev/null +++ b/test_libs/pyspec/eth2spec/phase1/state_transition.py @@ -0,0 +1,77 @@ +from . import spec + +from typing import ( + Any, + Callable, + List +) + +from .spec import ( + BeaconState, + BeaconBlock, + Slot, +) + +from eth2spec.phase0.state_transition import ( + expected_deposit_count, + process_operation_type, + process_operations as process_operations_phase0, + process_block as process_block_phase0, + process_epoch_transition as process_epoch_transition_phase0, + state_transition_to as state_transition_to_phase0, + state_transition as state_transition_phase0 +) + +def process_operations(state: BeaconState, block: BeaconBlock) -> None: + process_operations_phase0(state, block) + + process_operation_type( + state, + block.body.custody_key_reveals, + spec.MAX_CUSTODY_KEY_REVEALS, + spec.process_custody_key_reveal, + ) + + process_operation_type( + state, + block.body.early_derived_secret_reveals, + spec.MAX_EARLY_DERIVED_SECRET_REVEALS, + spec.process_early_derived_secret_reveal, + ) + +def process_block(state: BeaconState, + block: BeaconBlock, + verify_state_root: bool=False) -> None: + spec.process_block_header(state, block) + spec.process_randao(state, block) + spec.process_eth1_data(state, block) + + process_operations(state, block) + if verify_state_root: + spec.verify_block_state_root(state, block) + +def process_epoch_transition(state: BeaconState) -> None: + spec.process_justification_and_finalization(state) + spec.process_crosslinks(state) + # TODO: Eligible + spec.process_rewards_and_penalties(state) + spec.process_registry_updates(state) + spec.process_reveal_deadlines(state) + spec.process_challenge_deadlines(state) + spec.process_slashings(state) + spec.process_final_updates(state) + spec.after_process_final_updates(state) + +def state_transition_to(state: BeaconState, up_to: Slot) -> BeaconState: + while state.slot < up_to: + spec.cache_state(state) + if (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0: + process_epoch_transition(state) + spec.advance_slot(state) + + +def state_transition(state: BeaconState, + block: BeaconBlock, + verify_state_root: bool=False) -> BeaconState: + state_transition_to(state, block.slot) + process_block(state, block, verify_state_root) \ No newline at end of file diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py index cfd2e56ac4..8a210f41ba 100644 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py @@ -6,9 +6,14 @@ BYTES_PER_LENGTH_OFFSET = 4 ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK +cached_typedefs = {} -def SSZType(fields): +def SSZType(name, fields): class SSZObject(): + if name != None: + __name__ = name + __qualname__ = name + def __init__(self, **kwargs): for f, t in fields.items(): if f not in kwargs: @@ -25,8 +30,11 @@ def __hash__(self): def __str__(self): output = [] for field in self.fields: - output.append(f'{field}: {getattr(self, field)}') + output.append(f'{field}: {repr(getattr(self, field))},') return "\n".join(output) + + def __repr__(self): + return name + "(**{\n " + str(self).replace("\n", "\n ") + "\n})" def serialize(self): return serialize_value(self, self.__class__) @@ -35,12 +43,16 @@ def hash_tree_root(self): return hash_tree_root(self, self.__class__) SSZObject.fields = fields + + if name != None: + cached_typedefs[name] = SSZObject + return SSZObject def SSZTypeExtension(original_type, new_fields): - fields = original_type.fields.copy() - fields.update(original_type.fields) - return SSZType(fields) + typedef = cached_typedefs[original_type] + typedef.fields.update(new_fields) + return typedef class Vector(): def __init__(self, items): @@ -319,7 +331,7 @@ def truncate(container): key: container.fields[key] for key in field_keys[:-1] } - truncated_class = SSZType(truncated_fields) + truncated_class = SSZType(None, truncated_fields) kwargs = { field: getattr(container, field) for field in field_keys[:-1] diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index bcf71376ce..18706c9e57 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -1,12 +1,12 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec -from eth2spec.phase0.state_transition import ( +from eth2spec.phase1.state_transition import ( state_transition, ) -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( get_current_epoch, process_attestation, slot_to_epoch, diff --git a/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py index 2ea16f13d9..880ca829a7 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py @@ -1,8 +1,8 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec -from eth2spec.phase0.spec import ( +import eth2spec.phase1.spec as spec +from eth2spec.phase1.spec import ( get_beacon_proposer_index, process_attester_slashing, ) diff --git a/test_libs/pyspec/tests/block_processing/test_process_block_header.py b/test_libs/pyspec/tests/block_processing/test_process_block_header.py index b35b0a9c11..bcaf3c6d80 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/tests/block_processing/test_process_block_header.py @@ -2,7 +2,7 @@ import pytest -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( get_beacon_proposer_index, cache_state, advance_slot, diff --git a/test_libs/pyspec/tests/block_processing/test_process_deposit.py b/test_libs/pyspec/tests/block_processing/test_process_deposit.py index bbfb390efb..768f616786 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/tests/block_processing/test_process_deposit.py @@ -1,9 +1,9 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( ZERO_HASH, process_deposit, ) diff --git a/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py index 4752210366..513e607b1e 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py @@ -1,8 +1,8 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec -from eth2spec.phase0.spec import ( +import eth2spec.phase1.spec as spec +from eth2spec.phase1.spec import ( get_current_epoch, process_proposer_slashing, ) diff --git a/test_libs/pyspec/tests/block_processing/test_process_transfer.py b/test_libs/pyspec/tests/block_processing/test_process_transfer.py index 0eeaa77929..02f448fd02 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/tests/block_processing/test_process_transfer.py @@ -1,9 +1,9 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( get_active_validator_indices, get_beacon_proposer_index, get_current_epoch, diff --git a/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py b/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py index c58c5238a9..c789b55d83 100644 --- a/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py +++ b/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py @@ -1,9 +1,9 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( get_active_validator_indices, get_churn_limit, get_current_epoch, diff --git a/test_libs/pyspec/tests/conftest.py b/test_libs/pyspec/tests/conftest.py index 9840dc7b20..98b64c73bf 100644 --- a/test_libs/pyspec/tests/conftest.py +++ b/test_libs/pyspec/tests/conftest.py @@ -1,6 +1,6 @@ import pytest -from eth2spec.phase0 import spec +from eth2spec.phase1 import spec from preset_loader import loader from .helpers import ( @@ -20,7 +20,6 @@ def config(request): presets = loader.load_presets('../../configs/', config_name) spec.apply_constants_preset(presets) - @pytest.fixture def num_validators(config): return spec.SLOTS_PER_EPOCH * 8 diff --git a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py index d6765e3a72..9a6254715f 100644 --- a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py @@ -1,12 +1,12 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec -from eth2spec.phase0.state_transition import ( +from eth2spec.phase1.state_transition import ( state_transition, ) -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( cache_state, get_crosslink_deltas, process_crosslinks, diff --git a/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py index 11f5de2ad4..15f7ae6ecc 100644 --- a/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py @@ -2,9 +2,9 @@ import pytest -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( get_current_epoch, is_active_validator, ) diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 3b9b6904d5..3156a7f28c 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -2,12 +2,12 @@ from py_ecc import bls -from eth2spec.phase0.state_transition import ( +from eth2spec.phase1.state_transition import ( state_transition, ) -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec from eth2spec.utils.minimal_ssz import signing_root -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( # constants ZERO_HASH, # SSZ @@ -420,3 +420,6 @@ def get_state_root(state, slot) -> bytes: """ assert slot < state.slot <= slot + spec.SLOTS_PER_HISTORICAL_ROOT return state.latest_state_roots[slot % spec.SLOTS_PER_HISTORICAL_ROOT] + +# Stub to be overwritten by config +import_spec = None \ No newline at end of file diff --git a/test_libs/pyspec/tests/test_finality.py b/test_libs/pyspec/tests/test_finality.py index ca048c2b2a..46f08cfbb8 100644 --- a/test_libs/pyspec/tests/test_finality.py +++ b/test_libs/pyspec/tests/test_finality.py @@ -2,9 +2,9 @@ import pytest -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec -from eth2spec.phase0.state_transition import ( +from eth2spec.phase1.state_transition import ( state_transition, ) from .helpers import ( diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 1b4d20f4c5..ea488fd1f6 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -3,10 +3,10 @@ import pytest from py_ecc import bls -import eth2spec.phase0.spec as spec +import eth2spec.phase1.spec as spec from eth2spec.utils.minimal_ssz import signing_root -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( # constants ZERO_HASH, SLOTS_PER_HISTORICAL_ROOT, @@ -25,7 +25,7 @@ verify_merkle_branch, hash, ) -from eth2spec.phase0.state_transition import ( +from eth2spec.phase1.state_transition import ( state_transition, ) from eth2spec.utils.merkle_minimal import ( @@ -326,7 +326,7 @@ def test_voluntary_exit(state): def test_transfer(state): # overwrite default 0 to test - spec.MAX_TRANSFERS = 1 + spec.apply_constants_preset({"MAX_TRANSFERS": 1}) pre_state = deepcopy(state) current_epoch = get_current_epoch(pre_state) From 909158ed2d26964615c5f2e9d45b0a62632643a5 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Wed, 8 May 2019 23:41:44 +0100 Subject: [PATCH 075/308] Correct makefile --- Makefile | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 65c8bc33dc..c48b81b68e 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENER PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py PY_SPEC_PHASE_0_DEPS = $(SPEC_DIR)/core/0_*.md -PY_SPEC_PHASE_1_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase1/custody_game.py $(PY_SPEC_DIR)/eth2spec/phase1/shard_data_chains.py +PY_SPEC_PHASE_1_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase1/spec.py PY_SPEC_PHASE_1_DEPS = $(SPEC_DIR)/core/1_*.md PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) @@ -46,7 +46,7 @@ citest: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml . # "make pyspec" to create the pyspec for all phases. -pyspec: $(PY_SPEC_ALL_TARGETS) +pyspec: $(PY_SPEC_ALL_TARGETS) $(PY_SPEC_PHASE_1_TARGETS) # "make phase0" to create pyspec for phase0 phase0: $(PY_SPEC_PHASE_0_TARGETS) @@ -57,13 +57,9 @@ phase1: $(PY_SPEC_PHASE_1_TARGETS) $(PY_SPEC_DIR)/eth2spec/phase0/spec.py: $(PY_SPEC_PHASE_0_DEPS) python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@ -$(PY_SPEC_DIR)/eth2spec/phase1/custody_game.py: $(PY_SPEC_PHASE_1_DEPS) +$(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_custody-game.md $@ -$(PY_SPEC_DIR)/eth2spec/phase1/shard_data_chains.py: $(PY_SPEC_PHASE_0_DEPS) - python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_shard-data-chains.md $@ - - CURRENT_DIR = ${CURDIR} # The function that builds a set of suite files, by calling a generator for the given type (param 1) From 3217938b6f611892ed3fa71a06daa5891c155216 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Wed, 8 May 2019 23:48:12 +0100 Subject: [PATCH 076/308] Re-add early derived secret reveal tests --- .../phase1_test_process_randao_key_reveal.py | 109 ------------------ ...est_process_early_derived_secret_reveal.py | 109 ++++++++++++++++++ test_libs/pyspec/tests/helpers_phase1.py | 14 ++- 3 files changed, 117 insertions(+), 115 deletions(-) delete mode 100644 test_libs/pyspec/tests/block_processing_phase1/phase1_test_process_randao_key_reveal.py create mode 100644 test_libs/pyspec/tests/block_processing_phase1/test_process_early_derived_secret_reveal.py diff --git a/test_libs/pyspec/tests/block_processing_phase1/phase1_test_process_randao_key_reveal.py b/test_libs/pyspec/tests/block_processing_phase1/phase1_test_process_randao_key_reveal.py deleted file mode 100644 index 0be8ab4a97..0000000000 --- a/test_libs/pyspec/tests/block_processing_phase1/phase1_test_process_randao_key_reveal.py +++ /dev/null @@ -1,109 +0,0 @@ -from copy import deepcopy -import pytest - -import eth2spec.phase1.spec as spec -from eth2spec.phase1.spec import ( - get_current_epoch, - process_randao_key_reveal, - RANDAO_PENALTY_EPOCHS, - CUSTODY_PERIOD_TO_RANDAO_PADDING, - RANDAO_PENALTY_MAX_FUTURE_EPOCHS, -) -from tests.helpers_phase1 import ( - get_valid_randao_key_reveal, -) - -mark entire file as 'randao_key_reveals' -pytestmark = pytest.mark.randao_key_reveals - - -def run_randao_key_reveal_processing(state, randao_key_reveal, valid=True): - """ - Run ``process_randao_key_reveal`` returning the pre and post state. - If ``valid == False``, run expecting ``AssertionError`` - """ - post_state = deepcopy(state) - - if not valid: - with pytest.raises(AssertionError): - process_randao_key_reveal(post_state, randao_key_reveal) - return state, None - - process_randao_key_reveal(post_state, randao_key_reveal) - - slashed_validator = post_state.validator_registry[randao_key_reveal.revealed_index] - - if randao_key_reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - # lost whistleblower reward - # FIXME: Currently broken because get_base_reward in genesis epoch is 0 - assert ( - post_state.balances[randao_key_reveal.revealed_index] < - state.balances[randao_key_reveal.revealed_index] - ) - - return state, post_state - - -def test_success(state): - randao_key_reveal = get_valid_randao_key_reveal(state) - - pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal) - - return pre_state, randao_key_reveal, post_state - - -def test_reveal_from_current_epoch(state): - randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state)) - - pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, False) - - return pre_state, randao_key_reveal, post_state - -# Not currently possible as we are testing at epoch 0 -# -#def test_reveal_from_past_epoch(state): -# randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state) - 1) -# -# pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, False) -# -# return pre_state, randao_key_reveal, post_state - -def test_reveal_with_custody_padding(state): - randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING) - pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, True) - - return pre_state, randao_key_reveal, post_state - -def test_reveal_with_custody_padding_minus_one(state): - randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING - 1) - pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, True) - - return pre_state, randao_key_reveal, post_state - -def test_double_reveal(state): - - randao_key_reveal1 = get_valid_randao_key_reveal(state, get_current_epoch(state) + RANDAO_PENALTY_EPOCHS + 1) - pre_state, intermediate_state = run_randao_key_reveal_processing(state, randao_key_reveal1) - - randao_key_reveal2 = get_valid_randao_key_reveal(intermediate_state, get_current_epoch(pre_state) + RANDAO_PENALTY_EPOCHS + 1) - intermediate_state_, post_state = run_randao_key_reveal_processing(intermediate_state, randao_key_reveal2, False) - - return pre_state, [randao_key_reveal1, randao_key_reveal2], post_state - -def test_revealer_is_slashed(state): - randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state)) - state.validator_registry[randao_key_reveal.revealed_index].slashed = True - - pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, False) - - return pre_state, randao_key_reveal, post_state - -def test_far_future_epoch(state): - randao_key_reveal = get_valid_randao_key_reveal(state, get_current_epoch(state) + RANDAO_PENALTY_MAX_FUTURE_EPOCHS) - - pre_state, post_state = run_randao_key_reveal_processing(state, randao_key_reveal, False) - - return pre_state, randao_key_reveal, post_state diff --git a/test_libs/pyspec/tests/block_processing_phase1/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/tests/block_processing_phase1/test_process_early_derived_secret_reveal.py new file mode 100644 index 0000000000..30ae1e77ef --- /dev/null +++ b/test_libs/pyspec/tests/block_processing_phase1/test_process_early_derived_secret_reveal.py @@ -0,0 +1,109 @@ +from copy import deepcopy +import pytest + +import eth2spec.phase1.spec as spec +from eth2spec.phase1.spec import ( + get_current_epoch, + process_early_derived_secret_reveal, + RANDAO_PENALTY_EPOCHS, + CUSTODY_PERIOD_TO_RANDAO_PADDING, + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS, +) +from tests.helpers_phase1 import ( + get_valid_early_derived_secret_reveal, +) + +#mark entire file as 'randao_key_reveals' +pytestmark = pytest.mark.randao_key_reveals + + +def run_early_derived_secret_reveal_processing(state, randao_key_reveal, valid=True): + """ + Run ``process_randao_key_reveal`` returning the pre and post state. + If ``valid == False``, run expecting ``AssertionError`` + """ + post_state = deepcopy(state) + + if not valid: + with pytest.raises(AssertionError): + process_early_derived_secret_reveal(post_state, randao_key_reveal) + return state, None + + process_early_derived_secret_reveal(post_state, randao_key_reveal) + + slashed_validator = post_state.validator_registry[randao_key_reveal.revealed_index] + + if randao_key_reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + # lost whistleblower reward + # FIXME: Currently broken because get_base_reward in genesis epoch is 0 + assert ( + post_state.balances[randao_key_reveal.revealed_index] < + state.balances[randao_key_reveal.revealed_index] + ) + + return state, post_state + + +def test_success(state): + randao_key_reveal = get_valid_early_derived_secret_reveal(state) + + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal) + + return pre_state, randao_key_reveal, post_state + + +def test_reveal_from_current_epoch(state): + randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state)) + + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + + return pre_state, randao_key_reveal, post_state + +# Not currently possible as we are testing at epoch 0 +# +#def test_reveal_from_past_epoch(state): +# randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) - 1) +# +# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) +# +# return pre_state, randao_key_reveal, post_state + +def test_reveal_with_custody_padding(state): + randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING) + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) + + return pre_state, randao_key_reveal, post_state + +def test_reveal_with_custody_padding_minus_one(state): + randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING - 1) + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) + + return pre_state, randao_key_reveal, post_state + +def test_double_reveal(state): + + randao_key_reveal1 = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + RANDAO_PENALTY_EPOCHS + 1) + pre_state, intermediate_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal1) + + randao_key_reveal2 = get_valid_early_derived_secret_reveal(intermediate_state, get_current_epoch(pre_state) + RANDAO_PENALTY_EPOCHS + 1) + intermediate_state_, post_state = run_early_derived_secret_reveal_processing(intermediate_state, randao_key_reveal2, False) + + return pre_state, [randao_key_reveal1, randao_key_reveal2], post_state + +def test_revealer_is_slashed(state): + randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state)) + state.validator_registry[randao_key_reveal.revealed_index].slashed = True + + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + + return pre_state, randao_key_reveal, post_state + +def test_far_future_epoch(state): + randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS) + + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + + return pre_state, randao_key_reveal, post_state diff --git a/test_libs/pyspec/tests/helpers_phase1.py b/test_libs/pyspec/tests/helpers_phase1.py index aba93e1596..5b67e9aa65 100644 --- a/test_libs/pyspec/tests/helpers_phase1.py +++ b/test_libs/pyspec/tests/helpers_phase1.py @@ -1,12 +1,12 @@ from py_ecc import bls import eth2spec.phase1.spec as spec -from eth2spec.phase0.spec import ( +from eth2spec.phase1.spec import ( # constants ZERO_HASH, CUSTODY_PERIOD_TO_RANDAO_PADDING, # SSZ - RandaoKeyReveal, + EarlyDerivedSecretReveal, # functions get_active_validator_indices, get_current_epoch, @@ -14,7 +14,9 @@ hash_tree_root, ) -def get_valid_randao_key_reveal(state, epoch=None): +from .helpers import privkeys + +def get_valid_early_derived_secret_reveal(state, epoch=None): current_epoch = get_current_epoch(state) revealed_index = get_active_validator_indices(state, current_epoch)[-1] masker_index = get_active_validator_indices(state, current_epoch)[0] @@ -24,7 +26,7 @@ def get_valid_randao_key_reveal(state, epoch=None): reveal = bls.sign( message_hash=hash_tree_root(epoch), - privkey=pubkey_to_privkey[state.validator_registry[revealed_index].pubkey], + privkey=privkeys[revealed_index], domain=get_domain( state=state, domain_type=spec.DOMAIN_RANDAO, @@ -33,7 +35,7 @@ def get_valid_randao_key_reveal(state, epoch=None): ) mask = bls.sign( message_hash=hash_tree_root(epoch), - privkey=pubkey_to_privkey[state.validator_registry[masker_index].pubkey], + privkey=privkeys[masker_index], domain=get_domain( state=state, domain_type=spec.DOMAIN_RANDAO, @@ -41,7 +43,7 @@ def get_valid_randao_key_reveal(state, epoch=None): ), ) - return RandaoKeyReveal( + return EarlyDerivedSecretReveal( revealed_index=revealed_index, epoch=epoch, reveal=reveal, From 838496c3d4945f619924166f0edd9d3b7de857b1 Mon Sep 17 00:00:00 2001 From: Dankrad Feist Date: Thu, 9 May 2019 00:42:59 +0100 Subject: [PATCH 077/308] Fix Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c48b81b68e..5ca35c2575 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ PY_SPEC_PHASE_0_DEPS = $(SPEC_DIR)/core/0_*.md PY_SPEC_PHASE_1_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase1/spec.py PY_SPEC_PHASE_1_DEPS = $(SPEC_DIR)/core/1_*.md -PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) +PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) $(PY_SPEC_PHASE_1_TARGETS) .PHONY: clean all test citest gen_yaml_tests pyspec phase0 install_test @@ -46,7 +46,7 @@ citest: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml . # "make pyspec" to create the pyspec for all phases. -pyspec: $(PY_SPEC_ALL_TARGETS) $(PY_SPEC_PHASE_1_TARGETS) +pyspec: $(PY_SPEC_ALL_TARGETS) # "make phase0" to create pyspec for phase0 phase0: $(PY_SPEC_PHASE_0_TARGETS) From d7c1305ce4056e8778266fd83ced806aa5ef9ad4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 9 May 2019 13:11:07 +0800 Subject: [PATCH 078/308] Add flake8 check max-line-length=120 --- .circleci/config.yml | 14 ++++++++++++++ Makefile | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f806b00f2..fd7708f8da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,11 +84,25 @@ jobs: command: make citest - store_test_results: path: test_libs/pyspec/test-reports + lint: + docker: + - image: circleci/python:3.6 + working_directory: ~/specs-repo + steps: + - restore_cache: + key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} + - restore_cached_venv: + venv_name: v1-pyspec + reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + - run: + name: Run linter + command: make install_lint && make pyspec && make lint workflows: version: 2.1 test_spec: jobs: - checkout_specs + - lint - install_test: requires: - checkout_specs diff --git a/Makefile b/Makefile index 73d8adea89..8cc889f214 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,13 @@ test: $(PY_SPEC_ALL_TARGETS) citest: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml . +install_lint: + cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.5.0 + +lint: + cd $(PY_SPEC_DIR); . venv/bin/activate; \ + flake8 --max-line-length=120 ./eth2spec; + # "make pyspec" to create the pyspec for all phases. pyspec: $(PY_SPEC_ALL_TARGETS) From ef41d7a0649de997939c65618cde5ad0401b015e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 9 May 2019 13:45:00 +0800 Subject: [PATCH 079/308] No "import *" --- scripts/phase0/build_spec.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index e3c431a4aa..7ec84bed0e 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -12,8 +12,16 @@ def build_phase0_spec(sourcefile, outfile): NewType, Tuple, ) -from eth2spec.utils.minimal_ssz import * -from eth2spec.utils.bls_stub import * +from eth2spec.utils.minimal_ssz import ( + SSZType, + hash_tree_root, + signing_root, +) +from eth2spec.utils.bls_stub import ( + bls_aggregate_pubkeys, + bls_verify, + bls_verify_multiple, +) # stub, will get overwritten by real var SLOTS_PER_EPOCH = 64 From a4ba283d6770630a51b1524f8ba5bc696619e553 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 9 May 2019 13:54:38 +0800 Subject: [PATCH 080/308] Limit to 120 characters per line Limit to 120 characters per line --- specs/core/0_beacon-chain.md | 45 +++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a0c26bf59d..1b3c1e9d1f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -579,8 +579,10 @@ The types are defined topologically to aid in facilitating an executable version 'latest_block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], 'latest_state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], 'latest_active_index_roots': ['bytes32', LATEST_ACTIVE_INDEX_ROOTS_LENGTH], - 'latest_slashed_balances': ['uint64', LATEST_SLASHED_EXIT_LENGTH], # Balances slashed at every withdrawal period - 'latest_block_header': BeaconBlockHeader, # `latest_block_header.state_root == ZERO_HASH` temporarily + # Balances slashed at every withdrawal period + 'latest_slashed_balances': ['uint64', LATEST_SLASHED_EXIT_LENGTH], + # `latest_block_header.state_root == ZERO_HASH` temporarily + 'latest_block_header': BeaconBlockHeader, 'historical_roots': ['bytes32'], # Ethereum 1.0 chain data @@ -1149,7 +1151,9 @@ def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: #### `slash_validator` ```python -def slash_validator(state: BeaconState, slashed_index: ValidatorIndex, whistleblower_index: ValidatorIndex=None) -> None: +def slash_validator(state: BeaconState, + slashed_index: ValidatorIndex, + whistleblower_index: ValidatorIndex=None) -> None: """ Slash the validator with index ``slashed_index``. """ @@ -1287,7 +1291,8 @@ def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> List[Pen ``` ```python -def get_unslashed_attesting_indices(state: BeaconState, attestations: List[PendingAttestation]) -> List[ValidatorIndex]: +def get_unslashed_attesting_indices(state: BeaconState, + attestations: List[PendingAttestation]) -> List[ValidatorIndex]: output = set() for a in attestations: output = output.union(get_attesting_indices(state, a.data, a.aggregation_bitfield)) @@ -1300,7 +1305,9 @@ def get_attesting_balance(state: BeaconState, attestations: List[PendingAttestat ``` ```python -def get_winning_crosslink_and_attesting_indices(state: BeaconState, epoch: Epoch, shard: Shard) -> Tuple[Crosslink, List[ValidatorIndex]]: +def get_winning_crosslink_and_attesting_indices(state: BeaconState, + epoch: Epoch, + shard: Shard) -> Tuple[Crosslink, List[ValidatorIndex]]: attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.crosslink.shard == shard] crosslinks = list(filter( lambda c: hash_tree_root(state.current_crosslinks[shard]) in (c.parent_root, hash_tree_root(c)), @@ -1332,12 +1339,16 @@ def process_justification_and_finalization(state: BeaconState) -> None: state.previous_justified_epoch = state.current_justified_epoch state.previous_justified_root = state.current_justified_root state.justification_bitfield = (state.justification_bitfield << 1) % 2**64 - previous_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, previous_epoch)) + previous_epoch_matching_target_balance = get_attesting_balance( + state, get_matching_target_attestations(state, previous_epoch) + ) if previous_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = previous_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) state.justification_bitfield |= (1 << 1) - current_epoch_matching_target_balance = get_attesting_balance(state, get_matching_target_attestations(state, current_epoch)) + current_epoch_matching_target_balance = get_attesting_balance( + state, get_matching_target_attestations(state, current_epoch) + ) if current_epoch_matching_target_balance * 3 >= get_total_active_balance(state) * 2: state.current_justified_epoch = current_epoch state.current_justified_root = get_block_root(state, state.current_justified_epoch) @@ -1431,7 +1442,9 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: for index in eligible_validator_indices: penalties[index] += BASE_REWARDS_PER_EPOCH * get_base_reward(state, index) if index not in matching_target_attesting_indices: - penalties[index] += state.validator_registry[index].effective_balance * finality_delay // INACTIVITY_PENALTY_QUOTIENT + penalties[index] += ( + state.validator_registry[index].effective_balance * finality_delay // INACTIVITY_PENALTY_QUOTIENT + ) return rewards, penalties ``` @@ -1478,7 +1491,10 @@ Run the following function: def process_registry_updates(state: BeaconState) -> None: # Process activation eligibility and ejections for index, validator in enumerate(state.validator_registry): - if validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and validator.effective_balance >= MAX_EFFECTIVE_BALANCE: + if ( + validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH and + validator.effective_balance >= MAX_EFFECTIVE_BALANCE + ): validator.activation_eligibility_epoch = get_current_epoch(state) if is_active_validator(validator, get_current_epoch(state)) and validator.effective_balance <= EJECTION_BALANCE: @@ -1603,7 +1619,12 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: def process_randao(state: BeaconState, block: BeaconBlock) -> None: proposer = state.validator_registry[get_beacon_proposer_index(state)] # Verify that the provided randao value is valid - assert bls_verify(proposer.pubkey, hash_tree_root(get_current_epoch(state)), block.body.randao_reveal, get_domain(state, DOMAIN_RANDAO)) + assert bls_verify( + proposer.pubkey, + hash_tree_root(get_current_epoch(state)), + block.body.randao_reveal, + get_domain(state, DOMAIN_RANDAO), + ) # Mix it in state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = ( xor(get_randao_mix(state, get_current_epoch(state)), @@ -1748,7 +1769,9 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: validator_pubkeys = [v.pubkey for v in state.validator_registry] if pubkey not in validator_pubkeys: # Verify the deposit signature (proof of possession) - if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, get_domain(state, DOMAIN_DEPOSIT)): + if not bls_verify( + pubkey, signing_root(deposit.data), deposit.data.signature, get_domain(state, DOMAIN_DEPOSIT) + ): return # Add validator and balance entries From 84472a5a6e06324e457f3776e43f1192f005a418 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 9 May 2019 14:25:08 +0800 Subject: [PATCH 081/308] fix pyspec lint --- scripts/phase0/build_spec.py | 3 +++ scripts/phase0/function_puller.py | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index 7ec84bed0e..c8cd7348bb 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -22,6 +22,8 @@ def build_phase0_spec(sourcefile, outfile): bls_verify, bls_verify_multiple, ) +from eth2spec.utils.hash_function import hash + # stub, will get overwritten by real var SLOTS_PER_EPOCH = 64 @@ -69,6 +71,7 @@ def hash(x): hash_cache[x] = ret return ret + # Access to overwrite spec constants based on configuration def apply_constants_preset(preset: Dict[str, Any]): global_vars = globals() diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 1fad41fa9d..750f195904 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -55,15 +55,19 @@ def get_spec(file_name: str) -> List[str]: if eligible: code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890'))) # Build type-def re-initialization - code_lines.append('') + code_lines.append('\n') code_lines.append('def init_SSZ_types():') code_lines.append(' global_vars = globals()') for ssz_type_name, ssz_type in type_defs: code_lines.append('') for type_line in ssz_type: - code_lines.append(' ' + type_line) + if len(type_line) > 0: + code_lines.append(' ' + type_line) code_lines.append('\n') - code_lines.append('ssz_types = [' + ', '.join([f'\'{ssz_type_name}\'' for (ssz_type_name, _) in type_defs]) + ']') + code_lines.append('ssz_types = [\n') + for (ssz_type_name, _) in type_defs: + code_lines.append(f' {ssz_type_name},\n') + code_lines.append(']') code_lines.append('\n') code_lines.append('def get_ssz_type_by_name(name: str) -> SSZType:') code_lines.append(' return globals()[name]') From e1343f99bbff2054a89e06a3b6d904ec7c3519d7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 9 May 2019 14:29:21 +0800 Subject: [PATCH 082/308] Fix pyspec/eth2spec/debug lint --- test_libs/pyspec/eth2spec/debug/decode.py | 4 ++-- test_libs/pyspec/eth2spec/debug/encode.py | 1 - .../pyspec/eth2spec/debug/random_value.py | 23 +++++++++++++++---- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index aeac3924d9..e9aa8bc2b4 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -16,11 +16,11 @@ def decode(json, typ): for field, subtype in typ.fields.items(): temp[field] = decode(json[field], subtype) if field + "_hash_tree_root" in json: - assert(json[field + "_hash_tree_root"][2:] == + assert(json[field + "_hash_tree_root"][2:] == hash_tree_root(temp[field], subtype).hex()) ret = typ(**temp) if "hash_tree_root" in json: - assert(json["hash_tree_root"][2:] == + assert(json["hash_tree_root"][2:] == hash_tree_root(ret, typ).hex()) return ret else: diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index d3513e6382..b38e5fe98b 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -25,4 +25,3 @@ def encode(value, typ, include_hash_tree_roots=False): else: print(value, typ) raise Exception("Type not recognized") - diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index a853d2328c..f28181943c 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -31,7 +31,12 @@ def is_changing(self): return self.value in [0, 4, 5] -def get_random_ssz_object(rng: Random, typ: Any, max_bytes_length: int, max_list_length: int, mode: RandomizationMode, chaos: bool) -> Any: +def get_random_ssz_object(rng: Random, + typ: Any, + max_bytes_length: int, + max_list_length: int, + mode: RandomizationMode, + chaos: bool) -> Any: """ Create an object for a given type, filled with random data. :param rng: The random number generator to use. @@ -77,7 +82,10 @@ def get_random_ssz_object(rng: Random, typ: Any, max_bytes_length: int, max_list return get_random_basic_value(rng, typ) # Vector: elif isinstance(typ, list) and len(typ) == 2: - return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) for _ in range(typ[1])] + return [ + get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) + for _ in range(typ[1]) + ] # List: elif isinstance(typ, list) and len(typ) == 1: length = rng.randint(0, max_list_length) @@ -85,10 +93,17 @@ def get_random_ssz_object(rng: Random, typ: Any, max_bytes_length: int, max_list length = 1 if mode == RandomizationMode.mode_max_count: length = max_list_length - return [get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) for _ in range(length)] + return [ + get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) + for _ in range(length) + ] # Container: elif hasattr(typ, 'fields'): - return typ(**{field: get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode, chaos) for field, subtype in typ.fields.items()}) + return typ(**{ + field: + get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode, chaos) + for field, subtype in typ.fields.items() + }) else: print(typ) raise Exception("Type not recognized") From 4db4d879301b130eba3667d8f455de8eb07e7c80 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 9 May 2019 14:57:36 +0800 Subject: [PATCH 083/308] Refactor `process_final_updates` --- specs/core/1_custody-game.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 1b6b1d2e47..e03e54ed03 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -683,13 +683,12 @@ Append this to `process_final_updates(state)`: # Clean up exposed RANDAO key reveals state.exposed_derived_secrets[current_epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] # Reset withdrawable epochs if challenge records are empty + records = state.custody_chunk_challenge_records + state.bit_challenge_records + validator_indices_in_records = set( + [record.challenger_index for record in records] + [record.responder_index for record in records] + ) for index, validator in enumerate(state.validator_registry): - eligible = True - for records in (state.custody_chunk_challenge_records, state.bit_challenge_records): - for filter_func in (lambda rec: rec.challenger_index == index, lambda rec: rec.responder_index == index): - if len(list(filter(filter_func, records))) > 0: - eligible = False - if eligible: + if index not in validator_indices_in_records: if validator.exit_epoch == FAR_FUTURE_EPOCH: validator.withdrawable_epoch = FAR_FUTURE_EPOCH else: From 39fd625d350a456f4020d07e13cd56dbe3514d9b Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Thu, 9 May 2019 23:49:59 +1000 Subject: [PATCH 084/308] Started porting the API proposal into OpenAPI 3 format. --- specs/validator/beacon_node_oapi.yaml | 152 ++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 specs/validator/beacon_node_oapi.yaml diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml new file mode 100644 index 0000000000..3eac70a35e --- /dev/null +++ b/specs/validator/beacon_node_oapi.yaml @@ -0,0 +1,152 @@ +openapi: "3.0.2" +info: + title: "Beacon Node API for Validator" + description: "A beacon node API for enabling a validator to perform its obligations on the Ethereum 2.0 phase 0 beacon chain." + version: "0.1" +paths: + /node/version: + get: + summary: "Get version string of the running beacon node." + description: "Requests that the BeaconNode identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field." + responses: + 200: + description: Success + content: + application/json: + schema: + title: Duties + type: object + + /node/genesis_time: + get: + summary: "Get the genesis_time parameter from beacon node configuration." + description: "Requests the genesis_time parameter from the BeaconNode, which should be consistent across all BeaconNodes that follow the same beacon chain." + responses: + 200: + description: Success + content: + application/json: + schema: + title: genesis_time + type: integer + + /node/syncing: + get: + summary: "Poll to see if the the beacon node is syncing." + description: "Requests the beacon node to describe if it's currently syncing or not, and if it is, what block it is up to. This is modelled after the Eth1.0 JSON-RPC eth_syncing call.." + responses: + 200: + description: Success + content: + application/json: + schema: + title: genesis_time + type: integer + + + + /validator/duties: + get: + summary: "Get validator duties for the requested validators." + description: "Requests the BeaconNode to provide a set of _duties_, which are actions that should be performed by ValidatorClients. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected BeaconNode is properly synchronised." + parameters: + - name: validator_pubkeys + in: query + required: true + schema: + type: array + items: + type: string + format: byte + responses: + 200: + description: Success response + content: + application/json: + schema: + title: Duties + type: object + 503: + description: Beacon node syncing + + + /validator/block: + get: + summary: "Produce a new block, without signature." + description: "Requests a BeaconNode to produce a valid block, which can then be signed by a ValidatorClient." + parameters: + - name: slot + in: query + required: true + schema: + type: integer + - name: randao_reveal + in: query + required: true + schema: + type: string + format: byte + responses: + 200: + description: Success response + content: + application/json: + schema: + type: object + title: BeaconBlock + post: + summary: "Publish a signed block" + description: "Instructs the BeaconNode to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." + parameters: + - name: beacon_block + in: query + required: true + schema: + type: object + title: BeaconBlock + responses: + 200: + description: Success response + 503: + description: Beacon node syncing + + + /validator/attestation: + get: + summary: "Produce an attestation, without signature." + description: "Requests that the BeaconNode produce an IndexedAttestation, with a blank signature field, which the ValidatorClient will then sign." + parameters: + - name: slot + in: query + required: true + schema: + type: integer + - name: shard + in: query + required: true + schema: + type: integer + responses: + 200: + description: Success response + content: + application/json: + schema: + type: object + title: IndexedAttestation + post: + summary: "Published a signed attestation." + description: "Instructs the BeaconNode to publish a newly signed IndexedAttestation object, to be incorporated into the beacon chain." + parameters: + - name: attestation + in: query + required: true + description: "An IndexedAttestation structure, as originally provided by the BeaconNode, but now with the signature field completed." + schema: + type: object + title: IndexedAttestation + responses: + 200: + description: Success response + 503: + description: Beacon node syncing From b918cc3de35d6474f022c95e39402bac900dfec1 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Fri, 10 May 2019 14:03:59 +1000 Subject: [PATCH 085/308] Fleshed out a whole lot more of the OpenAPI specification for the API. --- specs/validator/beacon_node_oapi.yaml | 240 +++++++++++++++++++++++--- 1 file changed, 213 insertions(+), 27 deletions(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 3eac70a35e..0a583bdd6e 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -10,12 +10,13 @@ paths: description: "Requests that the BeaconNode identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field." responses: 200: - description: Success + description: Request successful content: application/json: schema: - title: Duties - type: object + $ref: '#/components/schemas/client_version' + 500: + $ref: '#/components/responses/InternalError' /node/genesis_time: get: @@ -23,12 +24,14 @@ paths: description: "Requests the genesis_time parameter from the BeaconNode, which should be consistent across all BeaconNodes that follow the same beacon chain." responses: 200: - description: Success + description: Request successful content: application/json: schema: - title: genesis_time - type: integer + $ref: '#/components/schemas/genesis_time' + 500: + $ref: '#/components/responses/InternalError' + /node/syncing: get: @@ -36,13 +39,19 @@ paths: description: "Requests the beacon node to describe if it's currently syncing or not, and if it is, what block it is up to. This is modelled after the Eth1.0 JSON-RPC eth_syncing call.." responses: 200: - description: Success + description: Request successful content: application/json: schema: - title: genesis_time - type: integer - + type: object + properties: + is_syncing: + type: boolean + description: "A boolean of whether the node is currently syncing or not." + sync_status: + $ref: '#/components/schemas/SyncingStatus' + 500: + $ref: '#/components/responses/InternalError' /validator/duties: @@ -53,21 +62,28 @@ paths: - name: validator_pubkeys in: query required: true + description: "An array of hex-encoded BLS public keys" schema: type: array items: - type: string - format: byte + $ref: '#/components/schemas/pubkey' + minItems: 1 responses: 200: description: Success response content: application/json: schema: - title: Duties - type: object + type: array + items: + $ref: '#/components/schemas/ValidatorDuty' + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' 503: - description: Beacon node syncing + $ref: '#/components/responses/CurrentlySyncing' + /validator/block: @@ -78,11 +94,14 @@ paths: - name: slot in: query required: true + description: "The slot for which the block should be proposed." schema: type: integer + format: uint64 - name: randao_reveal in: query required: true + description: "The ValidatorClient's randao reveal value." schema: type: string format: byte @@ -92,8 +111,13 @@ paths: content: application/json: schema: - type: object - title: BeaconBlock + $ref: '#/components/schemas/BeaconBlock' + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' + 503: + $ref: '#/components/responses/CurrentlySyncing' post: summary: "Publish a signed block" description: "Instructs the BeaconNode to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." @@ -101,14 +125,18 @@ paths: - name: beacon_block in: query required: true + description: "The BeaconBlock object, as sent from the BeaconNode originally, but now with the signature field completed." schema: - type: object - title: BeaconBlock + $ref: '#/components/schemas/BeaconBlock' responses: 200: - description: Success response + $ref: '#/components/responses/Success' + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' 503: - description: Beacon node syncing + $ref: '#/components/responses/CurrentlySyncing' /validator/attestation: @@ -119,11 +147,13 @@ paths: - name: slot in: query required: true + description: "The slot for which the attestation should be proposed." schema: type: integer - name: shard in: query required: true + description: "The shard number for which the attestation is to be proposed." schema: type: integer responses: @@ -132,8 +162,14 @@ paths: content: application/json: schema: - type: object - title: IndexedAttestation + $ref: '#/components/schemas/IndexedAttestation' + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' + 503: + $ref: '#/components/responses/CurrentlySyncing' + post: summary: "Published a signed attestation." description: "Instructs the BeaconNode to publish a newly signed IndexedAttestation object, to be incorporated into the beacon chain." @@ -143,10 +179,160 @@ paths: required: true description: "An IndexedAttestation structure, as originally provided by the BeaconNode, but now with the signature field completed." schema: - type: object - title: IndexedAttestation + $ref: '#/components/schemas/IndexedAttestation' responses: 200: - description: Success response + $ref: '#/components/responses/Success' + 400: + $ref: '#/components/responses/InvalidRequest' + 500: + $ref: '#/components/responses/InternalError' 503: - description: Beacon node syncing + $ref: '#/components/responses/CurrentlySyncing' + +components: + schemas: + ValidatorDuty: + type: object + properties: + validator_pubkey: + $ref: '#/components/schemas/pubkey' + committee_index: + type: integer + format: uint64 + description: "The index of the validator in the committee." + attestation_slot: + type: integer + format: uint64 + description: "The slot at which the validator must attest." + attestation_shard: + type: integer + format: uint64 + description: "The shard in which the validator must attest." + block_production_slot: + type: integer + format: uint64 + nullable: true + description: "The slot in which a validator must propose a block, or `null` if block production is not required." + SyncingStatus: + type: object + nullable: true + properties: + starting_block: + type: integer + format: uint64 + description: "The block at which syncing started (will only be reset after the sync reached its head)" + current_block: + type: integer + format: uint64 + description: "The current highest block sync'd by the beacon node." + highest_block: + type: integer + format: uint64 + description: "The estimated highest block, or current target block number." + BeaconBlock: + type: object + description: "The [BeaconBlock](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblock) from the Eth2.0 spec." + Fork: + type: object + description: "The [Fork](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#Fork) from the Eth2.0 spec." + properties: + previous_version: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{8}$" + description: "Previous fork version" + current_version: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{8}$" + description: "Current fork version" + epoch: + type: integer + format: uint64 + description: "Fork epoch number" + IndexedAttestation: + type: object + description: "The [IndexedAttestation](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#indexedattestation) from the Eth2.0 spec." + properties: + custody_bit_0_indicies: + type: array + description: "Validator indicies for 0 bits." + items: + type: integer + format: uint64 + custody_bit_1_indicies: + type: array + description: "Validator indicies for 1 bits." + items: + type: integer + format: uint64 + data: + $ref: '#/components/schemas/AttestationData' + + AttestationData: + type: object + description: "The [AttestationData](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attestationdata) from the Eth2.0 spec." + properties: + beacon_block_root: + $ref: '#/components/schemas/merkle_root' + source_epoch: + type: integer + format: uint64 + description: "Source epoch from FFG vote" + source_root: + $ref: '#/components/schemas/merkle_root' + target_epoch: + type: integer + format: uint64 + description: "Target epoch from FFG vote" + target_root: + $ref: '#/components/schemas/merkle_root' + crosslink: + $ref: '#/components/schemas/CrossLink' + CrossLink: + type: object + description: "The [Crosslink](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) from the Eth2.0 spec." + properties: + shard: + type: integer + format: uint64 + description: "The shard number" + epoch: + type: integer + format: uint64 + description: "The epoch number" + parent_root: + $ref: '#/components/schemas/merkle_root' + data_root: + $ref: '#/components/schemas/merkle_root' + + pubkey: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{96}$" + description: "The validator's BLS public key, uniquely identifying them." + client_version: + type: string + description: "A string which uniquely identifies the client implementation and its version; similar to [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3)." + example: "Lighthouse / v0.1.5 (Linux x86_64)" + genesis_time: + type: integer + format: uint64 + description: "The genesis_time configured for the beacon node, which is the time the Eth1.0 validator deposit smart contract has enough ETH staked (i.e. Eth2.0 begins)." + merkle_root: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "A 32 byte merkle root, used in all kinds of scenarios." + + + responses: + Success: + description: Request successful + InvalidRequest: + description: Invalid request syntax + InternalError: + description: Beacon node internal error + CurrentlySyncing: + description: Beacon node is currently syncing, try again later From a552b94d46d744e5ec005d09da1f0b4e9b49c81e Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Fri, 10 May 2019 11:02:39 -0700 Subject: [PATCH 086/308] Update 0_beacon-chain-validator.md typo fix --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 35de666d72..49290b432d 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -246,7 +246,7 @@ Set `attestation_data.beacon_block_root = signing_root(head_block)`. ##### FFG vote -* Set `attestation_data.source_epoch = head_state.justified_epoch`. +* Set `attestation_data.source_epoch = head_state.current_justified_epoch`. * Set `attestation_data.source_root = head_state.current_justified_root`. * Set `attestation_data.target_epoch = get_current_epoch(head_state)` * Set `attestation_data.target_root = signing_root(epoch_boundary_block)` where `epoch_boundary_block` is the block at the most recent epoch boundary. From c0d4334ced5216b002b65bd682a1f5ab8f65d43b Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 11 May 2019 15:12:11 +0800 Subject: [PATCH 087/308] Fix SSZ union type example --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index b4ea8135a7..8633c7ed15 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -47,7 +47,7 @@ * **list**: ordered variable-length homogeneous collection of values * angle bracket notation `[type]`, e.g. `["uint64"]` * **union**: union type containing one of the given subtypes - * round bracket notation `(type_1, type_2, ...)`, e.g. `("uint64", "null")` + * round bracket notation `(type_1, type_2, ...)`, e.g. `("null", "uint64")` ### Variable-size and fixed-size From ae19521ea8e0e66ff9f0d1592d28bec1799d9d61 Mon Sep 17 00:00:00 2001 From: Ryuya Nakamura Date: Sat, 11 May 2019 20:41:05 +0900 Subject: [PATCH 088/308] Update the docstring of initiate_validator_exit (#1072) --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a0c26bf59d..103870ce20 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1127,7 +1127,7 @@ def get_churn_limit(state: BeaconState) -> int: ```python def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: """ - Initiate the validator of the given ``index``. + Initiate the exit of the validator of the given ``index``. """ # Return if validator already initiated exit validator = state.validator_registry[index] From 36514d1c384290870e726c306e57d298be22a40b Mon Sep 17 00:00:00 2001 From: Jonny Rhea Date: Sat, 11 May 2019 16:49:46 -0500 Subject: [PATCH 089/308] Update 0_deposit-contract.md get_genesis_beacon_state is not defined here. i modified the spec to link to the function. --- specs/core/0_deposit-contract.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index 6843e407ec..68ba027567 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -72,7 +72,7 @@ Every Ethereum 1.0 deposit, of size at least `MIN_DEPOSIT_AMOUNT`, emits a `Depo ### `Eth2Genesis` log -When `CHAIN_START_FULL_DEPOSIT_THRESHOLD` of full deposits have been made, the deposit contract emits the `Eth2Genesis` log. The beacon chain state may then be initialized by calling the `get_genesis_beacon_state` function (defined below) where: +When `CHAIN_START_FULL_DEPOSIT_THRESHOLD` of full deposits have been made, the deposit contract emits the `Eth2Genesis` log. The beacon chain state may then be initialized by calling the `get_genesis_beacon_state` function (defined [here](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#genesis-state)) where: * `genesis_time` equals `time` in the `Eth2Genesis` log * `latest_eth1_data.deposit_root` equals `deposit_root` in the `Eth2Genesis` log From e85847719f9b4b9d4ca7ef4a719cd823b09e51e2 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 12 May 2019 08:50:46 -0600 Subject: [PATCH 090/308] use relative path for beacon chain link --- specs/core/0_deposit-contract.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index 68ba027567..f0259a1799 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -72,7 +72,7 @@ Every Ethereum 1.0 deposit, of size at least `MIN_DEPOSIT_AMOUNT`, emits a `Depo ### `Eth2Genesis` log -When `CHAIN_START_FULL_DEPOSIT_THRESHOLD` of full deposits have been made, the deposit contract emits the `Eth2Genesis` log. The beacon chain state may then be initialized by calling the `get_genesis_beacon_state` function (defined [here](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#genesis-state)) where: +When `CHAIN_START_FULL_DEPOSIT_THRESHOLD` of full deposits have been made, the deposit contract emits the `Eth2Genesis` log. The beacon chain state may then be initialized by calling the `get_genesis_beacon_state` function (defined [here](./0_beacon-chain.md#genesis-state)) where: * `genesis_time` equals `time` in the `Eth2Genesis` log * `latest_eth1_data.deposit_root` equals `deposit_root` in the `Eth2Genesis` log From ba1949b2bc59be426bca8db1f0cd5a9eb0c2c376 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Sun, 12 May 2019 13:46:17 -0700 Subject: [PATCH 091/308] Update 1_custody-game.md --- specs/core/1_custody-game.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index a030c577bf..429e8e035b 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -104,6 +104,8 @@ This document details the beacon chain additions and changes in Phase 1 of Ether ### Reward and penalty quotients +| Name | Value | +| - | - | | `EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE` | `2**1` (= 2) | ### Signature domains From 761c9e55fe879a4e19b907e1a2d9673e93aebc20 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 12 May 2019 23:16:17 +0200 Subject: [PATCH 092/308] SSZ impl. rework started, see issue 1064 --- .../pyspec/eth2spec/utils/merkle_minimal.py | 32 +- .../pyspec/eth2spec/utils/minimal_ssz.py | 331 ------------------ .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 117 +++++++ .../pyspec/eth2spec/utils/ssz/ssz_switch.py | 105 ++++++ .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 115 ++++++ 5 files changed, 368 insertions(+), 332 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/utils/minimal_ssz.py create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index 7c5483de31..e3e5d35d8c 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -1,7 +1,9 @@ from .hash_function import hash -zerohashes = [b'\x00' * 32] +ZERO_BYTES32 = b'\x00' * 32 + +zerohashes = [ZERO_BYTES32] for layer in range(1, 32): zerohashes.append(hash(zerohashes[layer - 1] + zerohashes[layer - 1])) @@ -28,3 +30,31 @@ def get_merkle_proof(tree, item_index): subindex = (item_index // 2**i) ^ 1 proof.append(tree[i][subindex] if subindex < len(tree[i]) else zerohashes[i]) return proof + + +def next_power_of_two(v: int) -> int: + """ + Get the next power of 2. (for 64 bit range ints) + Examples: + 0 -> 0, 1 -> 1, 2 -> 2, 3 -> 4, 32 -> 32, 33 -> 64 + """ + # effectively fill the bitstring (1 less, do not want to with ones, then increment for next power of 2. + v -= 1 + v |= v >> (1 << 0) + v |= v >> (1 << 1) + v |= v >> (1 << 2) + v |= v >> (1 << 3) + v |= v >> (1 << 4) + v |= v >> (1 << 5) + v += 1 + return v + + +def merkleize_chunks(chunks): + tree = chunks[::] + margin = next_power_of_two(len(chunks)) - len(chunks) + tree.extend([ZERO_BYTES32] * margin) + tree = [ZERO_BYTES32] * len(tree) + tree + for i in range(len(tree) // 2 - 1, 0, -1): + tree[i] = hash(tree[i * 2] + tree[i * 2 + 1]) + return tree[1] diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py deleted file mode 100644 index 9cc2baebb3..0000000000 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ /dev/null @@ -1,331 +0,0 @@ -from typing import Any - -from .hash_function import hash - -BYTES_PER_CHUNK = 32 -BYTES_PER_LENGTH_OFFSET = 4 -ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK - - -def SSZType(fields): - class SSZObject(): - def __init__(self, **kwargs): - for f, t in fields.items(): - if f not in kwargs: - setattr(self, f, get_zero_value(t)) - else: - setattr(self, f, kwargs[f]) - - def __eq__(self, other): - return self.fields == other.fields and self.serialize() == other.serialize() - - def __hash__(self): - return int.from_bytes(self.hash_tree_root(), byteorder="little") - - def __str__(self): - output = [] - for field in self.fields: - output.append(f'{field}: {getattr(self, field)}') - return "\n".join(output) - - def serialize(self): - return serialize_value(self, self.__class__) - - def hash_tree_root(self): - return hash_tree_root(self, self.__class__) - - SSZObject.fields = fields - return SSZObject - - -class Vector(): - def __init__(self, items): - self.items = items - self.length = len(items) - - def __getitem__(self, key): - return self.items[key] - - def __setitem__(self, key, value): - self.items[key] = value - - def __iter__(self): - return iter(self.items) - - def __len__(self): - return self.length - - -def is_basic(typ): - # if not a string, it is a complex, and cannot be basic - if not isinstance(typ, str): - return False - # "uintN": N-bit unsigned integer (where N in [8, 16, 32, 64, 128, 256]) - elif typ[:4] == 'uint' and typ[4:] in ['8', '16', '32', '64', '128', '256']: - return True - # "bool": True or False - elif typ == 'bool': - return True - # alias: "byte" -> "uint8" - elif typ == 'byte': - return True - # default - else: - return False - - -def is_constant_sized(typ): - # basic objects are fixed size by definition - if is_basic(typ): - return True - # dynamic size array type, "list": [elem_type]. - # Not constant size by definition. - elif isinstance(typ, list) and len(typ) == 1: - return False - # fixed size array type, "vector": [elem_type, length] - # Constant size, but only if the elements are. - elif isinstance(typ, list) and len(typ) == 2: - return is_constant_sized(typ[0]) - # bytes array (fixed or dynamic size) - elif isinstance(typ, str) and typ[:5] == 'bytes': - # if no length suffix, it has a dynamic size - return typ != 'bytes' - # containers are only constant-size if all of the fields are constant size. - elif hasattr(typ, 'fields'): - for subtype in typ.fields.values(): - if not is_constant_sized(subtype): - return False - return True - else: - raise Exception("Type not recognized") - - -def coerce_to_bytes(x): - if isinstance(x, str): - o = x.encode('utf-8') - assert len(o) == len(x) - return o - elif isinstance(x, bytes): - return x - else: - raise Exception("Expecting bytes") - - -def encode_series(values, types): - # Recursively serialize - parts = [(is_constant_sized(types[i]), serialize_value(values[i], types[i])) for i in range(len(values))] - - # Compute and check lengths - fixed_lengths = [len(serialized) if constant_size else BYTES_PER_LENGTH_OFFSET - for (constant_size, serialized) in parts] - variable_lengths = [len(serialized) if not constant_size else 0 - for (constant_size, serialized) in parts] - - # Check if integer is not out of bounds (Python) - assert sum(fixed_lengths + variable_lengths) < 2 ** (BYTES_PER_LENGTH_OFFSET * 8) - - # Interleave offsets of variable-size parts with fixed-size parts. - # Avoid quadratic complexity in calculation of offsets. - offset = sum(fixed_lengths) - variable_parts = [] - fixed_parts = [] - for (constant_size, serialized) in parts: - if constant_size: - fixed_parts.append(serialized) - else: - fixed_parts.append(offset.to_bytes(BYTES_PER_LENGTH_OFFSET, 'little')) - variable_parts.append(serialized) - offset += len(serialized) - - # Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts - return b"".join(fixed_parts + variable_parts) - - -def serialize_value(value, typ=None): - if typ is None: - typ = infer_type(value) - # "uintN" - if isinstance(typ, str) and typ[:4] == 'uint': - length = int(typ[4:]) - assert length in (8, 16, 32, 64, 128, 256) - return value.to_bytes(length // 8, 'little') - # "bool" - elif isinstance(typ, str) and typ == 'bool': - assert value in (True, False) - return b'\x01' if value is True else b'\x00' - # Vector - elif isinstance(typ, list) and len(typ) == 2: - # (regardless of element type, sanity-check if the length reported in the vector type matches the value length) - assert len(value) == typ[1] - return encode_series(value, [typ[0]] * len(value)) - # List - elif isinstance(typ, list) and len(typ) == 1: - return encode_series(value, [typ[0]] * len(value)) - # "bytes" (variable size) - elif isinstance(typ, str) and typ == 'bytes': - return coerce_to_bytes(value) - # "bytesN" (fixed size) - elif isinstance(typ, str) and len(typ) > 5 and typ[:5] == 'bytes': - assert len(value) == int(typ[5:]), (value, int(typ[5:])) - return coerce_to_bytes(value) - # containers - elif hasattr(typ, 'fields'): - values = [getattr(value, field) for field in typ.fields.keys()] - types = list(typ.fields.values()) - return encode_series(values, types) - else: - print(value, typ) - raise Exception("Type not recognized") - - -def get_zero_value(typ: Any) -> Any: - if isinstance(typ, str): - # Bytes array - if typ == 'bytes': - return b'' - # bytesN - elif typ[:5] == 'bytes' and len(typ) > 5: - length = int(typ[5:]) - return b'\x00' * length - # Basic types - elif typ == 'bool': - return False - elif typ[:4] == 'uint': - return 0 - elif typ == 'byte': - return 0x00 - else: - raise ValueError("Type not recognized") - # Vector: - elif isinstance(typ, list) and len(typ) == 2: - return [get_zero_value(typ[0]) for _ in range(typ[1])] - # List: - elif isinstance(typ, list) and len(typ) == 1: - return [] - # Container: - elif hasattr(typ, 'fields'): - return typ(**{field: get_zero_value(subtype) for field, subtype in typ.fields.items()}) - else: - print(typ) - raise Exception("Type not recognized") - - -def chunkify(bytez): - bytez += b'\x00' * (-len(bytez) % BYTES_PER_CHUNK) - return [bytez[i:i + 32] for i in range(0, len(bytez), 32)] - - -def pack(values, subtype): - return chunkify(b''.join([serialize_value(value, subtype) for value in values])) - - -def is_power_of_two(x): - return x > 0 and x & (x - 1) == 0 - - -def merkleize(chunks): - tree = chunks[::] - while not is_power_of_two(len(tree)): - tree.append(ZERO_CHUNK) - tree = [ZERO_CHUNK] * len(tree) + tree - for i in range(len(tree) // 2 - 1, 0, -1): - tree[i] = hash(tree[i * 2] + tree[i * 2 + 1]) - return tree[1] - - -def mix_in_length(root, length): - return hash(root + length.to_bytes(32, 'little')) - - -def infer_type(value): - """ - Note: defaults to uint64 for integer type inference due to lack of information. - Other integer sizes are still supported, see spec. - :param value: The value to infer a SSZ type for. - :return: The SSZ type. - """ - if hasattr(value.__class__, 'fields'): - return value.__class__ - elif isinstance(value, Vector): - if len(value) > 0: - return [infer_type(value[0]), len(value)] - else: - # Element type does not matter too much, - # assumed to be a basic type for size-encoding purposes, vector is empty. - return ['uint64'] - elif isinstance(value, list): - if len(value) > 0: - return [infer_type(value[0])] - else: - # Element type does not matter, list-content size will be encoded regardless, list is empty. - return ['uint64'] - elif isinstance(value, (bytes, str)): - return 'bytes' - elif isinstance(value, int): - return 'uint64' - else: - raise Exception("Failed to infer type") - - -def hash_tree_root(value, typ=None): - if typ is None: - typ = infer_type(value) - # ------------------------------------- - # merkleize(pack(value)) - # basic object: merkleize packed version (merkleization pads it to 32 bytes if it is not already) - if is_basic(typ): - return merkleize(pack([value], typ)) - # or a vector of basic objects - elif isinstance(typ, list) and len(typ) == 2 and is_basic(typ[0]): - assert len(value) == typ[1] - return merkleize(pack(value, typ[0])) - # ------------------------------------- - # mix_in_length(merkleize(pack(value)), len(value)) - # if value is a list of basic objects - elif isinstance(typ, list) and len(typ) == 1 and is_basic(typ[0]): - return mix_in_length(merkleize(pack(value, typ[0])), len(value)) - # (needs some extra work for non-fixed-sized bytes array) - elif typ == 'bytes': - return mix_in_length(merkleize(chunkify(coerce_to_bytes(value))), len(value)) - # ------------------------------------- - # merkleize([hash_tree_root(element) for element in value]) - # if value is a vector of composite objects - elif isinstance(typ, list) and len(typ) == 2 and not is_basic(typ[0]): - return merkleize([hash_tree_root(element, typ[0]) for element in value]) - # (needs some extra work for fixed-sized bytes array) - elif isinstance(typ, str) and typ[:5] == 'bytes' and len(typ) > 5: - assert len(value) == int(typ[5:]) - return merkleize(chunkify(coerce_to_bytes(value))) - # or a container - elif hasattr(typ, 'fields'): - return merkleize([hash_tree_root(getattr(value, field), subtype) for field, subtype in typ.fields.items()]) - # ------------------------------------- - # mix_in_length(merkleize([hash_tree_root(element) for element in value]), len(value)) - # if value is a list of composite objects - elif isinstance(typ, list) and len(typ) == 1 and not is_basic(typ[0]): - return mix_in_length(merkleize([hash_tree_root(element, typ[0]) for element in value]), len(value)) - # ------------------------------------- - else: - raise Exception("Type not recognized") - - -def truncate(container): - field_keys = list(container.fields.keys()) - truncated_fields = { - key: container.fields[key] - for key in field_keys[:-1] - } - truncated_class = SSZType(truncated_fields) - kwargs = { - field: getattr(container, field) - for field in field_keys[:-1] - } - return truncated_class(**kwargs) - - -def signing_root(container): - return hash_tree_root(truncate(container)) - - -def serialize(ssz_object): - return getattr(ssz_object, 'serialize')() diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py new file mode 100644 index 0000000000..74bc1bf99b --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -0,0 +1,117 @@ +from eth2spec.utils.merkle_minimal import merkleize_chunks +from .ssz_switch import * + +# SSZ Helpers +# ----------------------------- + + +def pack(values, subtype): + return b''.join([serialize(value, subtype) for value in values]) + + +def chunkify(byte_string): + byte_string += b'\x00' * (-len(byte_string) % 32) + return [byte_string[i:i + 32] for i in range(0, len(byte_string), 32)] + + +BYTES_PER_LENGTH_OFFSET = 4 + + +# SSZ Implementation +# ----------------------------- + +get_zero_value = ssz_type_switch({ + ssz_bool: lambda: False, + ssz_uint: lambda: 0, + ssz_list: lambda byte_form: b'' if byte_form else [], + ssz_vector: lambda length, elem_typ, byte_form: + (b'\x00' * length if length > 0 else b'') if byte_form else + [get_zero_value(elem_typ) for _ in range(length)], + ssz_container: lambda typ, field_names, field_types: + typ(**{f_name: get_zero_value(f_typ) for f_name, f_typ in zip(field_names, field_types)}), +}) + + +serialize = ssz_switch({ + ssz_bool: lambda value: b'\x01' if value else b'\x00', + ssz_uint: lambda value, byte_len: value.to_bytes(byte_len, 'little'), + ssz_list: lambda value, elem_typ: encode_series(value, [elem_typ] * len(value)), + ssz_vector: lambda value, elem_typ, length: encode_series(value, [elem_typ] * length), + ssz_container: lambda value, get_field_values, field_types: encode_series(get_field_values(value), field_types), +}) + +ssz_basic_type = (ssz_bool, ssz_uint) + +is_basic_type = ssz_type_switch({ + ssz_basic_type: lambda: True, + ssz_default: lambda: False, +}) + +is_fixed_size = ssz_type_switch({ + ssz_basic_type: lambda: True, + ssz_vector: lambda elem_typ: is_fixed_size(elem_typ), + ssz_container: lambda field_types: all(is_fixed_size(f_typ) for f_typ in field_types), + ssz_list: lambda: False, +}) + + +def hash_tree_root_list(value, elem_typ): + if is_basic_type(elem_typ): + return merkleize_chunks(chunkify(pack(value, elem_typ))) + else: + return merkleize_chunks([hash_tree_root(element, elem_typ) for element in value]) + + +def mix_in_length(root, length): + return hash(root + length.to_bytes(32, 'little')) + + +def hash_tree_root_container(fields): + return merkleize_chunks([hash_tree_root(field, subtype) for field, subtype in fields]) + + +hash_tree_root = ssz_switch({ + ssz_basic_type: lambda value, typ: merkleize_chunks(chunkify(pack([value], typ))), + ssz_list: lambda value, elem_typ: mix_in_length(hash_tree_root_list(value, elem_typ), len(value)), + ssz_vector: lambda value, elem_typ: hash_tree_root_list(value, elem_typ), + ssz_container: lambda value, get_field_values, field_types: hash_tree_root_container(zip(get_field_values(value), field_types)), +}) + +signing_root = ssz_switch({ + ssz_container: lambda value, get_field_values, field_types: hash_tree_root_container(zip(get_field_values(value), field_types)[:-1]), + ssz_default: lambda value, typ: hash_tree_root(value, typ), +}) + + +def encode_series(values, types): + # bytes and bytesN are already in the right format. + if isinstance(values, bytes): + return values + + # Recursively serialize + parts = [(is_fixed_size(types[i]), serialize(values[i], types[i])) for i in range(len(values))] + + # Compute and check lengths + fixed_lengths = [len(serialized) if constant_size else BYTES_PER_LENGTH_OFFSET + for (constant_size, serialized) in parts] + variable_lengths = [len(serialized) if not constant_size else 0 + for (constant_size, serialized) in parts] + + # Check if integer is not out of bounds (Python) + assert sum(fixed_lengths + variable_lengths) < 2 ** (BYTES_PER_LENGTH_OFFSET * 8) + + # Interleave offsets of variable-size parts with fixed-size parts. + # Avoid quadratic complexity in calculation of offsets. + offset = sum(fixed_lengths) + variable_parts = [] + fixed_parts = [] + for (constant_size, serialized) in parts: + if constant_size: + fixed_parts.append(serialized) + else: + fixed_parts.append(offset.to_bytes(BYTES_PER_LENGTH_OFFSET, 'little')) + variable_parts.append(serialized) + offset += len(serialized) + + # Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts + return b''.join(fixed_parts + variable_parts) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py new file mode 100644 index 0000000000..3da1e7cb1f --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py @@ -0,0 +1,105 @@ +from typing import Dict, Any + +from .ssz_typing import * + +# SSZ Switch statement runner factory +# ----------------------------- + + +def ssz_switch(sw: Dict[Any, Any], arg_names=None): + """ + Creates an SSZ switch statement: a function, that when executed, checks every switch-statement + """ + if arg_names is None: + arg_names = ["value", "typ"] + + # Runner, the function that executes the switch when called. + # Accepts a arguments based on the arg_names declared in the ssz_switch. + def run_switch(*args): + # value may be None + value = None + try: + value = args[arg_names.index("value")] + except ValueError: + pass # no value argument + + # typ may be None when value is not None + typ = None + try: + typ = args[arg_names.index("typ")] + except ValueError: + # no typ argument expected + pass + except IndexError: + # typ argument expected, but not passed. Try to get it from the class info + typ = value.__class__ + if hasattr(typ, '__forward_arg__'): + typ = typ.__forward_arg__ + + # Now, go over all switch cases + for matchers, worker in sw.items(): + if not isinstance(matchers, tuple): + matchers = (matchers,) + # for each matcher of the case key + for m in matchers: + data = m(typ) + # if we have data, the matcher matched, and we can return the result + if data is not None: + # Supply value and type by default, and any data presented by the matcher. + kwargs = {"value": value, "typ": typ, **data} + # Filter out unwanted arguments + filtered_kwargs = {k: kwargs[k] for k in worker.__code__.co_varnames} + # run the switch case and return result + return worker(**filtered_kwargs) + raise Exception("cannot find matcher for type: %s (value: %s)" % (typ, value)) + return run_switch + + +def ssz_type_switch(sw: Dict[Any, Any]): + return ssz_switch(sw, ["typ"]) + + +# SSZ Switch matchers +# ----------------------------- + +def ssz_bool(typ): + if typ == bool: + return {} + + +def ssz_uint(typ): + # Note: only the type reference exists, + # but it really resolves to 'int' during run-time for zero computational/memory overhead. + # Hence, we check equality to the type references (which are really just 'NewType' instances), + # and don't use any sub-classing like we normally would. + if typ == uint8 or typ == uint16 or typ == uint32 or typ == uint64\ + or typ == uint128 or typ == uint256 or typ == byte: + return {"byte_len": typ.byte_len} + + +def ssz_list(typ): + if hasattr(typ, '__bases__') and List in typ.__bases__: + return {"elem_typ": read_list_elem_typ(typ), "byte_form": False} + if typ == bytes: + return {"elem_typ": uint8, "byte_form": True} + + +def ssz_vector(typ): + if hasattr(typ, '__bases__'): + if Vector in typ.__bases__: + return {"elem_typ": read_vec_elem_typ(typ), "length": read_vec_len(typ), "byte_form": False} + if BytesN in typ.__bases__: + return {"elem_typ": uint8, "length": read_bytesN_len(typ), "byte_form": True} + + +def ssz_container(typ): + if hasattr(typ, '__bases__') and SSZContainer in typ.__bases__: + def get_field_values(value): + return [getattr(value, field) for field in typ.__annotations__.keys()] + field_names = list(typ.__annotations__.keys()) + field_types = list(typ.__annotations__.values()) + return {"get_field_values": get_field_values, "field_names": field_names, "field_types": field_types} + + +def ssz_default(typ): + return {} diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py new file mode 100644 index 0000000000..6a8a22586e --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -0,0 +1,115 @@ +from typing import Generic, List, TypeVar, Type, Iterable, NewType + +# SSZ base length, to limit length generic type param of vector/bytesN +SSZLenAny = type('SSZLenAny', (), {}) + + +def SSZLen(length: int): + """ + SSZ length factory. Creates a type corresponding to a given length. To be used as parameter in type generics. + """ + assert length >= 0 + typ = type('SSZLen_%d' % length, (SSZLenAny,), {}) + typ.length = length + return typ + + +# SSZ element type +T = TypeVar('T') +# SSZ vector/bytesN length +L = TypeVar('L', bound=SSZLenAny) + + +# SSZ vector +# ----------------------------- + +class Vector(Generic[T, L]): + def __init__(self, *args: Iterable[T]): + self.items = list(args) + + def __getitem__(self, key): + return self.items[key] + + def __setitem__(self, key, value): + self.items[key] = value + + def __iter__(self): + return iter(self.items) + + def __len__(self): + return len(self.items) + + +def read_vec_elem_typ(vec_typ: Type[Vector[T,L]]) -> T: + assert vec_typ.__args__ is not None + return vec_typ.__args__[0] + + +def read_vec_len(vec_typ: Type[Vector[T,L]]) -> int: + assert vec_typ.__args__ is not None + return vec_typ.__args__[1].length + + +# SSZ list +# ----------------------------- +def read_list_elem_typ(list_typ: Type[List[T]]) -> T: + assert list_typ.__args__ is not None + return list_typ.__args__[0] + + +# SSZ bytesN +# ----------------------------- +class BytesN(Generic[L]): + pass + + +def read_bytesN_len(bytesN_typ: Type[BytesN[L]]) -> int: + assert bytesN_typ.__args__ is not None + return bytesN_typ.__args__[0].length + + +# SSZ integer types, with 0 computational overhead (NewType) +# ----------------------------- + +uint8 = NewType('uint8', int) +uint8.byte_len = 1 +uint16 = NewType('uint16', int) +uint16.byte_len = 2 +uint32 = NewType('uint32', int) +uint32.byte_len = 4 +uint64 = NewType('uint64', int) +uint64.byte_len = 8 +uint128 = NewType('uint128', int) +uint128.byte_len = 16 +uint256 = NewType('uint256', int) +uint256.byte_len = 32 +byte = NewType('byte', uint8) + + +# SSZ Container base class +# ----------------------------- + +# Note: importing ssz functionality locally, to avoid import loop + +class SSZContainer(object): + + def __init__(self, **kwargs): + from .ssz_impl import get_zero_value + for f, t in self.__annotations__.items(): + if f not in kwargs: + setattr(self, f, get_zero_value(t)) + else: + setattr(self, f, kwargs[f]) + + def serialize(self): + from .ssz_impl import serialize + return serialize(self, self.__class__) + + def hash_tree_root(self): + from .ssz_impl import hash_tree_root + return hash_tree_root(self, self.__class__) + + def signing_root(self): + from .ssz_impl import signing_root + return signing_root(self, self.__class__) + From 08faa86d706c31cd9690cb483d0df32f66696396 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sun, 12 May 2019 23:56:53 +0200 Subject: [PATCH 093/308] POC ssz types spec build + update spec defs, typing still needs work --- scripts/phase0/build_spec.py | 4 +- scripts/phase0/function_puller.py | 41 ++-- specs/core/0_beacon-chain.md | 230 +++++++++--------- .../pyspec/eth2spec/utils/ssz/__init__.py | 0 4 files changed, 141 insertions(+), 134 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/__init__.py diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index c8cd7348bb..b226194d65 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -12,11 +12,11 @@ def build_phase0_spec(sourcefile, outfile): NewType, Tuple, ) -from eth2spec.utils.minimal_ssz import ( - SSZType, +from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, ) +from eth2spec.utils.ssz.ssz_typing import * from eth2spec.utils.bls_stub import ( bls_aggregate_pubkeys, bls_verify, diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 750f195904..f94687344e 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -2,6 +2,22 @@ from typing import List +def translate_ssz_type_line(line: str) -> str: + if ':' not in line: + return line + start = line[:line.index(':')] + field_type = line[line.index(':')+2:] + if field_type.startswith('['): + if ',' in line: + # TODO: translate [Foobar, SOME_THING] to Vector[Foobar, SSZLen(SOME_THING)] cleanly. + # just brute it here + field_type = 'Vector[%s, SSLen(%s)]' % (field_type[1:field_type.index(',')], field_type[field_type.index(',')+2:len(field_type)-1]) + else: + field_type = 'List[%s]' % field_type[1:len(field_type)-1] + line = start + ': ' + field_type + return line + + def get_spec(file_name: str) -> List[str]: code_lines = [] pulling_from = None @@ -21,24 +37,23 @@ def get_spec(file_name: str) -> List[str]: else: if current_typedef is not None: assert code_lines[-1] == '}' - code_lines[-1] = '})' - current_typedef[-1] = '})' - type_defs.append((current_name, current_typedef)) + code_lines[-1] = '' + code_lines.append('') pulling_from = None current_typedef = None else: if pulling_from == linenum and line == '{': - code_lines.append('%s = SSZType({' % current_name) - current_typedef = ['global_vars["%s"] = SSZType({' % current_name] + code_lines.append('class %s(SSZContainer):' % current_name) + current_typedef = current_name + type_defs.append(current_name) elif pulling_from is not None: # Add some whitespace between functions if line[:3] == 'def': code_lines.append('') code_lines.append('') - code_lines.append(line) - # Remember type def lines if current_typedef is not None: - current_typedef.append(line) + line = translate_ssz_type_line(line) + code_lines.append(line) elif pulling_from is None and len(line) > 0 and line[0] == '|': row = line[1:].split('|') if len(row) >= 2: @@ -56,16 +71,8 @@ def get_spec(file_name: str) -> List[str]: code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890'))) # Build type-def re-initialization code_lines.append('\n') - code_lines.append('def init_SSZ_types():') - code_lines.append(' global_vars = globals()') - for ssz_type_name, ssz_type in type_defs: - code_lines.append('') - for type_line in ssz_type: - if len(type_line) > 0: - code_lines.append(' ' + type_line) - code_lines.append('\n') code_lines.append('ssz_types = [\n') - for (ssz_type_name, _) in type_defs: + for ssz_type_name in type_defs: code_lines.append(f' {ssz_type_name},\n') code_lines.append(']') code_lines.append('\n') diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index bbca333cd5..ff5a082efa 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -270,11 +270,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Previous fork version - 'previous_version': 'bytes4', + previous_version: bytes4 # Current fork version - 'current_version': 'bytes4', + current_version: bytes4 # Fork epoch number - 'epoch': 'uint64', + epoch: uint64 } ``` @@ -283,13 +283,13 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Shard number - 'shard': 'uint64', + shard: uint64 # Epoch number - 'epoch': 'uint64', + epoch: uint64 # Root of the previous crosslink - 'parent_root': 'bytes32', + parent_root: bytes32 # Root of the crosslinked shard data since the previous crosslink - 'data_root': 'bytes32', + data_root: bytes32 } ``` @@ -298,11 +298,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Root of the deposit tree - 'deposit_root': 'bytes32', + deposit_root: bytes32 # Total number of deposits - 'deposit_count': 'uint64', + deposit_count: uint64 # Block hash - 'block_hash': 'bytes32', + block_hash: bytes32 } ``` @@ -311,16 +311,16 @@ The types are defined topologically to aid in facilitating an executable version ```python { # LMD GHOST vote - 'beacon_block_root': 'bytes32', - + beacon_block_root: bytes32 + # FFG vote - 'source_epoch': 'uint64', - 'source_root': 'bytes32', - 'target_epoch': 'uint64', - 'target_root': 'bytes32', - + source_epoch: uint64 + source_root: bytes32 + target_epoch: uint64 + target_root: bytes32 + # Crosslink vote - 'crosslink': Crosslink, + crosslink: Crosslink } ``` @@ -329,9 +329,9 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Attestation data - 'data': AttestationData, + data: AttestationData # Custody bit - 'custody_bit': 'bool', + custody_bit: bool } ``` @@ -340,12 +340,12 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Validator indices - 'custody_bit_0_indices': ['uint64'], - 'custody_bit_1_indices': ['uint64'], + custody_bit_0_indices: [uint64] + custody_bit_1_indices: [uint64] # Attestation data - 'data': AttestationData, + data: AttestationData # Aggregate signature - 'signature': 'bytes96', + signature: bytes96 } ``` @@ -354,13 +354,13 @@ The types are defined topologically to aid in facilitating an executable version ```python { # BLS pubkey - 'pubkey': 'bytes48', + pubkey: bytes48 # Withdrawal credentials - 'withdrawal_credentials': 'bytes32', + withdrawal_credentials: bytes32 # Amount in Gwei - 'amount': 'uint64', + amount: uint64 # Container self-signature - 'signature': 'bytes96', + signature: bytes96 } ``` @@ -368,11 +368,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { - 'slot': 'uint64', - 'parent_root': 'bytes32', - 'state_root': 'bytes32', - 'body_root': 'bytes32', - 'signature': 'bytes96', + slot: uint64 + parent_root: bytes32 + state_root: bytes32 + body_root: bytes32 + signature: bytes96 } ``` #### `Validator` @@ -380,21 +380,21 @@ The types are defined topologically to aid in facilitating an executable version ```python { # BLS public key - 'pubkey': 'bytes48', + pubkey: bytes48 # Withdrawal credentials - 'withdrawal_credentials': 'bytes32', + withdrawal_credentials: bytes32 # Epoch when became eligible for activation - 'activation_eligibility_epoch': 'uint64', + activation_eligibility_epoch: uint64 # Epoch when validator activated - 'activation_epoch': 'uint64', + activation_epoch: uint64 # Epoch when validator exited - 'exit_epoch': 'uint64', + exit_epoch: uint64 # Epoch when validator is eligible to withdraw - 'withdrawable_epoch': 'uint64', + withdrawable_epoch: uint64 # Was the validator slashed - 'slashed': 'bool', + slashed: bool # Effective balance - 'effective_balance': 'uint64', + effective_balance: uint64 } ``` @@ -403,13 +403,13 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Attester aggregation bitfield - 'aggregation_bitfield': 'bytes', + aggregation_bitfield: bytes # Attestation data - 'data': AttestationData, + data: AttestationData # Inclusion delay - 'inclusion_delay': 'uint64', + inclusion_delay: uint64 # Proposer index - 'proposer_index': 'uint64', + proposer_index: uint64 } ``` @@ -418,9 +418,9 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Block roots - 'block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], + block_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] # State roots - 'state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], + state_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] } ``` @@ -431,11 +431,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Proposer index - 'proposer_index': 'uint64', + proposer_index: uint64 # First block header - 'header_1': BeaconBlockHeader, + header_1: BeaconBlockHeader # Second block header - 'header_2': BeaconBlockHeader, + header_2: BeaconBlockHeader } ``` @@ -444,9 +444,9 @@ The types are defined topologically to aid in facilitating an executable version ```python { # First attestation - 'attestation_1': IndexedAttestation, + attestation_1: IndexedAttestation # Second attestation - 'attestation_2': IndexedAttestation, + attestation_2: IndexedAttestation } ``` @@ -455,13 +455,13 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Attester aggregation bitfield - 'aggregation_bitfield': 'bytes', + aggregation_bitfield: bytes # Attestation data - 'data': AttestationData, + data: AttestationData # Custody bitfield - 'custody_bitfield': 'bytes', + custody_bitfield: bytes # BLS aggregate signature - 'signature': 'bytes96', + signature: bytes96 } ``` @@ -470,11 +470,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Branch in the deposit tree - 'proof': ['bytes32', DEPOSIT_CONTRACT_TREE_DEPTH], + proof: [bytes32, DEPOSIT_CONTRACT_TREE_DEPTH] # Index in the deposit tree - 'index': 'uint64', + index: uint64 # Data - 'data': DepositData, + data: DepositData } ``` @@ -483,11 +483,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Minimum epoch for processing exit - 'epoch': 'uint64', + epoch: uint64 # Index of the exiting validator - 'validator_index': 'uint64', + validator_index: uint64 # Validator signature - 'signature': 'bytes96', + signature: bytes96 } ``` @@ -496,19 +496,19 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Sender index - 'sender': 'uint64', + sender: uint64 # Recipient index - 'recipient': 'uint64', + recipient: uint64 # Amount in Gwei - 'amount': 'uint64', + amount: uint64 # Fee in Gwei for block proposer - 'fee': 'uint64', + fee: uint64 # Inclusion slot - 'slot': 'uint64', + slot: uint64 # Sender withdrawal pubkey - 'pubkey': 'bytes48', + pubkey: bytes48 # Sender signature - 'signature': 'bytes96', + signature: bytes96 } ``` @@ -518,15 +518,15 @@ The types are defined topologically to aid in facilitating an executable version ```python { - 'randao_reveal': 'bytes96', - 'eth1_data': Eth1Data, - 'graffiti': 'bytes32', - 'proposer_slashings': [ProposerSlashing], - 'attester_slashings': [AttesterSlashing], - 'attestations': [Attestation], - 'deposits': [Deposit], - 'voluntary_exits': [VoluntaryExit], - 'transfers': [Transfer], + randao_reveal: bytes96 + eth1_data: Eth1Data + graffiti: bytes32 + proposer_slashings: [ProposerSlashing] + attester_slashings: [AttesterSlashing] + attestations: [Attestation] + deposits: [Deposit] + voluntary_exits: [VoluntaryExit] + transfers: [Transfer] } ``` @@ -535,11 +535,11 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Header - 'slot': 'uint64', - 'parent_root': 'bytes32', - 'state_root': 'bytes32', - 'body': BeaconBlockBody, - 'signature': 'bytes96', + slot: uint64 + parent_root: bytes32 + state_root: bytes32 + body: BeaconBlockBody + signature: bytes96 } ``` @@ -550,45 +550,45 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Misc - 'slot': 'uint64', - 'genesis_time': 'uint64', - 'fork': Fork, # For versioning hard forks - + slot: uint64 + genesis_time: uint64 + fork: Fork # For versioning hard forks + # Validator registry - 'validator_registry': [Validator], - 'balances': ['uint64'], - + validator_registry: [Validator] + balances: [uint64] + # Randomness and committees - 'latest_randao_mixes': ['bytes32', LATEST_RANDAO_MIXES_LENGTH], - 'latest_start_shard': 'uint64', - + latest_randao_mixes: [bytes32, LATEST_RANDAO_MIXES_LENGTH] + latest_start_shard: uint64 + # Finality - 'previous_epoch_attestations': [PendingAttestation], - 'current_epoch_attestations': [PendingAttestation], - 'previous_justified_epoch': 'uint64', - 'current_justified_epoch': 'uint64', - 'previous_justified_root': 'bytes32', - 'current_justified_root': 'bytes32', - 'justification_bitfield': 'uint64', - 'finalized_epoch': 'uint64', - 'finalized_root': 'bytes32', - + previous_epoch_attestations: [PendingAttestation] + current_epoch_attestations: [PendingAttestation] + previous_justified_epoch: uint64 + current_justified_epoch: uint64 + previous_justified_root: bytes32 + current_justified_root: bytes32 + justification_bitfield: uint64 + finalized_epoch: uint64 + finalized_root: bytes32 + # Recent state - 'current_crosslinks': [Crosslink, SHARD_COUNT], - 'previous_crosslinks': [Crosslink, SHARD_COUNT], - 'latest_block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], - 'latest_state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], - 'latest_active_index_roots': ['bytes32', LATEST_ACTIVE_INDEX_ROOTS_LENGTH], + current_crosslinks: [Crosslink, SHARD_COUNT] + previous_crosslinks: [Crosslink, SHARD_COUNT] + latest_block_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] + latest_state_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] + latest_active_index_roots: [bytes32, LATEST_ACTIVE_INDEX_ROOTS_LENGTH] # Balances slashed at every withdrawal period - 'latest_slashed_balances': ['uint64', LATEST_SLASHED_EXIT_LENGTH], + latest_slashed_balances: [uint64, LATEST_SLASHED_EXIT_LENGTH] # `latest_block_header.state_root == ZERO_HASH` temporarily - 'latest_block_header': BeaconBlockHeader, - 'historical_roots': ['bytes32'], - + latest_block_header: BeaconBlockHeader + historical_roots: [bytes32] + # Ethereum 1.0 chain data - 'latest_eth1_data': Eth1Data, - 'eth1_data_votes': [Eth1Data], - 'deposit_index': 'uint64', + latest_eth1_data: Eth1Data + eth1_data_votes: [Eth1Data] + deposit_index: uint64 } ``` diff --git a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py b/test_libs/pyspec/eth2spec/utils/ssz/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From feb3b5ea0d021192c9b6759c20f10d2e21d6f711 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 13 May 2019 10:55:08 +1000 Subject: [PATCH 094/308] Correct typo --- specs/networking/libp2p-standardization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index 1f728fa544..d9e8565866 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -152,7 +152,7 @@ The [RPC Interface](./rpc-interface.md) is specified in this repository. **Note: This protocol is a placeholder and will be updated once the discv5 discovery protocol is added to this document** -#### Protocol Id: `/eth/serentiy/id/1.0.0` +#### Protocol Id: `/eth/serenity/id/1.0.0` The Identify protocol (defined in go - [identify-go](https://github.com/ipfs/go-ipfs/blob/master/core/commands/id.go) and rust [rust-identify](https://github.com/libp2p/rust-libp2p/blob/master/protocols/identify/src/lib.rs)) allows a node A to query another node B which information B knows about A. This also includes the addresses B is listening on. From 48ed25b2bdb1807b4170fbde4414ffd9baa08609 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 13 May 2019 15:07:15 +1000 Subject: [PATCH 095/308] Fleshed out a lot more of the API, nearly ready. - Added all the fields from BeaconBlock(Body) - Tagged all paths as 'Minimum for validator' - Removed BeaconNode and ValidatorClient conventions - Moved the basic non-object schema components to the top - Broke out common beacon block properties into the BeaconBlockCommon object - Fixed links to Eth2.0 spec --- specs/validator/beacon_node_oapi.yaml | 371 ++++++++++++++++++++++---- 1 file changed, 315 insertions(+), 56 deletions(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 0a583bdd6e..a2e4559f22 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -1,27 +1,31 @@ openapi: "3.0.2" info: - title: "Beacon Node API for Validator" - description: "A beacon node API for enabling a validator to perform its obligations on the Ethereum 2.0 phase 0 beacon chain." + title: "Minimal Beacon Node API for Validator" + description: "A minimal API specification for the beacon node, which enabling a validator to connect and perform its obligations on the Ethereum 2.0 phase 0 beacon chain." version: "0.1" paths: /node/version: get: + tags: + - Minimum for validator summary: "Get version string of the running beacon node." - description: "Requests that the BeaconNode identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field." + description: "Requests that the beacon node identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field." responses: 200: description: Request successful content: application/json: schema: - $ref: '#/components/schemas/client_version' + $ref: '#/components/schemas/version' 500: - $ref: '#/components/responses/InternalError' - + $ref: '#/components/responses/InternalError' + /node/genesis_time: get: + tags: + - Minimum for validator summary: "Get the genesis_time parameter from beacon node configuration." - description: "Requests the genesis_time parameter from the BeaconNode, which should be consistent across all BeaconNodes that follow the same beacon chain." + description: "Requests the genesis_time parameter from the beacon node, which should be consistent across all beacon nodes that follow the same beacon chain." responses: 200: description: Request successful @@ -31,10 +35,12 @@ paths: $ref: '#/components/schemas/genesis_time' 500: $ref: '#/components/responses/InternalError' - + /node/syncing: get: + tags: + - Minimum for validator summary: "Poll to see if the the beacon node is syncing." description: "Requests the beacon node to describe if it's currently syncing or not, and if it is, what block it is up to. This is modelled after the Eth1.0 JSON-RPC eth_syncing call.." responses: @@ -52,13 +58,15 @@ paths: $ref: '#/components/schemas/SyncingStatus' 500: $ref: '#/components/responses/InternalError' - - + + /validator/duties: get: + tags: + - Minimum for validator summary: "Get validator duties for the requested validators." - description: "Requests the BeaconNode to provide a set of _duties_, which are actions that should be performed by ValidatorClients. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected BeaconNode is properly synchronised." - parameters: + description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected beacon node is properly synchronised." + parameters: - name: validator_pubkeys in: query required: true @@ -83,14 +91,16 @@ paths: $ref: '#/components/responses/InternalError' 503: $ref: '#/components/responses/CurrentlySyncing' - - - + + + /validator/block: get: + tags: + - Minimum for validator summary: "Produce a new block, without signature." - description: "Requests a BeaconNode to produce a valid block, which can then be signed by a ValidatorClient." - parameters: + description: "Requests a beacon node to produce a valid block, which can then be signed by a validator." + parameters: - name: slot in: query required: true @@ -101,7 +111,7 @@ paths: - name: randao_reveal in: query required: true - description: "The ValidatorClient's randao reveal value." + description: "The validator's randao reveal value." schema: type: string format: byte @@ -119,13 +129,15 @@ paths: 503: $ref: '#/components/responses/CurrentlySyncing' post: + tags: + - Minimum for validator summary: "Publish a signed block" - description: "Instructs the BeaconNode to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." - parameters: + description: "Instructs the beacon node to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." + parameters: - name: beacon_block in: query required: true - description: "The BeaconBlock object, as sent from the BeaconNode originally, but now with the signature field completed." + description: "The `BeaconBlock` object, as sent from the beacon node originally, but now with the signature field completed." schema: $ref: '#/components/schemas/BeaconBlock' responses: @@ -137,13 +149,15 @@ paths: $ref: '#/components/responses/InternalError' 503: $ref: '#/components/responses/CurrentlySyncing' - - + + /validator/attestation: get: + tags: + - Minimum for validator summary: "Produce an attestation, without signature." - description: "Requests that the BeaconNode produce an IndexedAttestation, with a blank signature field, which the ValidatorClient will then sign." - parameters: + description: "Requests that the beacon node produce an IndexedAttestation, with a blank signature field, which the validator will then sign." + parameters: - name: slot in: query required: true @@ -169,15 +183,17 @@ paths: $ref: '#/components/responses/InternalError' 503: $ref: '#/components/responses/CurrentlySyncing' - + post: + tags: + - Minimum for validator summary: "Published a signed attestation." - description: "Instructs the BeaconNode to publish a newly signed IndexedAttestation object, to be incorporated into the beacon chain." + description: "Instructs the beacon node to publish a newly signed IndexedAttestation object, to be incorporated into the beacon chain." parameters: - name: attestation in: query required: true - description: "An IndexedAttestation structure, as originally provided by the BeaconNode, but now with the signature field completed." + description: "An `IndexedAttestation` structure, as originally provided by the beacon node, but now with the signature field completed." schema: $ref: '#/components/schemas/IndexedAttestation' responses: @@ -192,6 +208,30 @@ paths: components: schemas: + pubkey: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{96}$" + description: "The validator's BLS public key, uniquely identifying them. _48-bytes, hex encoded with 0x prefix, case insensitive._" + example: "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc" + + version: + type: string + description: "A string which uniquely identifies the client implementation and its version; similar to [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3)." + example: "Lighthouse / v0.1.5 (Linux x86_64)" + + genesis_time: + type: integer + format: uint64 + description: "The genesis_time configured for the beacon node, which is the time the Eth1.0 validator deposit smart contract has enough ETH staked (i.e. Eth2.0 begins)." + example: 1557716289 + + merkle_root: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "A 32 byte merkle root, used in all kinds of scenarios." + ValidatorDuty: type: object properties: @@ -214,6 +254,7 @@ components: format: uint64 nullable: true description: "The slot in which a validator must propose a block, or `null` if block production is not required." + SyncingStatus: type: object nullable: true @@ -230,12 +271,248 @@ components: type: integer format: uint64 description: "The estimated highest block, or current target block number." + + BeaconBlock: + allOf: + - $ref: '#/components/schemas/BeaconBlockCommon' + - type: object + description: "The [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblock) object from the Eth2.0 spec." + properties: + body: + $ref: '#/components/schemas/BeaconBlockBody' + + + BeaconBlockHeader: + allOf: + - $ref: '#/components/schemas/BeaconBlockCommon' + - type: object + properties: + body_root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The tree hash merkle root of the `BeaconBlockBody` for the `BeaconBlock`" + + + + + BeaconBlockBody: + type: object + description: "The [`BeaconBlockBody`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblockbody) object from the Eth2.0 spec." + properties: + randao_reveal: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{192}$" + description: "The RanDAO reveal value provided by the validator." + eth1_data: + title: Eth1Data + type: object + description: "The [`Eth1Data`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#eth1data) object from the Eth2.0 spec." + properties: + deposit_root: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Root of the deposit tree." + deposit_count: + type: integer + format: uint64 + description: "Total number of deposits." + block_hash: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Ethereum 1.x block hash" + graffiti: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + proposer_slashings: + type: array + items: + title: ProposerSlashings + type: object + description: "The [`ProposerSlashing`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#proposerslashing) object from the Eth2.0 spec." + properties: + proposer_index: + type: integer + format: uint64 + description: "The index of the proposer to be slashed." + header_1: + $ref: '#/components/schemas/BeaconBlockHeader' + header_2: + $ref: '#/components/schemas/BeaconBlockHeader' + attester_slashings: + type: array + items: + title: AttesterSlashings + type: object + description: "The [`AttesterSlashing`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attesterslashing) object from the Eth2.0 spec." + properties: + attestation_1: + $ref: '#/components/schemas/IndexedAttestation' + attestation_2: + $ref: '#/components/schemas/IndexedAttestation' + attestations: + type: array + items: + title: Attestation + type: object + description: "The [`Attestation`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attestation) object from the Eth2.0 spec." + properties: + aggregation_bitfield: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]+$" + description: "Attester aggregation bitfield." + custody_bitfield: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]+$" + description: "Custody bitfield" + signature: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{192}$" + description: "BLS aggregate signature." + data: + $ref: '#/components/schemas/AttestationData' + deposits: + type: array + items: + title: Deposit + type: object + description: "The [`Deposit`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#deposit) object from the Eth2.0 spec." + properties: + proof: + type: array + description: "Branch in the deposit tree." + items: + oneOf: + - type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + - type: integer + format: uint64 + enum: [32] + example: 32 + description: "The DEPOSIT_CONTRACT_TREE_DEPTH value." + minItems: 2 + maxItems: 2 + index: + type: integer + format: uint64 + description: "Index in the deposit tree." + data: + title: DepositData + type: object + description: "The [`DepositData`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#depositdata) object from the Eth2.0 spec." + properties: + pubkey: + $ref: '#/components/schemas/pubkey' + withdrawal_credentials: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The withdrawal credentials." + amount: + type: integer + format: uint64 + description: "Amount in Gwei." + signature: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{192}$" + description: "Container self-signature." + + voluntary_exits: + type: array + items: + title: VoluntaryExit + type: object + description: "The [`VoluntaryExit`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#voluntaryexit) object from the Eth2.0 spec." + properties: + epoch: + type: integer + format: uint64 + description: "Minimum epoch for processing exit." + validator_index: + type: integer + format: uint64 + description: "Index of the exiting validator." + signature: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{192}$" + description: "Validator signature." + transfers: + type: array + items: + title: Transfer + type: object + description: "The [`Transfer`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#transfer) object from the Eth2.0 spec." + properties: + sender: + type: integer + format: uint64 + description: "Sender index." + recipient: + type: integer + format: uint64 + description: "Recipient index." + amount: + type: integer + format: uint64 + description: "Amount in Gwei" + fee: + type: integer + format: uint64 + description: "Fee in Gwei for block producer" + slot: + type: integer + format: uint64 + description: "Inclusion slot" + pubkey: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{96}$" + description: "Sender withdrawal public key" + signature: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{192}$" + description: "Sender signature" + + BeaconBlockCommon: + # An object to collect the common fields between the BeaconBlockHeader and the BeaconBlock objects type: object - description: "The [BeaconBlock](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblock) from the Eth2.0 spec." + properties: + slot: + type: integer + format: uint64 + description: "The slot to which this block corresponds." + parent_root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The signing merkle root of the parent `BeaconBlock`" + state_root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The tree hash merkle root of the `BeaconState` for the `BeaconBlock`" + signature: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{192}$" + example: "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + description: "The BLS signature of the `BeaconBlock` made by the validator of the block" + Fork: type: object - description: "The [Fork](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#Fork) from the Eth2.0 spec." + description: "The [`Fork`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#Fork) object from the Eth2.0 spec." properties: previous_version: type: string @@ -251,9 +528,10 @@ components: type: integer format: uint64 description: "Fork epoch number" + IndexedAttestation: type: object - description: "The [IndexedAttestation](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#indexedattestation) from the Eth2.0 spec." + description: "The [`IndexedAttestation`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#indexedattestation) object from the Eth2.0 spec." properties: custody_bit_0_indicies: type: array @@ -269,10 +547,10 @@ components: format: uint64 data: $ref: '#/components/schemas/AttestationData' - - AttestationData: + + AttestationData: type: object - description: "The [AttestationData](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attestationdata) from the Eth2.0 spec." + description: "The [`AttestationData`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attestationdata) object from the Eth2.0 spec." properties: beacon_block_root: $ref: '#/components/schemas/merkle_root' @@ -290,9 +568,10 @@ components: $ref: '#/components/schemas/merkle_root' crosslink: $ref: '#/components/schemas/CrossLink' + CrossLink: type: object - description: "The [Crosslink](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) from the Eth2.0 spec." + description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec." properties: shard: type: integer @@ -306,27 +585,7 @@ components: $ref: '#/components/schemas/merkle_root' data_root: $ref: '#/components/schemas/merkle_root' - - pubkey: - type: string - format: byte - pattern: "^0x[a-fA-F0-9]{96}$" - description: "The validator's BLS public key, uniquely identifying them." - client_version: - type: string - description: "A string which uniquely identifies the client implementation and its version; similar to [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3)." - example: "Lighthouse / v0.1.5 (Linux x86_64)" - genesis_time: - type: integer - format: uint64 - description: "The genesis_time configured for the beacon node, which is the time the Eth1.0 validator deposit smart contract has enough ETH staked (i.e. Eth2.0 begins)." - merkle_root: - type: string - format: byte - pattern: "^0x[a-fA-F0-9]{64}$" - description: "A 32 byte merkle root, used in all kinds of scenarios." - - + responses: Success: description: Request successful From d10baf1dcea19b29fc9e755dc322bb149a7474d1 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 13 May 2019 15:18:56 +1000 Subject: [PATCH 096/308] Added optional path, , renamed tags, and fixed up whitespace issues. --- specs/validator/beacon_node_oapi.yaml | 51 ++++++++++++++++----------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index a2e4559f22..01eef87a4e 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -7,7 +7,7 @@ paths: /node/version: get: tags: - - Minimum for validator + - Necessary for validator summary: "Get version string of the running beacon node." description: "Requests that the beacon node identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field." responses: @@ -23,7 +23,7 @@ paths: /node/genesis_time: get: tags: - - Minimum for validator + - Necessary for validator summary: "Get the genesis_time parameter from beacon node configuration." description: "Requests the genesis_time parameter from the beacon node, which should be consistent across all beacon nodes that follow the same beacon chain." responses: @@ -36,11 +36,10 @@ paths: 500: $ref: '#/components/responses/InternalError' - /node/syncing: get: tags: - - Minimum for validator + - Necessary for validator summary: "Poll to see if the the beacon node is syncing." description: "Requests the beacon node to describe if it's currently syncing or not, and if it is, what block it is up to. This is modelled after the Eth1.0 JSON-RPC eth_syncing call.." responses: @@ -59,11 +58,10 @@ paths: 500: $ref: '#/components/responses/InternalError' - /validator/duties: get: tags: - - Minimum for validator + - Necessary for validator summary: "Get validator duties for the requested validators." description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected beacon node is properly synchronised." parameters: @@ -92,12 +90,10 @@ paths: 503: $ref: '#/components/responses/CurrentlySyncing' - - /validator/block: get: tags: - - Minimum for validator + - Necessary for validator summary: "Produce a new block, without signature." description: "Requests a beacon node to produce a valid block, which can then be signed by a validator." parameters: @@ -130,7 +126,7 @@ paths: $ref: '#/components/responses/CurrentlySyncing' post: tags: - - Minimum for validator + - Necessary for validator summary: "Publish a signed block" description: "Instructs the beacon node to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." parameters: @@ -150,11 +146,10 @@ paths: 503: $ref: '#/components/responses/CurrentlySyncing' - /validator/attestation: get: tags: - - Minimum for validator + - Necessary for validator summary: "Produce an attestation, without signature." description: "Requests that the beacon node produce an IndexedAttestation, with a blank signature field, which the validator will then sign." parameters: @@ -183,10 +178,9 @@ paths: $ref: '#/components/responses/InternalError' 503: $ref: '#/components/responses/CurrentlySyncing' - post: tags: - - Minimum for validator + - Necessary for validator summary: "Published a signed attestation." description: "Instructs the beacon node to publish a newly signed IndexedAttestation object, to be incorporated into the beacon chain." parameters: @@ -206,6 +200,29 @@ paths: 503: $ref: '#/components/responses/CurrentlySyncing' + /node/fork: + get: + tags: + - Optional + summary: "Get fork information from running beacon node." + description: "Requests the beacon node to provide which fork version it is currently on." + responses: + 200: + description: Request successful + content: + application/json: + schema: + type: object + properties: + fork: + $ref: '#/components/schemas/Fork' + chain_id: + type: integer + format: uint64 + description: "Sometimes called the network id, this number discerns the active chain for the BeaconNode. Analagous to Eth1.0 JSON-RPC net_version." + 500: + $ref: '#/components/responses/InternalError' + components: schemas: pubkey: @@ -272,7 +289,6 @@ components: format: uint64 description: "The estimated highest block, or current target block number." - BeaconBlock: allOf: - $ref: '#/components/schemas/BeaconBlockCommon' @@ -282,7 +298,6 @@ components: body: $ref: '#/components/schemas/BeaconBlockBody' - BeaconBlockHeader: allOf: - $ref: '#/components/schemas/BeaconBlockCommon' @@ -294,9 +309,6 @@ components: pattern: "^0x[a-fA-F0-9]{64}$" description: "The tree hash merkle root of the `BeaconBlockBody` for the `BeaconBlock`" - - - BeaconBlockBody: type: object description: "The [`BeaconBlockBody`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblockbody) object from the Eth2.0 spec." @@ -426,7 +438,6 @@ components: format: byte pattern: "^0x[a-fA-F0-9]{192}$" description: "Container self-signature." - voluntary_exits: type: array items: From 2035aea0b8e5123536985bf76b176cb33232a6a9 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 13 May 2019 15:52:34 +1000 Subject: [PATCH 097/308] Formatting clean up. - Moved /node/fork up with other node endpoints - Added descriptions and ordering to tags - Removed common merkle_root schema, to be more specific in descriptions. - Moved BeaconBlockCommon next to appropriate schemas. - Lots of small grammar improvements, full stops at end of descriptions. --- specs/validator/beacon_node_oapi.yaml | 206 +++++++++++++------------- 1 file changed, 104 insertions(+), 102 deletions(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 01eef87a4e..8d72679536 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -3,6 +3,11 @@ info: title: "Minimal Beacon Node API for Validator" description: "A minimal API specification for the beacon node, which enabling a validator to connect and perform its obligations on the Ethereum 2.0 phase 0 beacon chain." version: "0.1" +tags: + - name: Necessary for validator + description: The minimal set of endpoints to enable a working validator implementation. + - name: Optional + description: Extra endpoints which are nice-to-haves. paths: /node/version: get: @@ -19,7 +24,6 @@ paths: $ref: '#/components/schemas/version' 500: $ref: '#/components/responses/InternalError' - /node/genesis_time: get: tags: @@ -57,6 +61,28 @@ paths: $ref: '#/components/schemas/SyncingStatus' 500: $ref: '#/components/responses/InternalError' + /node/fork: + get: + tags: + - Optional + summary: "Get fork information from running beacon node." + description: "Requests the beacon node to provide which fork version it is currently on." + responses: + 200: + description: Request successful + content: + application/json: + schema: + type: object + properties: + fork: + $ref: '#/components/schemas/Fork' + chain_id: + type: integer + format: uint64 + description: "Sometimes called the network id, this number discerns the active chain for the BeaconNode. Analagous to Eth1.0 JSON-RPC net_version." + 500: + $ref: '#/components/responses/InternalError' /validator/duties: get: @@ -127,7 +153,7 @@ paths: post: tags: - Necessary for validator - summary: "Publish a signed block" + summary: "Publish a signed block." description: "Instructs the beacon node to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." parameters: - name: beacon_block @@ -200,29 +226,6 @@ paths: 503: $ref: '#/components/responses/CurrentlySyncing' - /node/fork: - get: - tags: - - Optional - summary: "Get fork information from running beacon node." - description: "Requests the beacon node to provide which fork version it is currently on." - responses: - 200: - description: Request successful - content: - application/json: - schema: - type: object - properties: - fork: - $ref: '#/components/schemas/Fork' - chain_id: - type: integer - format: uint64 - description: "Sometimes called the network id, this number discerns the active chain for the BeaconNode. Analagous to Eth1.0 JSON-RPC net_version." - 500: - $ref: '#/components/responses/InternalError' - components: schemas: pubkey: @@ -231,24 +234,15 @@ components: pattern: "^0x[a-fA-F0-9]{96}$" description: "The validator's BLS public key, uniquely identifying them. _48-bytes, hex encoded with 0x prefix, case insensitive._" example: "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc" - version: type: string description: "A string which uniquely identifies the client implementation and its version; similar to [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3)." example: "Lighthouse / v0.1.5 (Linux x86_64)" - genesis_time: type: integer format: uint64 description: "The genesis_time configured for the beacon node, which is the time the Eth1.0 validator deposit smart contract has enough ETH staked (i.e. Eth2.0 begins)." example: 1557716289 - - merkle_root: - type: string - format: byte - pattern: "^0x[a-fA-F0-9]{64}$" - description: "A 32 byte merkle root, used in all kinds of scenarios." - ValidatorDuty: type: object properties: @@ -271,7 +265,6 @@ components: format: uint64 nullable: true description: "The slot in which a validator must propose a block, or `null` if block production is not required." - SyncingStatus: type: object nullable: true @@ -290,15 +283,15 @@ components: description: "The estimated highest block, or current target block number." BeaconBlock: + description: "The [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblock) object from the Eth2.0 spec." allOf: - $ref: '#/components/schemas/BeaconBlockCommon' - type: object - description: "The [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblock) object from the Eth2.0 spec." properties: body: $ref: '#/components/schemas/BeaconBlockBody' - BeaconBlockHeader: + description: "The [`BeaconBlockHeader`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblockheader) object from the Eth2.0 spec." allOf: - $ref: '#/components/schemas/BeaconBlockCommon' - type: object @@ -308,7 +301,30 @@ components: format: bytes pattern: "^0x[a-fA-F0-9]{64}$" description: "The tree hash merkle root of the `BeaconBlockBody` for the `BeaconBlock`" - + BeaconBlockCommon: + # An abstract object to collect the common fields between the BeaconBlockHeader and the BeaconBlock objects + type: object + properties: + slot: + type: integer + format: uint64 + description: "The slot to which this block corresponds." + parent_root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The signing merkle root of the parent `BeaconBlock`." + state_root: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{64}$" + description: "The tree hash merkle root of the `BeaconState` for the `BeaconBlock`." + signature: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{192}$" + example: "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + description: "The BLS signature of the `BeaconBlock` made by the validator of the block." BeaconBlockBody: type: object description: "The [`BeaconBlockBody`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblockbody) object from the Eth2.0 spec." @@ -336,7 +352,7 @@ components: type: string format: byte pattern: "^0x[a-fA-F0-9]{64}$" - description: "Ethereum 1.x block hash" + description: "Ethereum 1.x block hash." graffiti: type: string format: byte @@ -383,7 +399,7 @@ components: type: string format: byte pattern: "^0x[a-fA-F0-9]+$" - description: "Custody bitfield" + description: "Custody bitfield." signature: type: string format: byte @@ -476,50 +492,25 @@ components: amount: type: integer format: uint64 - description: "Amount in Gwei" + description: "Amount in Gwei." fee: type: integer format: uint64 - description: "Fee in Gwei for block producer" + description: "Fee in Gwei for block producer." slot: type: integer format: uint64 - description: "Inclusion slot" + description: "Inclusion slot." pubkey: type: string format: byte pattern: "^0x[a-fA-F0-9]{96}$" - description: "Sender withdrawal public key" + description: "Sender withdrawal public key." signature: type: string format: byte pattern: "^0x[a-fA-F0-9]{192}$" - description: "Sender signature" - - BeaconBlockCommon: - # An object to collect the common fields between the BeaconBlockHeader and the BeaconBlock objects - type: object - properties: - slot: - type: integer - format: uint64 - description: "The slot to which this block corresponds." - parent_root: - type: string - format: bytes - pattern: "^0x[a-fA-F0-9]{64}$" - description: "The signing merkle root of the parent `BeaconBlock`" - state_root: - type: string - format: bytes - pattern: "^0x[a-fA-F0-9]{64}$" - description: "The tree hash merkle root of the `BeaconState` for the `BeaconBlock`" - signature: - type: string - format: bytes - pattern: "^0x[a-fA-F0-9]{192}$" - example: "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" - description: "The BLS signature of the `BeaconBlock` made by the validator of the block" + description: "Sender signature." Fork: type: object @@ -529,17 +520,16 @@ components: type: string format: byte pattern: "^0x[a-fA-F0-9]{8}$" - description: "Previous fork version" + description: "Previous fork version." current_version: type: string format: byte pattern: "^0x[a-fA-F0-9]{8}$" - description: "Current fork version" + description: "Current fork version." epoch: type: integer format: uint64 - description: "Fork epoch number" - + description: "Fork epoch number." IndexedAttestation: type: object description: "The [`IndexedAttestation`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#indexedattestation) object from the Eth2.0 spec." @@ -558,51 +548,63 @@ components: format: uint64 data: $ref: '#/components/schemas/AttestationData' - AttestationData: type: object description: "The [`AttestationData`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attestationdata) object from the Eth2.0 spec." properties: beacon_block_root: - $ref: '#/components/schemas/merkle_root' + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "LMD GHOST vote." source_epoch: type: integer format: uint64 - description: "Source epoch from FFG vote" + description: "Source epoch from FFG vote." source_root: - $ref: '#/components/schemas/merkle_root' + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Source root from FFG vote." target_epoch: type: integer format: uint64 - description: "Target epoch from FFG vote" + description: "Target epoch from FFG vote." target_root: - $ref: '#/components/schemas/merkle_root' + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Target root from FFG vote." crosslink: - $ref: '#/components/schemas/CrossLink' - - CrossLink: - type: object - description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec." - properties: - shard: - type: integer - format: uint64 - description: "The shard number" - epoch: - type: integer - format: uint64 - description: "The epoch number" - parent_root: - $ref: '#/components/schemas/merkle_root' - data_root: - $ref: '#/components/schemas/merkle_root' + title: CrossLink + type: object + description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec." + properties: + shard: + type: integer + format: uint64 + description: "The shard number." + epoch: + type: integer + format: uint64 + description: "The epoch number." + parent_root: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Root of the previous crosslink." + data_root: + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + description: "Root of the crosslinked shard data since the previous crosslink." responses: Success: - description: Request successful + description: "Request successful." InvalidRequest: - description: Invalid request syntax + description: "Invalid request syntax." InternalError: - description: Beacon node internal error + description: "Beacon node internal error." CurrentlySyncing: - description: Beacon node is currently syncing, try again later + description: "Beacon node is currently syncing, try again later." From 0b2c7acdb32a0daf81cf3f54e8ab35679f7083a7 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 13 May 2019 16:25:22 +1000 Subject: [PATCH 098/308] Fixed up markdown. - Removed TOC - Removed all the old spec stuff - Uploaded spec to SwaggerHub and provided a link to it. - Added a 'license' section to the API description. --- .../validator/0_beacon-node-validator-api.md | 216 +----------------- specs/validator/beacon_node_oapi.yaml | 5 +- 2 files changed, 7 insertions(+), 214 deletions(-) diff --git a/specs/validator/0_beacon-node-validator-api.md b/specs/validator/0_beacon-node-validator-api.md index 342e7f934a..cf763f7781 100644 --- a/specs/validator/0_beacon-node-validator-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -2,12 +2,6 @@ __NOTICE__: This document is a work-in-progress for researchers and implementers. This is an accompanying document to [Ethereum 2.0 Phase 0 -- Honest Validator](0_beacon-chain-validator.md) that describes an API exposed by the beacon node, which enables the validator client to participate in the Ethereum 2.0 protocol. -## Table of Contents - - - - - ## Outline This document outlines a minimal application programming interface (API) which is exposed by a `BeaconNode` for use by a `ValidatorClient` which aims to facilitate [_phase 0_](../../README.md#phase-0) of Ethereum 2.0. @@ -28,211 +22,7 @@ This specification is derived from a proposal and discussion on Issues [#1011](h ## Specification -### Entities -The following are the two entities that participate in this protocol: - - **`BeaconNode`**: - A beacon node instance, run with a `--rpc` flag to enable the RPC interface. Runs stand-alone. - - - **`ValidatorClient`**: -A validator client instance, which must connect to at least one instance of `BeaconNode`. - - -### Endpoints -This section summarises API endpoints which are published by an instance of `BeaconNode`, for the exclusive use of `ValidatorClient` implementations. - -This proposal is a minimum set of messages necessary to enable effective communication, without any extra features. Anything extra is beyond the scope of this document. - -#### Summary Table -| Name | Type | Parameters | Returns | -| -------- | --- | ----- | ----- | -| [`get_client_version`](#get_client_version) | GET | N/A | `client_version` | -| [`get_genesis_time`](#get_genesis_time) | GET | N/A | `genesis_time` | -| [`get_syncing_status`](#get_syncing_status) | GET | N/A | `syncing_status` | -| [`get_duties`](#get_duties) | GET | `validator_pubkeys` | `syncing_status`, `current_version`, [`ValidatorDuty`]| -| [`produce_block`](#produce_block) | GET | `slot`, `randao_reveal` | `beacon_block` | -| [`publish_block`](#publish_block) | POST | `beacon_block` | N/A | -| [`produce_attestation`](#produce_attestation) | GET | `slot`, `shard` | `indexed_attestation` | -| [`publish_attestation`](#publish_attestation) | POST | `indexed_attestation` | N/A | Publishes the IndexedAttestation after having been signed by the ValidatorClient | - -#### Status Codes -For each of these endpoints the underlying transport protocol should provide status codes. Assuming this will be based on HTTP, one of the following standard status codes will always be included as part of a response: - -| Code | Meaning | -| --- | --- | -| `200` | The API call succeeded. | -| `40X` | The request was malformed. | -| `500` | The `BeaconNode` cannot complete the request due to an internal error. | -| `503` | The `BeaconNode` is currently syncing, try again later. _A call can be made to `get_syncing_status` to in order to find out how much has been synchronised._ | - -#### `get_client_version` -Requests that the `BeaconNode` identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field. - - - **Parameters**: N/A - - **Returns**: - - | Name | Type | Description | - | --- | --- | --- | - | `client_version` | bytes32 | An ASCII-encoded hex string which uniquely defines the implementation of the `BeaconNode` and its current software version. | - - **Note**: _Unlike most other endpoints, `get_client_version` does not return an error `503` while the `BeaconNode` is syncing, but instead returns status code `200`._ - - -#### `get_genesis_time` - Requests the `genesis_time` parameter from the `BeaconNode`, which should be consistent across all `BeaconNodes` that follow the same beacon chain. - - - **Parameters**: N/A - - **Returns**: - - | Name | Type | Description | - | --- | --- | --- | - | `genesis_time` | uint64 | The [`genesis_time`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#on-genesis), which is a fairly static configuration option for the `BeaconNode`. | - - **Note**: _Unlike most other endpoints, `get_genesis_time` does not return an error `503` while the `BeaconNode` is syncing, but instead returns status code `200`._ - - -#### `get_syncing_status` - Requests the `BeaconNode` to describe if it's currently syncing or not, and if it is, what block it is up to. This is modelled after the Eth1.0 JSON-RPC [`eth_syncing`](https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_syncing) call. - - **Parameters**: N/A - - **Returns**: - - | Name | Type | Description | - | --- | --- | --- | - | `syncing` | `false` OR `SyncingStatus` | Either `false` if the node is not syncing, or a [`SyncingStatus`](#SyncingStatus) object if it is. | - - **Note**: _Unlike most other endpoints, `get_syncing_status` does not return an error `503` while the `BeaconNode` is syncing, but instead returns status code `200` with the `SyncingStatus` object._ - - -#### `get_duties` - Requests the BeaconNode to provide a set of “duties”, which are actions that should be performed by ValidatorClients. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected `BeaconNode` is properly synchronised. - - - **Parameters**: - - | Name | Type | Description | - | --- | --- | --- | - | `validator_pubkeys` | [bytes48] | A list of unique validator public keys, where each item is a `0x` encoded hex string. | - - - **Returns**: - - | Name | Type | Description | - | --- | --- | --- | - | `current_version` | bytes4 | The `current_version`, as described by the current [`Fork`](#Fork). | - | `validator_duties` | [`ValidatorDuty`] | A list where each item is a custom [`ValidatorDuty`](#ValidatorDuty) object. | - - - #### `produce_block` - Requests a `BeaconNode` to produce a valid block, which can then be signed by a ValidatorClient. - - - **Parameters**: - - | Name | Type | Description | - | --- | --- | --- | - | `slot` | uint64 | The slot for which the block should be proposed. | - | `randao_reveal` | bytes | The ValidatorClient's randao reveal value. | - - - **Returns**: - - | Name | Type | Description | - | --- | --- | --- | - | `beacon_block` | `BeaconBlock` | A proposed [`BeaconBlock`](#BeaconBlock) object, but with the `signature` field left blank. - - - #### `publish_block` - Instructs the `BeaconNode` to publish a newly signed beacon block to the beacon network, to be included in the beacon chain. - - **Parameters**: - - | Name | Type | Description | - | --- | --- | --- | - | `beacon_block` | `BeaconBlock` | The [`BeaconBlock`](#BeaconBlock) object, as sent from the `BeaconNode` originally, but now with the `signature` field completed. - - - **Returns**: N/A - - - #### `produce_attestation` - Requests that the `BeaconNode` produce an `IndexedAttestation`, with a blank `signature` field, which the `ValidatorClient` will then sign. - - - **Parameters**: - - | Name | Type | Description | - | --- | --- | --- | - | `slot` | uint64 | The slot for which the attestation should be proposed. | - | `shard` | uint64 | The shard number for which the attestation is to be proposed. | - - - **Returns**: - - | Name | Type | Description | - | --- | --- | --- | - | `indexed_attestation` | `IndexedAttestation` | An [`IndexedAttestation`](#IndexedAttestation) structure with the `signature` field left blank. | - - #### `publish_attestation` - Instructs the `BeaconNode` to publish a newly signed `IndexedAttestation` object, to be incorporated into the beacon chain. - - - **Parameters**: - - | Name | Type | Description | - | --- | --- | --- | - | `indexed_attestation` | `IndexedAttestation` | An [`IndexedAttestation`](#IndexedAttestation) structure, as originally provided by the `BeaconNode`, but now with the `signature` field completed. | - - **Returns**: N/A - - - - ----- - -### Data Structures -Two new data objects are proposed for the sake of implementation, and several other data objects from the Eth2.0 specs are referenced. - -The `bytes` data types are encoded hex strings, with `0x` preceeding them. `uint64` are decimal encoded integers, and `None` may be `null`, which is distinct from `0`. - -#### `ValidatorDuty` -```asm -{ - - # The validator's public key, uniquely identifying them - 'validator_pubkey': 'bytes48', - # The index of the validator in the committee - 'committee_index': 'uint64', - # The slot at which the validator must attest. - 'attestation_slot': 'uint64', - # The shard in which the validator must attest - 'attestation_shard': 'uint64', - # The slot in which a validator must propose a block. This field can also be None. - 'block_production_slot': 'uint64' or None -} -``` - -#### `SyncingStatus` -As described by the [Eth1.0 JSON-RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_syncing).: -```asm -{ - # The block at which syncing started (will only be reset, after the sync reached his head) - 'startingBlock': 'uint64', - # The current block - 'currentBlock': 'uint64', - # The estimated highest block, or current target block number - 'highestBlock': 'uint64' -} -``` - -#### `Fork` -As described by [Fork](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#Fork) in the Eth2.0 specs. - -#### `BeaconBlock` -As described by [BeaconBlock](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#BeaconBlock) in the Eth2.0 specs. - -#### `IndexedAttestation` -As described by [IndexedAttestation](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#IndexedAttestation) in the Eth2.0 specs. - - - - -## Optional Extras +The API specification has been written in [OpenAPI 3.0](https://swagger.io/docs/specification/about/) and is provided in the [beacon_node_oapi.yaml](beacon_node_oapi.yaml) file alongside this document. -#### Endpoint: `get_fork` - Requests the `BeaconNode` to provide which fork version it is currently on. - - **Parameters**: N/A - - **Returns**: - - | Name | Type | Description | - | --- | --- | --- | - | `fork` | [`Fork`](#Fork) | Provides the current version information for the fork which the `BeaconNode` is currently following. | - | `chain_id` | uint64 | Sometimes called the network id, this number discerns the active chain for the `BeaconNode`. Analagous to Eth1.0 JSON-RPC [`net_version`](https://github.com/ethereum/wiki/wiki/JSON-RPC#net_version). | - +For convenience, this specification has been uploaded to [SwaggerHub](https://swagger.io/tools/swaggerhub/) at the following URL: +[https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator/0.1](https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator/0.1) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 8d72679536..0c2937f3df 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -1,8 +1,11 @@ openapi: "3.0.2" info: title: "Minimal Beacon Node API for Validator" - description: "A minimal API specification for the beacon node, which enabling a validator to connect and perform its obligations on the Ethereum 2.0 phase 0 beacon chain." + description: "A minimal API specification for the beacon node, which enables a validator to connect and perform its obligations on the Ethereum 2.0 phase 0 beacon chain." version: "0.1" + license: + name: "Apache 2.0" + url: "https://www.apache.org/licenses/LICENSE-2.0.html" tags: - name: Necessary for validator description: The minimal set of endpoints to enable a working validator implementation. From ac7a40376ba787b55abfc96aa87d0741a6fdb437 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 13 May 2019 16:26:45 +1000 Subject: [PATCH 099/308] Included a link to the API document in the root README.md file. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ad7204f214..ec80569569 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Core specifications for eth2.0 client validation can be found in [specs/core](sp * [General test format](specs/test_formats/README.md) * [Merkle proof formats](specs/light_client/merkle_proofs.md) * [Light client syncing protocol](specs/light_client/sync_protocol.md) +* [Beacon node API for validator](specs/validator/0_beacon-node-validator-api.md) ### Design goals From 67921ab96fdd1ed108fb5bc2f8d7d75871083a8b Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 13 May 2019 16:30:53 +1000 Subject: [PATCH 100/308] Fixed up some small wording in the API readme. --- specs/validator/0_beacon-node-validator-api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-node-validator-api.md b/specs/validator/0_beacon-node-validator-api.md index cf763f7781..5c597aa75c 100644 --- a/specs/validator/0_beacon-node-validator-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -4,9 +4,9 @@ __NOTICE__: This document is a work-in-progress for researchers and implementers ## Outline -This document outlines a minimal application programming interface (API) which is exposed by a `BeaconNode` for use by a `ValidatorClient` which aims to facilitate [_phase 0_](../../README.md#phase-0) of Ethereum 2.0. +This document outlines a minimal application programming interface (API) which is exposed by a beacon node for use by a validator client implementation which aims to facilitate [_phase 0_](../../README.md#phase-0) of Ethereum 2.0. -The API is a REST interface, accessed via HTTP, designed for use as a local communications protocol between binaries. +The API is a REST interface, accessed via HTTP, designed for use as a local communications protocol between binaries. The only supported return data type is currently JSON. ### Background The beacon node maintains the state of the beacon chain by communicating with other beacon nodes in the Ethereum Serenity network. Conceptually, it does not maintain keypairs that participate with the beacon chain. From 7bb85a69ed5759bbf64a73bac111865fc53b2412 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 13 May 2019 16:34:30 -0400 Subject: [PATCH 101/308] add process_slots usage to validator guide --- specs/validator/0_beacon-chain-validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 49290b432d..ef6e8bf25e 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -238,7 +238,7 @@ A validator should create and broadcast the attestation halfway through the `slo First the validator should construct `attestation_data`, an [`AttestationData`](../core/0_beacon-chain.md#attestationdata) object based upon the state at the assigned slot. * Let `head_block` be the result of running the fork choice during the assigned slot. -* Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot. +* Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot using `process_slots(state, slot)`. ##### LMD GHOST vote @@ -360,7 +360,7 @@ def is_proposer_at_slot(state: BeaconState, return get_beacon_proposer_index(state) == validator_index ``` -*Note*: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot. +*Note*: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot using `process_slots(state, current_slot)`. ### Lookahead From a0a2aa90de166d840ba69dcc487079c7b26571f7 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 13 May 2019 16:40:45 -0400 Subject: [PATCH 102/308] lint --- specs/core/0_beacon-chain.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 0d0e376791..524a40b17f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1253,7 +1253,6 @@ def process_slot(state: BeaconState) -> None: # Cache block root previous_block_root = signing_root(state.latest_block_header) state.latest_block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root - ``` ### Epoch processing @@ -1610,7 +1609,12 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: proposer = state.validator_registry[get_beacon_proposer_index(state)] # Verify that the provided randao value is valid - assert bls_verify(proposer.pubkey, hash_tree_root(get_current_epoch(state)), body.randao_reveal, get_domain(state, DOMAIN_RANDAO)) + assert bls_verify( + proposer.pubkey, + hash_tree_root(get_current_epoch(state)), + body.randao_reveal, + get_domain(state, DOMAIN_RANDAO), + ) # Mix it in state.latest_randao_mixes[get_current_epoch(state) % LATEST_RANDAO_MIXES_LENGTH] = ( xor(get_randao_mix(state, get_current_epoch(state)), From 5ba90d68e194310816f22f36caa8a7cc3d95cf52 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 13 May 2019 16:53:28 -0400 Subject: [PATCH 103/308] add flag for validate state root --- specs/core/0_beacon-chain.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 524a40b17f..a9c4f88112 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1220,11 +1220,14 @@ Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled excpetion (e.g. a failed `assert` or an out-of-range list access) are considered invalid. ```python -def state_transition(state: BeaconState, block: BeaconBlock) -> BeaconState: +def state_transition(state: BeaconState, block: BeaconBlock, validate_state_root: bool=False) -> BeaconState: # Process slots (including those with no blocks) since block process_slots(state, block.slot) # Process block process_block(state, block) + # Validate state root if received from external source + if validate_state_root: + process_state_root(state, block) # Return post-state return state ``` @@ -1579,7 +1582,6 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) - # process_state_root(state, block) ``` #### Block header From c2942c00c6555cd03aa0595b9ee947e5eb06e6f1 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 13 May 2019 17:02:20 -0400 Subject: [PATCH 104/308] lint requires install_test in ci --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fd7708f8da..2bfda20ffa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,10 +102,12 @@ workflows: test_spec: jobs: - checkout_specs - - lint - install_test: requires: - checkout_specs - test: requires: - install_test + - lint: + requires: + - install_test From f830f83a1d6ae1b6728bfd357da6cbef0d4a5107 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 14 May 2019 10:32:20 +0800 Subject: [PATCH 105/308] Update config.yml --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2bfda20ffa..439afc9a95 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -102,12 +102,12 @@ workflows: test_spec: jobs: - checkout_specs + - lint: + requires: + - checkout_specs - install_test: requires: - checkout_specs - test: requires: - install_test - - lint: - requires: - - install_test From c60635d2c9c829865189b4856ef4748cff7a3d9d Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 14 May 2019 06:15:03 +0100 Subject: [PATCH 106/308] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a9c4f88112..ee37d92175 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -117,7 +117,6 @@ - [Deposits](#deposits) - [Voluntary exits](#voluntary-exits) - [Transfers](#transfers) - - [State root verification](#state-root-verification) @@ -373,6 +372,7 @@ The types are defined topologically to aid in facilitating an executable version 'signature': 'bytes96', } ``` + #### `Validator` ```python @@ -577,9 +577,7 @@ The types are defined topologically to aid in facilitating an executable version 'latest_block_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], 'latest_state_roots': ['bytes32', SLOTS_PER_HISTORICAL_ROOT], 'latest_active_index_roots': ['bytes32', LATEST_ACTIVE_INDEX_ROOTS_LENGTH], - # Balances slashed at every withdrawal period 'latest_slashed_balances': ['uint64', LATEST_SLASHED_EXIT_LENGTH], - # `latest_block_header.state_root == ZERO_HASH` temporarily 'latest_block_header': BeaconBlockHeader, 'historical_roots': ['bytes32'], @@ -1225,9 +1223,9 @@ def state_transition(state: BeaconState, block: BeaconBlock, validate_state_root process_slots(state, block.slot) # Process block process_block(state, block) - # Validate state root if received from external source + # Validate state root (`validate_state_root == True` in production) if validate_state_root: - process_state_root(state, block) + assert block.state_root == hash_tree_root(state) # Return post-state return state ``` @@ -1838,10 +1836,3 @@ def process_transfer(state: BeaconState, transfer: Transfer) -> None: assert not (0 < state.balances[transfer.sender] < MIN_DEPOSIT_AMOUNT) assert not (0 < state.balances[transfer.recipient] < MIN_DEPOSIT_AMOUNT) ``` - -#### State root verification - -```python -def process_state_root(state: BeaconState, block: BeaconBlock) -> None: - assert block.state_root == hash_tree_root(state) -``` From 5e7b173b22e0d550092ac967f1dc84d2c794e935 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 14 May 2019 17:29:11 -0400 Subject: [PATCH 107/308] fix up validator guide crosslink committee instructions --- specs/validator/0_beacon-chain-validator.md | 119 +++++++++----------- 1 file changed, 53 insertions(+), 66 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ef6e8bf25e..67a09c797f 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -20,6 +20,8 @@ - [Process deposit](#process-deposit) - [Validator index](#validator-index) - [Activation](#activation) + - [Validator assignments](#validator-assignments) + - [Lookahead](#lookahead) - [Beacon chain responsibilities](#beacon-chain-responsibilities) - [Block proposal](#block-proposal) - [Block header](#block-header) @@ -45,8 +47,6 @@ - [Aggregation bitfield](#aggregation-bitfield) - [Custody bitfield](#custody-bitfield) - [Aggregate signature](#aggregate-signature) - - [Validator assignments](#validator-assignments) - - [Lookahead](#lookahead) - [How to avoid slashing](#how-to-avoid-slashing) - [Proposer slashing](#proposer-slashing) - [Attester slashing](#attester-slashing) @@ -127,13 +127,61 @@ Once a validator is activated, the validator is assigned [responsibilities](#bea *Note*: There is a maximum validator churn per finalized epoch so the delay until activation is variable depending upon finality, total active validator balance, and the number of validators in the queue to be activated. +## Validator assignments + +A validator can get committee assignments for a given epoch using the following helper via `get_committee_assignment(state, epoch, validator_index)` where `epoch <= next_epoch`. + +```python +def get_committee_assignment( + state: BeaconState, + epoch: Epoch, + validator_index: ValidatorIndex) -> Tuple[List[ValidatorIndex], Shard, Slot]: + """ + Return the committee assignment in the ``epoch`` for ``validator_index``. + ``assignment`` returned is a tuple of the following form: + * ``assignment[0]`` is the list of validators in the committee + * ``assignment[1]`` is the shard to which the committee is assigned + * ``assignment[2]`` is the slot at which the committee is assigned + """ + next_epoch = get_current_epoch(state) + 1 + assert epoch <= next_epoch + + committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH + epoch_start_slot = get_epoch_start_slot(epoch) + for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH) + offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) + for i in range(committees_per_slot): + shard = (get_epoch_start_shard(state, epoch) + i) % SHARD_COUNT + committee = get_crosslink_committee(state, epoch, shard) + if validator_index in committee: + return committee, shard, slot +``` + +A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run during the slot in question. Proposer selection is only stable within the context of the current epoch. + +```python +def is_proposer(state: BeaconState, + validator_index: ValidatorIndex) -> bool: + return get_beacon_proposer_index(state) == validator_index +``` + +*Note*: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot using `process_slots(state, current_slot)`. + +### Lookahead + +The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must checked during the slot in question. + +`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in Phase 1+). + +Specifically, a validator should call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. + ## Beacon chain responsibilities A validator has two primary responsibilities to the beacon chain: [proposing blocks](#block-proposal) and [creating attestations](#attestations-1). Proposals happen infrequently, whereas attestations should be created once per epoch. ### Block proposal -A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `get_beacon_proposer_index(state)` returns the validator's `validator_index`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator is to create, sign, and broadcast a `block` that is a child of `parent` and that executes a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). +A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator is to create, sign, and broadcast a `block` that is a child of `parent` and that executes a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). There is one proposer per slot, so if there are N active validators any individual validator will on average be assigned to propose once per N slots (e.g. at 312500 validators = 10 million ETH, that's once per ~3 weeks). @@ -229,7 +277,7 @@ Up to `MAX_VOLUNTARY_EXITS` [`VoluntaryExit`](../core/0_beacon-chain.md#voluntar ### Attestations -A validator is expected to create, sign, and broadcast an attestation during each epoch. The slot during which the validator performs this role is any slot at which `get_crosslink_committees_at_slot(state, slot)` contains a committee that contains `validator_index`. +A validator is expected to create, sign, and broadcast an attestation during each epoch. The committee, assigned shard, and assigned slot for which the validator performs this role during an epoch is defined by `get_committee_assignment(state, epoch, validator_index)`. A validator should create and broadcast the attestation halfway through the `slot` during which the validator is assigned ― that is, `SECONDS_PER_SLOT * 0.5` seconds after the start of `slot`. @@ -259,7 +307,7 @@ Set `attestation_data.beacon_block_root = signing_root(head_block)`. Construct `attestation_data.crosslink` via the following -* Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associated with the validator's committee defined by `get_crosslink_committees_at_slot`. +* Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associated with the validator's committee. * Set `attestation_data.crosslink.epoch = min(attestation_data.target_epoch, head_state.current_crosslinks[shard].epoch + MAX_EPOCHS_PER_CROSSLINK)`. * Set `attestation_data.crosslink.parent_root = hash_tree_root(head_state.current_crosslinks[shard])`. * Set `attestation_data.crosslink.data_root = ZERO_HASH`. *Note*: This is a stub for Phase 0. @@ -310,67 +358,6 @@ signed_attestation_data = bls_sign( ) ``` -## Validator assignments - -A validator can get the current, previous, and next epoch committee assignments using the following helper via `get_committee_assignment(state, epoch, validator_index)` where `previous_epoch <= epoch <= next_epoch`. - -```python -def get_committee_assignment( - state: BeaconState, - epoch: Epoch, - validator_index: ValidatorIndex) -> Tuple[List[ValidatorIndex], Shard, Slot]: - """ - Return the committee assignment in the ``epoch`` for ``validator_index``. - ``assignment`` returned is a tuple of the following form: - * ``assignment[0]`` is the list of validators in the committee - * ``assignment[1]`` is the shard to which the committee is assigned - * ``assignment[2]`` is the slot at which the committee is assigned - """ - previous_epoch = get_previous_epoch(state) - next_epoch = get_current_epoch(state) + 1 - assert previous_epoch <= epoch <= next_epoch - - epoch_start_slot = get_epoch_start_slot(epoch) - for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH): - crosslink_committees = get_crosslink_committees_at_slot( - state, - slot, - ) - selected_committees = [ - committee # Tuple[List[ValidatorIndex], Shard] - for committee in crosslink_committees - if validator_index in committee[0] - ] - if len(selected_committees) > 0: - validators = selected_committees[0][0] - shard = selected_committees[0][1] - - assignment = (validators, shard, slot) - return assignment -``` - -A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run during the slot in question. Proposer selection is only stable within the context of the current epoch. - -```python -def is_proposer_at_slot(state: BeaconState, - slot: Slot, - validator_index: ValidatorIndex) -> bool: - assert state.slot == slot - - return get_beacon_proposer_index(state) == validator_index -``` - -*Note*: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot using `process_slots(state, current_slot)`. - - -### Lookahead - -The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must checked during the slot in question. - -`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in Phase 1+). - -Specifically, a validator should call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. - ## How to avoid slashing "Slashing" is the burning of some amount of validator funds and immediate ejection from the active validator set. In Phase 0, there are two ways in which funds can be slashed -- [proposer slashing](#proposer-slashing) and [attester slashing](#attester-slashing). Although being slashed has serious repercussions, it is simple enough to avoid being slashed all together by remaining _consistent_ with respect to the messages a validator has previously signed. From 01efe52eb0d7523994b96ccb6e44b85929b0ad42 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 14 May 2019 17:32:44 -0400 Subject: [PATCH 108/308] fix start shard --- specs/validator/0_beacon-chain-validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 67a09c797f..3b498bfd45 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -149,9 +149,9 @@ def get_committee_assignment( committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH epoch_start_slot = get_epoch_start_slot(epoch) for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH) - offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) + slot_start_shard = get_epoch_start_shard(state, epoch) + committees_per_slot * (slot % SLOTS_PER_EPOCH) for i in range(committees_per_slot): - shard = (get_epoch_start_shard(state, epoch) + i) % SHARD_COUNT + shard = (slot_start_shard + i) % SHARD_COUNT committee = get_crosslink_committee(state, epoch, shard) if validator_index in committee: return committee, shard, slot From bc95906e4f706460f9e431178da45ae8d1244b41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20=C5=A0kvorc?= Date: Wed, 15 May 2019 10:40:42 +0200 Subject: [PATCH 109/308] Typo fix --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index ef6e8bf25e..ae0e02a921 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -365,7 +365,7 @@ def is_proposer_at_slot(state: BeaconState, ### Lookahead -The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must checked during the slot in question. +The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must be checked during the slot in question. `get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in Phase 1+). From f3742566f73e2d4d287e81193ac6d740ef8cded6 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Wed, 15 May 2019 15:02:27 +0200 Subject: [PATCH 110/308] The new test structure starts to take shape --- Makefile | 7 +- scripts/phase0/build_spec.py | 1 + test_libs/pyspec/__init__.py | 0 ...est_process_early_derived_secret_reveal.py | 109 ------------------ test_libs/pyspec/tests/phase0/__init__.py | 0 .../test_process_attestation.py | 9 +- .../test_process_attester_slashing.py | 6 +- .../test_process_block_header.py | 4 +- .../block_processing/test_process_deposit.py | 6 +- .../test_process_proposer_slashing.py | 6 +- .../block_processing/test_process_transfer.py | 6 +- .../block_processing/test_voluntary_exit.py | 6 +- .../pyspec/tests/{ => phase0}/conftest.py | 6 +- .../test_process_crosslinks.py | 11 +- .../test_process_registry_updates.py | 6 +- .../pyspec/tests/{ => phase0}/helpers.py | 10 +- .../tests/{ => phase0}/test_finality.py | 4 +- .../pyspec/tests/{ => phase0}/test_sanity.py | 7 +- test_libs/pyspec/tests/phase1/__init__.py | 0 ...est_process_early_derived_secret_reveal.py | 109 ++++++++++++++++++ test_libs/pyspec/tests/phase1/conftest.py | 9 ++ test_libs/pyspec/tests/phase1/helpers.py | 0 .../tests/{ => phase1}/helpers_phase1.py | 0 .../pyspec/tests/phase1/test_finality.py | 30 +++++ 24 files changed, 190 insertions(+), 162 deletions(-) create mode 100644 test_libs/pyspec/__init__.py delete mode 100644 test_libs/pyspec/tests/block_processing_phase1/test_process_early_derived_secret_reveal.py create mode 100644 test_libs/pyspec/tests/phase0/__init__.py rename test_libs/pyspec/tests/{ => phase0}/block_processing/test_process_attestation.py (96%) rename test_libs/pyspec/tests/{ => phase0}/block_processing/test_process_attester_slashing.py (97%) rename test_libs/pyspec/tests/{ => phase0}/block_processing/test_process_block_header.py (96%) rename test_libs/pyspec/tests/{ => phase0}/block_processing/test_process_deposit.py (97%) rename test_libs/pyspec/tests/{ => phase0}/block_processing/test_process_proposer_slashing.py (96%) rename test_libs/pyspec/tests/{ => phase0}/block_processing/test_process_transfer.py (97%) rename test_libs/pyspec/tests/{ => phase0}/block_processing/test_voluntary_exit.py (97%) rename test_libs/pyspec/tests/{ => phase0}/conftest.py (84%) rename test_libs/pyspec/tests/{ => phase0}/epoch_processing/test_process_crosslinks.py (95%) rename test_libs/pyspec/tests/{ => phase0}/epoch_processing/test_process_registry_updates.py (94%) rename test_libs/pyspec/tests/{ => phase0}/helpers.py (98%) rename test_libs/pyspec/tests/{ => phase0}/test_finality.py (98%) rename test_libs/pyspec/tests/{ => phase0}/test_sanity.py (99%) create mode 100644 test_libs/pyspec/tests/phase1/__init__.py create mode 100644 test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py create mode 100644 test_libs/pyspec/tests/phase1/conftest.py create mode 100644 test_libs/pyspec/tests/phase1/helpers.py rename test_libs/pyspec/tests/{ => phase1}/helpers_phase1.py (100%) create mode 100644 test_libs/pyspec/tests/phase1/test_finality.py diff --git a/Makefile b/Makefile index 4c4bfd88b7..49460bdf11 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,6 @@ clean: rm -rf $(GENERATOR_VENVS) rm -rf $(PY_SPEC_DIR)/venv $(PY_SPEC_DIR)/.pytest_cache rm -rf $(PY_SPEC_ALL_TARGETS) - rm -rf $(PY_SPEC_PHASE_1_TARGETS) # "make gen_yaml_tests" to run generators gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS) @@ -42,6 +41,12 @@ install_test: test: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest . +test_phase0: $(PY_SPEC_ALL_TARGETS) + cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest tests/phase0 + +test_phase1: $(PY_SPEC_ALL_TARGETS) + cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest tests/phase1 + citest: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml . diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index 0ebfd99068..4faf8cbf64 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -15,6 +15,7 @@ def build_phase0_spec(sourcefile, outfile): ) from eth2spec.utils.minimal_ssz import ( SSZType, + SSZTypeExtension, hash_tree_root, signing_root, ) diff --git a/test_libs/pyspec/__init__.py b/test_libs/pyspec/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_libs/pyspec/tests/block_processing_phase1/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/tests/block_processing_phase1/test_process_early_derived_secret_reveal.py deleted file mode 100644 index 30ae1e77ef..0000000000 --- a/test_libs/pyspec/tests/block_processing_phase1/test_process_early_derived_secret_reveal.py +++ /dev/null @@ -1,109 +0,0 @@ -from copy import deepcopy -import pytest - -import eth2spec.phase1.spec as spec -from eth2spec.phase1.spec import ( - get_current_epoch, - process_early_derived_secret_reveal, - RANDAO_PENALTY_EPOCHS, - CUSTODY_PERIOD_TO_RANDAO_PADDING, - EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS, -) -from tests.helpers_phase1 import ( - get_valid_early_derived_secret_reveal, -) - -#mark entire file as 'randao_key_reveals' -pytestmark = pytest.mark.randao_key_reveals - - -def run_early_derived_secret_reveal_processing(state, randao_key_reveal, valid=True): - """ - Run ``process_randao_key_reveal`` returning the pre and post state. - If ``valid == False``, run expecting ``AssertionError`` - """ - post_state = deepcopy(state) - - if not valid: - with pytest.raises(AssertionError): - process_early_derived_secret_reveal(post_state, randao_key_reveal) - return state, None - - process_early_derived_secret_reveal(post_state, randao_key_reveal) - - slashed_validator = post_state.validator_registry[randao_key_reveal.revealed_index] - - if randao_key_reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - # lost whistleblower reward - # FIXME: Currently broken because get_base_reward in genesis epoch is 0 - assert ( - post_state.balances[randao_key_reveal.revealed_index] < - state.balances[randao_key_reveal.revealed_index] - ) - - return state, post_state - - -def test_success(state): - randao_key_reveal = get_valid_early_derived_secret_reveal(state) - - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal) - - return pre_state, randao_key_reveal, post_state - - -def test_reveal_from_current_epoch(state): - randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state)) - - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) - - return pre_state, randao_key_reveal, post_state - -# Not currently possible as we are testing at epoch 0 -# -#def test_reveal_from_past_epoch(state): -# randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) - 1) -# -# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) -# -# return pre_state, randao_key_reveal, post_state - -def test_reveal_with_custody_padding(state): - randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING) - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) - - return pre_state, randao_key_reveal, post_state - -def test_reveal_with_custody_padding_minus_one(state): - randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING - 1) - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) - - return pre_state, randao_key_reveal, post_state - -def test_double_reveal(state): - - randao_key_reveal1 = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + RANDAO_PENALTY_EPOCHS + 1) - pre_state, intermediate_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal1) - - randao_key_reveal2 = get_valid_early_derived_secret_reveal(intermediate_state, get_current_epoch(pre_state) + RANDAO_PENALTY_EPOCHS + 1) - intermediate_state_, post_state = run_early_derived_secret_reveal_processing(intermediate_state, randao_key_reveal2, False) - - return pre_state, [randao_key_reveal1, randao_key_reveal2], post_state - -def test_revealer_is_slashed(state): - randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state)) - state.validator_registry[randao_key_reveal.revealed_index].slashed = True - - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) - - return pre_state, randao_key_reveal, post_state - -def test_far_future_epoch(state): - randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS) - - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) - - return pre_state, randao_key_reveal, post_state diff --git a/test_libs/pyspec/tests/phase0/__init__.py b/test_libs/pyspec/tests/phase0/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_attestation.py similarity index 96% rename from test_libs/pyspec/tests/block_processing/test_process_attestation.py rename to test_libs/pyspec/tests/phase0/block_processing/test_process_attestation.py index 0b1ceb79e9..441df21e23 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_attestation.py @@ -1,18 +1,15 @@ from copy import deepcopy import pytest -import eth2spec.phase1.spec as spec +import eth2spec.phase0.spec as spec -from eth2spec.phase1.state_transition import ( - state_transition, -) -from eth2spec.phase1.spec import ( +from eth2spec.phase0.spec import ( get_current_epoch, process_attestation, slot_to_epoch, state_transition, ) -from tests.helpers import ( +from tests.phase0.helpers import ( build_empty_block_for_next_slot, get_valid_attestation, next_epoch, diff --git a/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_attester_slashing.py similarity index 97% rename from test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py rename to test_libs/pyspec/tests/phase0/block_processing/test_process_attester_slashing.py index 880ca829a7..d5e150b1d2 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_attester_slashing.py @@ -1,12 +1,12 @@ from copy import deepcopy import pytest -import eth2spec.phase1.spec as spec -from eth2spec.phase1.spec import ( +import eth2spec.phase0.spec as spec +from eth2spec.phase0.spec import ( get_beacon_proposer_index, process_attester_slashing, ) -from tests.helpers import ( +from tests.phase0.helpers import ( get_balance, get_valid_attester_slashing, next_epoch, diff --git a/test_libs/pyspec/tests/block_processing/test_process_block_header.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_block_header.py similarity index 96% rename from test_libs/pyspec/tests/block_processing/test_process_block_header.py rename to test_libs/pyspec/tests/phase0/block_processing/test_process_block_header.py index bb9deab28a..4b66c2ff06 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_block_header.py @@ -2,12 +2,12 @@ import pytest -from eth2spec.phase1.spec import ( +from eth2spec.phase0.spec import ( get_beacon_proposer_index, process_slot, process_block_header, ) -from tests.helpers import ( +from tests.phase0.helpers import ( advance_slot, build_empty_block_for_next_slot, next_slot, diff --git a/test_libs/pyspec/tests/block_processing/test_process_deposit.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_deposit.py similarity index 97% rename from test_libs/pyspec/tests/block_processing/test_process_deposit.py rename to test_libs/pyspec/tests/phase0/block_processing/test_process_deposit.py index 768f616786..0aa057c148 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_deposit.py @@ -1,13 +1,13 @@ from copy import deepcopy import pytest -import eth2spec.phase1.spec as spec +import eth2spec.phase0.spec as spec -from eth2spec.phase1.spec import ( +from eth2spec.phase0.spec import ( ZERO_HASH, process_deposit, ) -from tests.helpers import ( +from tests.phase0.helpers import ( get_balance, build_deposit, privkeys, diff --git a/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_proposer_slashing.py similarity index 96% rename from test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py rename to test_libs/pyspec/tests/phase0/block_processing/test_process_proposer_slashing.py index 513e607b1e..366417d8b3 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_proposer_slashing.py @@ -1,12 +1,12 @@ from copy import deepcopy import pytest -import eth2spec.phase1.spec as spec -from eth2spec.phase1.spec import ( +import eth2spec.phase0.spec as spec +from eth2spec.phase0.spec import ( get_current_epoch, process_proposer_slashing, ) -from tests.helpers import ( +from tests.phase0.helpers import ( get_balance, get_valid_proposer_slashing, ) diff --git a/test_libs/pyspec/tests/block_processing/test_process_transfer.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py similarity index 97% rename from test_libs/pyspec/tests/block_processing/test_process_transfer.py rename to test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py index 02f448fd02..cb657bb9de 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py @@ -1,15 +1,15 @@ from copy import deepcopy import pytest -import eth2spec.phase1.spec as spec +import eth2spec.phase0.spec as spec -from eth2spec.phase1.spec import ( +from eth2spec.phase0.spec import ( get_active_validator_indices, get_beacon_proposer_index, get_current_epoch, process_transfer, ) -from tests.helpers import ( +from tests.phase0.helpers import ( get_valid_transfer, next_epoch, ) diff --git a/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py b/test_libs/pyspec/tests/phase0/block_processing/test_voluntary_exit.py similarity index 97% rename from test_libs/pyspec/tests/block_processing/test_voluntary_exit.py rename to test_libs/pyspec/tests/phase0/block_processing/test_voluntary_exit.py index c789b55d83..63be4c8f09 100644 --- a/test_libs/pyspec/tests/block_processing/test_voluntary_exit.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_voluntary_exit.py @@ -1,15 +1,15 @@ from copy import deepcopy import pytest -import eth2spec.phase1.spec as spec +import eth2spec.phase0.spec as spec -from eth2spec.phase1.spec import ( +from eth2spec.phase0.spec import ( get_active_validator_indices, get_churn_limit, get_current_epoch, process_voluntary_exit, ) -from tests.helpers import ( +from tests.phase0.helpers import ( build_voluntary_exit, pubkey_to_privkey, ) diff --git a/test_libs/pyspec/tests/conftest.py b/test_libs/pyspec/tests/phase0/conftest.py similarity index 84% rename from test_libs/pyspec/tests/conftest.py rename to test_libs/pyspec/tests/phase0/conftest.py index 98b64c73bf..4deb65da8f 100644 --- a/test_libs/pyspec/tests/conftest.py +++ b/test_libs/pyspec/tests/phase0/conftest.py @@ -1,16 +1,16 @@ import pytest -from eth2spec.phase1 import spec +from eth2spec.phase0 import spec from preset_loader import loader -from .helpers import ( +from tests.phase0.helpers import ( create_genesis_state, ) def pytest_addoption(parser): parser.addoption( - "--config", action="store", default="minimal", help="config: make the pyspec use the specified configuration" + "--config", action="store", default="minimal", help="config: make the p yspec use the specified configuration" ) diff --git a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py similarity index 95% rename from test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py rename to test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py index b2202255f7..6bd9b5d89e 100644 --- a/test_libs/pyspec/tests/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py @@ -1,20 +1,15 @@ from copy import deepcopy import pytest -import eth2spec.phase1.spec as spec +import eth2spec.phase0.spec as spec -<<<<<<< HEAD -from eth2spec.phase1.state_transition import ( - state_transition, -) -from eth2spec.phase1.spec import ( - cache_state, +from eth2spec.phase0.spec import ( process_slot, get_crosslink_deltas, process_crosslinks, state_transition, ) -from tests.helpers import ( +from tests.phase0.helpers import ( add_attestation_to_state, build_empty_block_for_next_slot, fill_aggregate_attestation, diff --git a/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py similarity index 94% rename from test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py rename to test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py index 15f7ae6ecc..54379de7dc 100644 --- a/test_libs/pyspec/tests/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py @@ -2,13 +2,13 @@ import pytest -import eth2spec.phase1.spec as spec +import eth2spec.phase0.spec as spec -from eth2spec.phase1.spec import ( +from eth2spec.phase0.spec import ( get_current_epoch, is_active_validator, ) -from tests.helpers import ( +from tests.phase0.helpers import ( next_epoch, ) diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/phase0/helpers.py similarity index 98% rename from test_libs/pyspec/tests/helpers.py rename to test_libs/pyspec/tests/phase0/helpers.py index ed08013f17..e8b4f789ee 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/phase0/helpers.py @@ -2,16 +2,10 @@ from py_ecc import bls -<<<<<<< HEAD -from eth2spec.phase1.state_transition import ( - state_transition, -) -import eth2spec.phase1.spec as spec -======= import eth2spec.phase0.spec as spec ->>>>>>> dev + from eth2spec.utils.minimal_ssz import signing_root -from eth2spec.phase1.spec import ( +from eth2spec.phase0.spec import ( # constants ZERO_HASH, MAX_EPOCHS_PER_CROSSLINK, diff --git a/test_libs/pyspec/tests/test_finality.py b/test_libs/pyspec/tests/phase0/test_finality.py similarity index 98% rename from test_libs/pyspec/tests/test_finality.py rename to test_libs/pyspec/tests/phase0/test_finality.py index 20590154be..49569f220c 100644 --- a/test_libs/pyspec/tests/test_finality.py +++ b/test_libs/pyspec/tests/phase0/test_finality.py @@ -2,9 +2,9 @@ import pytest -import eth2spec.phase1.spec as spec +import eth2spec.phase0.spec as spec -from eth2spec.phase1.state_transition import ( +from eth2spec.phase0.spec import ( state_transition, ) diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/phase0/test_sanity.py similarity index 99% rename from test_libs/pyspec/tests/test_sanity.py rename to test_libs/pyspec/tests/phase0/test_sanity.py index 33a6f72213..7f3ee48e1e 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/phase0/test_sanity.py @@ -3,10 +3,10 @@ import pytest from py_ecc import bls -import eth2spec.phase1.spec as spec +import eth2spec.phase0.spec as spec from eth2spec.utils.minimal_ssz import signing_root -from eth2spec.phase1.spec import ( +from eth2spec.phase0.spec import ( # constants ZERO_HASH, SLOTS_PER_HISTORICAL_ROOT, @@ -22,9 +22,6 @@ get_domain, process_slot, verify_merkle_branch, - hash, -) -from eth2spec.phase1.state_transition import ( state_transition, hash, ) diff --git a/test_libs/pyspec/tests/phase1/__init__.py b/test_libs/pyspec/tests/phase1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py new file mode 100644 index 0000000000..8c9bb0efbb --- /dev/null +++ b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py @@ -0,0 +1,109 @@ +# from copy import deepcopy +# import pytest + +# import eth2spec.phase1.spec as spec +# from eth2spec.phase1.spec import ( +# get_current_epoch, +# process_early_derived_secret_reveal, +# RANDAO_PENALTY_EPOCHS, +# CUSTODY_PERIOD_TO_RANDAO_PADDING, +# EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS, +# ) +# from tests.helpers_phase1 import ( +# get_valid_early_derived_secret_reveal, +# ) + +# #mark entire file as 'randao_key_reveals' +# pytestmark = pytest.mark.randao_key_reveals + + +# def run_early_derived_secret_reveal_processing(state, randao_key_reveal, valid=True): +# """ +# Run ``process_randao_key_reveal`` returning the pre and post state. +# If ``valid == False``, run expecting ``AssertionError`` +# """ +# post_state = deepcopy(state) + +# if not valid: +# with pytest.raises(AssertionError): +# process_early_derived_secret_reveal(post_state, randao_key_reveal) +# return state, None + +# process_early_derived_secret_reveal(post_state, randao_key_reveal) + +# slashed_validator = post_state.validator_registry[randao_key_reveal.revealed_index] + +# if randao_key_reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: +# assert slashed_validator.slashed +# assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH +# assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH +# # lost whistleblower reward +# # FIXME: Currently broken because get_base_reward in genesis epoch is 0 +# assert ( +# post_state.balances[randao_key_reveal.revealed_index] < +# state.balances[randao_key_reveal.revealed_index] +# ) + +# return state, post_state + + +# def test_success(state): +# randao_key_reveal = get_valid_early_derived_secret_reveal(state) + +# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal) + +# return pre_state, randao_key_reveal, post_state + + +# def test_reveal_from_current_epoch(state): +# randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state)) + +# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + +# return pre_state, randao_key_reveal, post_state + +# # Not currently possible as we are testing at epoch 0 +# # +# #def test_reveal_from_past_epoch(state): +# # randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) - 1) +# # +# # pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) +# # +# # return pre_state, randao_key_reveal, post_state + +# def test_reveal_with_custody_padding(state): +# randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING) +# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) + +# return pre_state, randao_key_reveal, post_state + +# def test_reveal_with_custody_padding_minus_one(state): +# randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING - 1) +# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) + +# return pre_state, randao_key_reveal, post_state + +# def test_double_reveal(state): + +# randao_key_reveal1 = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + RANDAO_PENALTY_EPOCHS + 1) +# pre_state, intermediate_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal1) + +# randao_key_reveal2 = get_valid_early_derived_secret_reveal(intermediate_state, get_current_epoch(pre_state) + RANDAO_PENALTY_EPOCHS + 1) +# intermediate_state_, post_state = run_early_derived_secret_reveal_processing(intermediate_state, randao_key_reveal2, False) + +# return pre_state, [randao_key_reveal1, randao_key_reveal2], post_state + +# def test_revealer_is_slashed(state): +# randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state)) +# state.validator_registry[randao_key_reveal.revealed_index].slashed = True + +# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + +# return pre_state, randao_key_reveal, post_state + +# def test_far_future_epoch(state): +# randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS) + +# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + +# return pre_state, randao_key_reveal, post_state diff --git a/test_libs/pyspec/tests/phase1/conftest.py b/test_libs/pyspec/tests/phase1/conftest.py new file mode 100644 index 0000000000..4bacc1ea7f --- /dev/null +++ b/test_libs/pyspec/tests/phase1/conftest.py @@ -0,0 +1,9 @@ +import pytest + +from tests.phase0.conftest import ( + pytest_addoption, + config, + num_validators, + deposit_data_leaves, + state, +) diff --git a/test_libs/pyspec/tests/phase1/helpers.py b/test_libs/pyspec/tests/phase1/helpers.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_libs/pyspec/tests/helpers_phase1.py b/test_libs/pyspec/tests/phase1/helpers_phase1.py similarity index 100% rename from test_libs/pyspec/tests/helpers_phase1.py rename to test_libs/pyspec/tests/phase1/helpers_phase1.py diff --git a/test_libs/pyspec/tests/phase1/test_finality.py b/test_libs/pyspec/tests/phase1/test_finality.py new file mode 100644 index 0000000000..b0acaa7cb2 --- /dev/null +++ b/test_libs/pyspec/tests/phase1/test_finality.py @@ -0,0 +1,30 @@ +from copy import deepcopy + +import pytest + +import eth2spec.phase1.spec as spec + +from eth2spec.phase1.spec import ( + state_transition, +) + +from tests.phase0.helpers import ( + build_empty_block_for_next_slot, + fill_aggregate_attestation, + get_current_epoch, + get_epoch_start_slot, + get_valid_attestation, + next_epoch, +) + +# mark entire file as 'state' +# pytestmark = pytest.mark.state + +from tests.phase0.test_finality import ( + pytestmark, + check_finality, + test_finality_rule_1, + test_finality_rule_2, + test_finality_rule_3, + test_finality_rule_4, +) From 77b29f4960b4b28790c754a5f389a24ea32e5d2e Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Wed, 15 May 2019 15:17:01 +0200 Subject: [PATCH 111/308] Early derived secret testing --- ...est_process_early_derived_secret_reveal.py | 155 +++++++++--------- test_libs/pyspec/tests/phase1/helpers.py | 54 ++++++ 2 files changed, 131 insertions(+), 78 deletions(-) diff --git a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py index 8c9bb0efbb..54a90aca19 100644 --- a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py @@ -1,109 +1,108 @@ -# from copy import deepcopy -# import pytest +from copy import deepcopy +import pytest -# import eth2spec.phase1.spec as spec -# from eth2spec.phase1.spec import ( -# get_current_epoch, -# process_early_derived_secret_reveal, -# RANDAO_PENALTY_EPOCHS, -# CUSTODY_PERIOD_TO_RANDAO_PADDING, -# EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS, -# ) -# from tests.helpers_phase1 import ( -# get_valid_early_derived_secret_reveal, -# ) +import eth2spec.phase1.spec as spec +from eth2spec.phase1.spec import ( + get_current_epoch, + process_early_derived_secret_reveal, + RANDAO_PENALTY_EPOCHS, + CUSTODY_PERIOD_TO_RANDAO_PADDING, + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS, +) +from tests.phase1.helpers import ( + get_valid_early_derived_secret_reveal, +) -# #mark entire file as 'randao_key_reveals' -# pytestmark = pytest.mark.randao_key_reveals +#mark entire file as 'randao_key_reveals' +pytestmark = pytest.mark.randao_key_reveals +def run_early_derived_secret_reveal_processing(state, randao_key_reveal, valid=True): + """ + Run ``process_randao_key_reveal`` returning the pre and post state. + If ``valid == False``, run expecting ``AssertionError`` + """ + post_state = deepcopy(state) -# def run_early_derived_secret_reveal_processing(state, randao_key_reveal, valid=True): -# """ -# Run ``process_randao_key_reveal`` returning the pre and post state. -# If ``valid == False``, run expecting ``AssertionError`` -# """ -# post_state = deepcopy(state) + if not valid: + with pytest.raises(AssertionError): + process_early_derived_secret_reveal(post_state, randao_key_reveal) + return state, None -# if not valid: -# with pytest.raises(AssertionError): -# process_early_derived_secret_reveal(post_state, randao_key_reveal) -# return state, None + process_early_derived_secret_reveal(post_state, randao_key_reveal) -# process_early_derived_secret_reveal(post_state, randao_key_reveal) + slashed_validator = post_state.validator_registry[randao_key_reveal.revealed_index] -# slashed_validator = post_state.validator_registry[randao_key_reveal.revealed_index] + if randao_key_reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + # lost whistleblower reward + # FIXME: Currently broken because get_base_reward in genesis epoch is 0 + assert ( + post_state.balances[randao_key_reveal.revealed_index] < + state.balances[randao_key_reveal.revealed_index] + ) -# if randao_key_reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: -# assert slashed_validator.slashed -# assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH -# assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH -# # lost whistleblower reward -# # FIXME: Currently broken because get_base_reward in genesis epoch is 0 -# assert ( -# post_state.balances[randao_key_reveal.revealed_index] < -# state.balances[randao_key_reveal.revealed_index] -# ) + return state, post_state -# return state, post_state +def test_success(state): + randao_key_reveal = get_valid_early_derived_secret_reveal(state) -# def test_success(state): -# randao_key_reveal = get_valid_early_derived_secret_reveal(state) + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal) -# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal) + return pre_state, randao_key_reveal, post_state -# return pre_state, randao_key_reveal, post_state +def test_reveal_from_current_epoch(state): + randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state)) -# def test_reveal_from_current_epoch(state): -# randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state)) + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) -# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + return pre_state, randao_key_reveal, post_state -# return pre_state, randao_key_reveal, post_state +# Not currently possible as we are testing at epoch 0 +# +#def test_reveal_from_past_epoch(state): +# randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) - 1) +# +# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) +# +# return pre_state, randao_key_reveal, post_state -# # Not currently possible as we are testing at epoch 0 -# # -# #def test_reveal_from_past_epoch(state): -# # randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) - 1) -# # -# # pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) -# # -# # return pre_state, randao_key_reveal, post_state +def test_reveal_with_custody_padding(state): + randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING) + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) -# def test_reveal_with_custody_padding(state): -# randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING) -# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) + return pre_state, randao_key_reveal, post_state -# return pre_state, randao_key_reveal, post_state +def test_reveal_with_custody_padding_minus_one(state): + randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING - 1) + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) -# def test_reveal_with_custody_padding_minus_one(state): -# randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING - 1) -# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) + return pre_state, randao_key_reveal, post_state -# return pre_state, randao_key_reveal, post_state - -# def test_double_reveal(state): +def test_double_reveal(state): -# randao_key_reveal1 = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + RANDAO_PENALTY_EPOCHS + 1) -# pre_state, intermediate_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal1) + randao_key_reveal1 = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + RANDAO_PENALTY_EPOCHS + 1) + pre_state, intermediate_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal1) -# randao_key_reveal2 = get_valid_early_derived_secret_reveal(intermediate_state, get_current_epoch(pre_state) + RANDAO_PENALTY_EPOCHS + 1) -# intermediate_state_, post_state = run_early_derived_secret_reveal_processing(intermediate_state, randao_key_reveal2, False) + randao_key_reveal2 = get_valid_early_derived_secret_reveal(intermediate_state, get_current_epoch(pre_state) + RANDAO_PENALTY_EPOCHS + 1) + intermediate_state_, post_state = run_early_derived_secret_reveal_processing(intermediate_state, randao_key_reveal2, False) -# return pre_state, [randao_key_reveal1, randao_key_reveal2], post_state + return pre_state, [randao_key_reveal1, randao_key_reveal2], post_state -# def test_revealer_is_slashed(state): -# randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state)) -# state.validator_registry[randao_key_reveal.revealed_index].slashed = True +def test_revealer_is_slashed(state): + randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state)) + state.validator_registry[randao_key_reveal.revealed_index].slashed = True -# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) -# return pre_state, randao_key_reveal, post_state + return pre_state, randao_key_reveal, post_state -# def test_far_future_epoch(state): -# randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS) +def test_far_future_epoch(state): + randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS) -# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) -# return pre_state, randao_key_reveal, post_state + return pre_state, randao_key_reveal, post_state diff --git a/test_libs/pyspec/tests/phase1/helpers.py b/test_libs/pyspec/tests/phase1/helpers.py index e69de29bb2..b12519b627 100644 --- a/test_libs/pyspec/tests/phase1/helpers.py +++ b/test_libs/pyspec/tests/phase1/helpers.py @@ -0,0 +1,54 @@ +from py_ecc import bls + +import eth2spec.phase1.spec as spec +from eth2spec.phase1.spec import ( + # constants + ZERO_HASH, + CUSTODY_PERIOD_TO_RANDAO_PADDING, + # SSZ + EarlyDerivedSecretReveal, + # functions + get_active_validator_indices, + get_current_epoch, + get_domain, + hash_tree_root, +) + +from tests.phase0.helpers import ( + privkeys, +) + +def get_valid_early_derived_secret_reveal(state, epoch=None): + current_epoch = get_current_epoch(state) + revealed_index = get_active_validator_indices(state, current_epoch)[-1] + masker_index = get_active_validator_indices(state, current_epoch)[0] + + if epoch is None: + epoch = current_epoch + CUSTODY_PERIOD_TO_RANDAO_PADDING + + reveal = bls.sign( + message_hash=hash_tree_root(epoch), + privkey=privkeys[revealed_index], + domain=get_domain( + state=state, + domain_type=spec.DOMAIN_RANDAO, + message_epoch=epoch, + ), + ) + mask = bls.sign( + message_hash=hash_tree_root(epoch), + privkey=privkeys[masker_index], + domain=get_domain( + state=state, + domain_type=spec.DOMAIN_RANDAO, + message_epoch=epoch, + ), + ) + + return EarlyDerivedSecretReveal( + revealed_index=revealed_index, + epoch=epoch, + reveal=reveal, + masker_index=masker_index, + mask=mask, + ) From becda4f463d6e7091ff95e6645afd44b2bc597dc Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Wed, 15 May 2019 16:26:49 +0200 Subject: [PATCH 112/308] Several changes before I break everything --- test_libs/pyspec/tests/phase0/conftest.py | 5 +- ...est_process_early_derived_secret_reveal.py | 18 ++--- test_libs/pyspec/tests/phase1/conftest.py | 21 +++++- test_libs/pyspec/tests/phase1/helpers.py | 11 +++ .../pyspec/tests/phase1/test_finality.py | 3 - test_libs/pyspec/tests/phase1/test_sanity.py | 68 +++++++++++++++++++ 6 files changed, 108 insertions(+), 18 deletions(-) create mode 100644 test_libs/pyspec/tests/phase1/test_sanity.py diff --git a/test_libs/pyspec/tests/phase0/conftest.py b/test_libs/pyspec/tests/phase0/conftest.py index 4deb65da8f..a3ea0a00c1 100644 --- a/test_libs/pyspec/tests/phase0/conftest.py +++ b/test_libs/pyspec/tests/phase0/conftest.py @@ -10,10 +10,9 @@ def pytest_addoption(parser): parser.addoption( - "--config", action="store", default="minimal", help="config: make the p yspec use the specified configuration" + "--config", action="store", default="minimal", help="config: make the pyspec use the specified configuration" ) - @pytest.fixture(autouse=True) def config(request): config_name = request.config.getoption("--config") @@ -24,12 +23,10 @@ def config(request): def num_validators(config): return spec.SLOTS_PER_EPOCH * 8 - @pytest.fixture def deposit_data_leaves(): return list() - @pytest.fixture def state(num_validators, deposit_data_leaves): return create_genesis_state(num_validators, deposit_data_leaves) diff --git a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py index 54a90aca19..6661e883fb 100644 --- a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py @@ -61,14 +61,14 @@ def test_reveal_from_current_epoch(state): return pre_state, randao_key_reveal, post_state -# Not currently possible as we are testing at epoch 0 -# -#def test_reveal_from_past_epoch(state): -# randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) - 1) -# -# pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) -# -# return pre_state, randao_key_reveal, post_state + +@pytest.mark.skip(reason="Not currently possible as we are testing at epoch 0") +def test_reveal_from_past_epoch(state): + randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) - 1) + + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + + return pre_state, randao_key_reveal, post_state def test_reveal_with_custody_padding(state): randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING) @@ -88,7 +88,7 @@ def test_double_reveal(state): pre_state, intermediate_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal1) randao_key_reveal2 = get_valid_early_derived_secret_reveal(intermediate_state, get_current_epoch(pre_state) + RANDAO_PENALTY_EPOCHS + 1) - intermediate_state_, post_state = run_early_derived_secret_reveal_processing(intermediate_state, randao_key_reveal2, False) + _, post_state = run_early_derived_secret_reveal_processing(intermediate_state, randao_key_reveal2, False) return pre_state, [randao_key_reveal1, randao_key_reveal2], post_state diff --git a/test_libs/pyspec/tests/phase1/conftest.py b/test_libs/pyspec/tests/phase1/conftest.py index 4bacc1ea7f..c73f25c1b1 100644 --- a/test_libs/pyspec/tests/phase1/conftest.py +++ b/test_libs/pyspec/tests/phase1/conftest.py @@ -1,9 +1,26 @@ import pytest +from eth2spec.phase1 import spec +from preset_loader import loader + +from tests.phase0.helpers import ( + create_genesis_state, +) + from tests.phase0.conftest import ( pytest_addoption, - config, num_validators, deposit_data_leaves, - state, ) + +# This is redfined so that the constants are re-applied +@pytest.fixture(autouse=True) +def config(request): + config_name = request.config.getoption("--config") + presets = loader.load_presets('../../configs/', config_name) + spec.apply_constants_preset(presets) + +#This is redefined so that the BeaconState is the new SSZ Object +@pytest.fixture +def state(num_validators, deposit_data_leaves): + return create_genesis_state(num_validators, deposit_data_leaves) diff --git a/test_libs/pyspec/tests/phase1/helpers.py b/test_libs/pyspec/tests/phase1/helpers.py index b12519b627..8520ce9d75 100644 --- a/test_libs/pyspec/tests/phase1/helpers.py +++ b/test_libs/pyspec/tests/phase1/helpers.py @@ -15,7 +15,18 @@ ) from tests.phase0.helpers import ( + advance_slot, + get_balance, + build_deposit_data, + build_empty_block_for_next_slot, + fill_aggregate_attestation, + get_state_root, + get_valid_attestation, + get_valid_attester_slashing, + get_valid_proposer_slashing, + next_slot, privkeys, + pubkeys, ) def get_valid_early_derived_secret_reveal(state, epoch=None): diff --git a/test_libs/pyspec/tests/phase1/test_finality.py b/test_libs/pyspec/tests/phase1/test_finality.py index b0acaa7cb2..4b57c94935 100644 --- a/test_libs/pyspec/tests/phase1/test_finality.py +++ b/test_libs/pyspec/tests/phase1/test_finality.py @@ -17,9 +17,6 @@ next_epoch, ) -# mark entire file as 'state' -# pytestmark = pytest.mark.state - from tests.phase0.test_finality import ( pytestmark, check_finality, diff --git a/test_libs/pyspec/tests/phase1/test_sanity.py b/test_libs/pyspec/tests/phase1/test_sanity.py new file mode 100644 index 0000000000..45a08bd0e3 --- /dev/null +++ b/test_libs/pyspec/tests/phase1/test_sanity.py @@ -0,0 +1,68 @@ +from copy import deepcopy + +import pytest + +from py_ecc import bls +import eth2spec.phase1.spec as spec + +from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.phase1.spec import ( + # constants + ZERO_HASH, + SLOTS_PER_HISTORICAL_ROOT, + # SSZ + Deposit, + Transfer, + VoluntaryExit, + # functions + get_active_validator_indices, + get_beacon_proposer_index, + get_block_root_at_slot, + get_current_epoch, + get_domain, + process_slot, + verify_merkle_branch, + state_transition, + hash, +) +from eth2spec.utils.merkle_minimal import ( + calc_merkle_tree_from_leaves, + get_merkle_proof, + get_merkle_root, +) +from .helpers import ( + advance_slot, + get_balance, + build_deposit_data, + build_empty_block_for_next_slot, + fill_aggregate_attestation, + get_state_root, + get_valid_attestation, + get_valid_attester_slashing, + get_valid_proposer_slashing, + next_slot, + privkeys, + pubkeys, +) + + +# mark entire file as 'sanity' +pytestmark = pytest.mark.sanity + +from tests.phase0.test_sanity import ( + test_slot_transition, + test_empty_block_transition, + test_skipped_slots, + test_empty_epoch_transition, + test_empty_epoch_transition_not_finalizing, + test_proposer_slashing, + test_attester_slashing, + test_deposit_in_block, + test_deposit_top_up, + test_attestation, + test_voluntary_exit, + test_transfer, + test_balance_driven_status_transitions, + test_historical_batch, + test_eth1_data_votes, +) From 2885f853c28d2442b4507b80d24471292a41ddb2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 16:34:07 +0800 Subject: [PATCH 113/308] clean up lint --- .circleci/config.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 439afc9a95..3fe6643b74 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,7 +96,7 @@ jobs: reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter - command: make install_lint && make pyspec && make lint + command: make install_lint && make lint workflows: version: 2.1 test_spec: diff --git a/Makefile b/Makefile index 8cc889f214..a6b379b719 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ citest: $(PY_SPEC_ALL_TARGETS) install_lint: cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.5.0 -lint: +lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ flake8 --max-line-length=120 ./eth2spec; From 1b2e7c1b6eab6a8b1b28e7fb87ed67b4c8d0a092 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 16 May 2019 11:07:21 +0200 Subject: [PATCH 114/308] Passes spec & helpers module to tests phase0 --- .../test_process_attestation.py | 97 +++---- .../test_process_attester_slashing.py | 66 ++--- .../test_process_block_header.py | 55 ++-- .../block_processing/test_process_deposit.py | 69 ++--- .../test_process_proposer_slashing.py | 53 ++-- .../block_processing/test_process_transfer.py | 83 +++--- .../block_processing/test_voluntary_exit.py | 89 +++---- test_libs/pyspec/tests/phase0/conftest.py | 21 +- .../test_process_crosslinks.py | 89 +++---- .../test_process_registry_updates.py | 32 +-- .../pyspec/tests/phase0/test_finality.py | 69 ++--- test_libs/pyspec/tests/phase0/test_sanity.py | 239 ++++++++---------- 12 files changed, 404 insertions(+), 558 deletions(-) diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_attestation.py index 441df21e23..5dcf17e383 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_attestation.py @@ -1,41 +1,26 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec - -from eth2spec.phase0.spec import ( - get_current_epoch, - process_attestation, - slot_to_epoch, - state_transition, -) -from tests.phase0.helpers import ( - build_empty_block_for_next_slot, - get_valid_attestation, - next_epoch, - next_slot, -) - # mark entire file as 'attestations' pytestmark = pytest.mark.attestations -def run_attestation_processing(state, attestation, valid=True): +def run_attestation_processing(spec, helpers, state, attestation, valid=True): """ - Run ``process_attestation`` returning the pre and post state. + Run ``spec.process_attestation`` returning the pre and post state. If ``valid == False``, run expecting ``AssertionError`` """ post_state = deepcopy(state) if not valid: with pytest.raises(AssertionError): - process_attestation(post_state, attestation) + spec.process_attestation(post_state, attestation) return state, None - process_attestation(post_state, attestation) + spec.process_attestation(post_state, attestation) - current_epoch = get_current_epoch(state) + current_epoch = spec.get_current_epoch(state) if attestation.data.target_epoch == current_epoch: assert len(post_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1 else: @@ -44,110 +29,110 @@ def run_attestation_processing(state, attestation, valid=True): return state, post_state -def test_success(state): - attestation = get_valid_attestation(state) +def test_success(spec, helpers, state): + attestation = helpers.get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - pre_state, post_state = run_attestation_processing(state, attestation) + pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation) return pre_state, attestation, post_state -def test_success_prevous_epoch(state): - attestation = get_valid_attestation(state) - block = build_empty_block_for_next_slot(state) +def test_success_prevous_epoch(spec, helpers, state): + attestation = helpers.get_valid_attestation(state) + block = helpers.build_empty_block_for_next_slot(state) block.slot = state.slot + spec.SLOTS_PER_EPOCH - state_transition(state, block) + spec.state_transition(state, block) - pre_state, post_state = run_attestation_processing(state, attestation) + pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation) return pre_state, attestation, post_state -def test_before_inclusion_delay(state): - attestation = get_valid_attestation(state) +def test_before_inclusion_delay(spec, helpers, state): + attestation = helpers.get_valid_attestation(state) # do not increment slot to allow for inclusion delay - pre_state, post_state = run_attestation_processing(state, attestation, False) + pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation, False) return pre_state, attestation, post_state -def test_after_epoch_slots(state): - attestation = get_valid_attestation(state) - block = build_empty_block_for_next_slot(state) +def test_after_epoch_slots(spec, helpers, state): + attestation = helpers.get_valid_attestation(state) + block = helpers.build_empty_block_for_next_slot(state) # increment past latest inclusion slot block.slot = state.slot + spec.SLOTS_PER_EPOCH + 1 - state_transition(state, block) + spec.state_transition(state, block) - pre_state, post_state = run_attestation_processing(state, attestation, False) + pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation, False) return pre_state, attestation, post_state -def test_bad_source_epoch(state): - attestation = get_valid_attestation(state) +def test_bad_source_epoch(spec, helpers, state): + attestation = helpers.get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.source_epoch += 10 - pre_state, post_state = run_attestation_processing(state, attestation, False) + pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation, False) return pre_state, attestation, post_state -def test_bad_source_root(state): - attestation = get_valid_attestation(state) +def test_bad_source_root(spec, helpers, state): + attestation = helpers.get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.source_root = b'\x42' * 32 - pre_state, post_state = run_attestation_processing(state, attestation, False) + pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation, False) return pre_state, attestation, post_state -def test_non_zero_crosslink_data_root(state): - attestation = get_valid_attestation(state) +def test_non_zero_crosslink_data_root(spec, helpers, state): + attestation = helpers.get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.crosslink.data_root = b'\x42' * 32 - pre_state, post_state = run_attestation_processing(state, attestation, False) + pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation, False) return pre_state, attestation, post_state -def test_bad_previous_crosslink(state): - next_epoch(state) - attestation = get_valid_attestation(state) +def test_bad_previous_crosslink(spec, helpers, state): + helpers.next_epoch(state) + attestation = helpers.get_valid_attestation(state) for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): - next_slot(state) + helpers.next_slot(state) state.current_crosslinks[attestation.data.crosslink.shard].epoch += 10 - pre_state, post_state = run_attestation_processing(state, attestation, False) + pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation, False) return pre_state, attestation, post_state -def test_non_empty_custody_bitfield(state): - attestation = get_valid_attestation(state) +def test_non_empty_custody_bitfield(spec, helpers, state): + attestation = helpers.get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) - pre_state, post_state = run_attestation_processing(state, attestation, False) + pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation, False) return pre_state, attestation, post_state -def test_empty_aggregation_bitfield(state): - attestation = get_valid_attestation(state) +def test_empty_aggregation_bitfield(spec, helpers, state): + attestation = helpers.get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.aggregation_bitfield = b'\x00' * len(attestation.aggregation_bitfield) - pre_state, post_state = run_attestation_processing(state, attestation) + pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation) return pre_state, attestation, post_state diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_attester_slashing.py index d5e150b1d2..ed48b17cff 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_attester_slashing.py @@ -1,34 +1,24 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec -from eth2spec.phase0.spec import ( - get_beacon_proposer_index, - process_attester_slashing, -) -from tests.phase0.helpers import ( - get_balance, - get_valid_attester_slashing, - next_epoch, -) # mark entire file as 'attester_slashing' pytestmark = pytest.mark.attester_slashings -def run_attester_slashing_processing(state, attester_slashing, valid=True): +def run_attester_slashing_processing(spec, helpers, state, attester_slashing, valid=True): """ - Run ``process_attester_slashing`` returning the pre and post state. + Run ``spec.process_attester_slashing`` returning the pre and post state. If ``valid == False``, run expecting ``AssertionError`` """ post_state = deepcopy(state) if not valid: with pytest.raises(AssertionError): - process_attester_slashing(post_state, attester_slashing) + spec.process_attester_slashing(post_state, attester_slashing) return state, None - process_attester_slashing(post_state, attester_slashing) + spec.process_attester_slashing(post_state, attester_slashing) slashed_index = attester_slashing.attestation_1.custody_bit_0_indices[0] slashed_validator = post_state.validator_registry[slashed_index] @@ -37,63 +27,63 @@ def run_attester_slashing_processing(state, attester_slashing, valid=True): assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH # lost whistleblower reward assert ( - get_balance(post_state, slashed_index) < - get_balance(state, slashed_index) + helpers.get_balance(post_state, slashed_index) < + helpers.get_balance(state, slashed_index) ) - proposer_index = get_beacon_proposer_index(state) + proposer_index = spec.get_beacon_proposer_index(state) # gained whistleblower reward assert ( - get_balance(post_state, proposer_index) > - get_balance(state, proposer_index) + helpers.get_balance(post_state, proposer_index) > + helpers.get_balance(state, proposer_index) ) return state, post_state -def test_success_double(state): - attester_slashing = get_valid_attester_slashing(state) +def test_success_double(spec, helpers, state): + attester_slashing = helpers.get_valid_attester_slashing(state) - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing) + pre_state, post_state = run_attester_slashing_processing(spec, helpers, state, attester_slashing) return pre_state, attester_slashing, post_state -def test_success_surround(state): - next_epoch(state) +def test_success_surround(spec, helpers, state): + helpers.next_epoch(state) state.current_justified_epoch += 1 - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = helpers.get_valid_attester_slashing(state) # set attestion1 to surround attestation 2 attester_slashing.attestation_1.data.source_epoch = attester_slashing.attestation_2.data.source_epoch - 1 attester_slashing.attestation_1.data.target_epoch = attester_slashing.attestation_2.data.target_epoch + 1 - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing) + pre_state, post_state = run_attester_slashing_processing(spec, helpers, state, attester_slashing) return pre_state, attester_slashing, post_state -def test_same_data(state): - attester_slashing = get_valid_attester_slashing(state) +def test_same_data(spec, helpers, state): + attester_slashing = helpers.get_valid_attester_slashing(state) attester_slashing.attestation_1.data = attester_slashing.attestation_2.data - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) + pre_state, post_state = run_attester_slashing_processing(spec, helpers, state, attester_slashing, False) return pre_state, attester_slashing, post_state -def test_no_double_or_surround(state): - attester_slashing = get_valid_attester_slashing(state) +def test_no_double_or_surround(spec, helpers, state): + attester_slashing = helpers.get_valid_attester_slashing(state) attester_slashing.attestation_1.data.target_epoch += 1 - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) + pre_state, post_state = run_attester_slashing_processing(spec, helpers, state, attester_slashing, False) return pre_state, attester_slashing, post_state -def test_participants_already_slashed(state): - attester_slashing = get_valid_attester_slashing(state) +def test_participants_already_slashed(spec, helpers, state): + attester_slashing = helpers.get_valid_attester_slashing(state) # set all indices to slashed attestation_1 = attester_slashing.attestation_1 @@ -101,17 +91,17 @@ def test_participants_already_slashed(state): for index in validator_indices: state.validator_registry[index].slashed = True - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) + pre_state, post_state = run_attester_slashing_processing(spec, helpers, state, attester_slashing, False) return pre_state, attester_slashing, post_state -def test_custody_bit_0_and_1(state): - attester_slashing = get_valid_attester_slashing(state) +def test_custody_bit_0_and_1(spec, helpers, state): + attester_slashing = helpers.get_valid_attester_slashing(state) attester_slashing.attestation_1.custody_bit_1_indices = ( attester_slashing.attestation_1.custody_bit_0_indices ) - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) + pre_state, post_state = run_attester_slashing_processing(spec, helpers, state, attester_slashing, False) return pre_state, attester_slashing, post_state diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_block_header.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_block_header.py index 4b66c2ff06..e4cd70f770 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_block_header.py @@ -2,75 +2,64 @@ import pytest -from eth2spec.phase0.spec import ( - get_beacon_proposer_index, - process_slot, - process_block_header, -) -from tests.phase0.helpers import ( - advance_slot, - build_empty_block_for_next_slot, - next_slot, -) - # mark entire file as 'header' pytestmark = pytest.mark.header -def prepare_state_for_header_processing(state): - process_slot(state) - advance_slot(state) +def prepare_state_for_header_processing(spec, helpers, state): + spec.process_slot(state) + helpers.advance_slot(state) -def run_block_header_processing(state, block, valid=True): +def run_block_header_processing(spec, helpers, state, block, valid=True): """ - Run ``process_block_header`` returning the pre and post state. + Run ``spec.process_block_header`` returning the pre and post state. If ``valid == False``, run expecting ``AssertionError`` """ - prepare_state_for_header_processing(state) + prepare_state_for_header_processing(spec, helpers, state) post_state = deepcopy(state) if not valid: with pytest.raises(AssertionError): - process_block_header(post_state, block) + spec.process_block_header(post_state, block) return state, None - process_block_header(post_state, block) + spec.process_block_header(post_state, block) return state, post_state -def test_success(state): - block = build_empty_block_for_next_slot(state) - pre_state, post_state = run_block_header_processing(state, block) +def test_success(spec, helpers, state): + block = helpers.build_empty_block_for_next_slot(state) + pre_state, post_state = run_block_header_processing(spec, helpers, state, block) return state, block, post_state -def test_invalid_slot(state): - block = build_empty_block_for_next_slot(state) +def test_invalid_slot(spec, helpers, state): + block = helpers.build_empty_block_for_next_slot(state) block.slot = state.slot + 2 # invalid slot - pre_state, post_state = run_block_header_processing(state, block, valid=False) + pre_state, post_state = run_block_header_processing(spec, helpers, state, block, valid=False) return pre_state, block, None -def test_invalid_parent_block_root(state): - block = build_empty_block_for_next_slot(state) +def test_invalid_parent_block_root(spec, helpers, state): + block = helpers.build_empty_block_for_next_slot(state) block.parent_root = b'\12' * 32 # invalid prev root - pre_state, post_state = run_block_header_processing(state, block, valid=False) + pre_state, post_state = run_block_header_processing(spec, helpers, state, block, valid=False) return pre_state, block, None -def test_proposer_slashed(state): +def test_proposer_slashed(spec, helpers, state): # use stub state to get proposer index of next slot stub_state = deepcopy(state) - next_slot(stub_state) - proposer_index = get_beacon_proposer_index(stub_state) + helpers.next_slot(stub_state) + proposer_index = spec.get_beacon_proposer_index(stub_state) # set proposer to slashed state.validator_registry[proposer_index].slashed = True - block = build_empty_block_for_next_slot(state) + block = helpers.build_empty_block_for_next_slot(state) - pre_state, post_state = run_block_header_processing(state, block, valid=False) + pre_state, post_state = run_block_header_processing(spec, helpers, state, block, valid=False) return pre_state, block, None diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_deposit.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_deposit.py index 0aa057c148..97d49114b6 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_deposit.py @@ -1,33 +1,20 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec - -from eth2spec.phase0.spec import ( - ZERO_HASH, - process_deposit, -) -from tests.phase0.helpers import ( - get_balance, - build_deposit, - privkeys, - pubkeys, -) - # mark entire file as 'deposits' pytestmark = pytest.mark.deposits -def test_success(state): +def test_success(spec, helpers, state): pre_state = deepcopy(state) # fill previous deposits with zero-hash - deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) + deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) index = len(deposit_data_leaves) - pubkey = pubkeys[index] - privkey = privkeys[index] - deposit, root, deposit_data_leaves = build_deposit( + pubkey = helpers.pubkeys[index] + privkey = helpers.privkeys[index] + deposit, root, deposit_data_leaves = helpers.build_deposit( pre_state, deposit_data_leaves, pubkey, @@ -40,26 +27,26 @@ def test_success(state): post_state = deepcopy(pre_state) - process_deposit(post_state, deposit) + spec.process_deposit(post_state, deposit) assert len(post_state.validator_registry) == len(state.validator_registry) + 1 assert len(post_state.balances) == len(state.balances) + 1 - assert post_state.validator_registry[index].pubkey == pubkeys[index] - assert get_balance(post_state, index) == spec.MAX_EFFECTIVE_BALANCE + assert post_state.validator_registry[index].pubkey == helpers.pubkeys[index] + assert helpers.get_balance(post_state, index) == spec.MAX_EFFECTIVE_BALANCE assert post_state.deposit_index == post_state.latest_eth1_data.deposit_count return pre_state, deposit, post_state -def test_success_top_up(state): +def test_success_top_up(spec, helpers, state): pre_state = deepcopy(state) - deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) + deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) validator_index = 0 amount = spec.MAX_EFFECTIVE_BALANCE // 4 - pubkey = pubkeys[validator_index] - privkey = privkeys[validator_index] - deposit, root, deposit_data_leaves = build_deposit( + pubkey = helpers.pubkeys[validator_index] + privkey = helpers.privkeys[validator_index] + deposit, root, deposit_data_leaves = helpers.build_deposit( pre_state, deposit_data_leaves, pubkey, @@ -69,28 +56,28 @@ def test_success_top_up(state): pre_state.latest_eth1_data.deposit_root = root pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - pre_balance = get_balance(pre_state, validator_index) + pre_balance = helpers.get_balance(pre_state, validator_index) post_state = deepcopy(pre_state) - process_deposit(post_state, deposit) + spec.process_deposit(post_state, deposit) assert len(post_state.validator_registry) == len(state.validator_registry) assert len(post_state.balances) == len(state.balances) assert post_state.deposit_index == post_state.latest_eth1_data.deposit_count - assert get_balance(post_state, validator_index) == pre_balance + amount + assert helpers.get_balance(post_state, validator_index) == pre_balance + amount return pre_state, deposit, post_state -def test_wrong_index(state): +def test_wrong_index(spec, helpers, state): pre_state = deepcopy(state) - deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) + deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) index = len(deposit_data_leaves) - pubkey = pubkeys[index] - privkey = privkeys[index] - deposit, root, deposit_data_leaves = build_deposit( + pubkey = helpers.pubkeys[index] + privkey = helpers.privkeys[index] + deposit, root, deposit_data_leaves = helpers.build_deposit( pre_state, deposit_data_leaves, pubkey, @@ -107,19 +94,19 @@ def test_wrong_index(state): post_state = deepcopy(pre_state) with pytest.raises(AssertionError): - process_deposit(post_state, deposit) + spec.process_deposit(post_state, deposit) return pre_state, deposit, None -def test_bad_merkle_proof(state): +def test_bad_merkle_proof(spec, helpers, state): pre_state = deepcopy(state) - deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) + deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) index = len(deposit_data_leaves) - pubkey = pubkeys[index] - privkey = privkeys[index] - deposit, root, deposit_data_leaves = build_deposit( + pubkey = helpers.pubkeys[index] + privkey = helpers.privkeys[index] + deposit, root, deposit_data_leaves = helpers.build_deposit( pre_state, deposit_data_leaves, pubkey, @@ -136,6 +123,6 @@ def test_bad_merkle_proof(state): post_state = deepcopy(pre_state) with pytest.raises(AssertionError): - process_deposit(post_state, deposit) + spec.process_deposit(post_state, deposit) return pre_state, deposit, None diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_proposer_slashing.py index 366417d8b3..6f82d6c33e 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_proposer_slashing.py @@ -1,33 +1,24 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec -from eth2spec.phase0.spec import ( - get_current_epoch, - process_proposer_slashing, -) -from tests.phase0.helpers import ( - get_balance, - get_valid_proposer_slashing, -) # mark entire file as 'proposer_slashings' pytestmark = pytest.mark.proposer_slashings -def run_proposer_slashing_processing(state, proposer_slashing, valid=True): +def run_proposer_slashing_processing(spec, helpers, state, proposer_slashing, valid=True): """ - Run ``process_proposer_slashing`` returning the pre and post state. + Run ``spec.process_proposer_slashing`` returning the pre and post state. If ``valid == False``, run expecting ``AssertionError`` """ post_state = deepcopy(state) if not valid: with pytest.raises(AssertionError): - process_proposer_slashing(post_state, proposer_slashing) + spec.process_proposer_slashing(post_state, proposer_slashing) return state, None - process_proposer_slashing(post_state, proposer_slashing) + spec.process_proposer_slashing(post_state, proposer_slashing) slashed_validator = post_state.validator_registry[proposer_slashing.proposer_index] assert slashed_validator.slashed @@ -35,62 +26,62 @@ def run_proposer_slashing_processing(state, proposer_slashing, valid=True): assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH # lost whistleblower reward assert ( - get_balance(post_state, proposer_slashing.proposer_index) < - get_balance(state, proposer_slashing.proposer_index) + helpers.get_balance(post_state, proposer_slashing.proposer_index) < + helpers.get_balance(state, proposer_slashing.proposer_index) ) return state, post_state -def test_success(state): - proposer_slashing = get_valid_proposer_slashing(state) +def test_success(spec, helpers, state): + proposer_slashing = helpers.get_valid_proposer_slashing(state) - pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing) + pre_state, post_state = run_proposer_slashing_processing(spec, helpers, state, proposer_slashing) return pre_state, proposer_slashing, post_state -def test_epochs_are_different(state): - proposer_slashing = get_valid_proposer_slashing(state) +def test_epochs_are_different(spec, helpers, state): + proposer_slashing = helpers.get_valid_proposer_slashing(state) # set slots to be in different epochs proposer_slashing.header_2.slot += spec.SLOTS_PER_EPOCH - pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) + pre_state, post_state = run_proposer_slashing_processing(spec, helpers, state, proposer_slashing, False) return pre_state, proposer_slashing, post_state -def test_headers_are_same(state): - proposer_slashing = get_valid_proposer_slashing(state) +def test_headers_are_same(spec, helpers, state): + proposer_slashing = helpers.get_valid_proposer_slashing(state) # set headers to be the same proposer_slashing.header_2 = proposer_slashing.header_1 - pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) + pre_state, post_state = run_proposer_slashing_processing(spec, helpers, state, proposer_slashing, False) return pre_state, proposer_slashing, post_state -def test_proposer_is_slashed(state): - proposer_slashing = get_valid_proposer_slashing(state) +def test_proposer_is_slashed(spec, helpers, state): + proposer_slashing = helpers.get_valid_proposer_slashing(state) # set proposer to slashed state.validator_registry[proposer_slashing.proposer_index].slashed = True - pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) + pre_state, post_state = run_proposer_slashing_processing(spec, helpers, state, proposer_slashing, False) return pre_state, proposer_slashing, post_state -def test_proposer_is_withdrawn(state): - proposer_slashing = get_valid_proposer_slashing(state) +def test_proposer_is_withdrawn(spec, helpers, state): + proposer_slashing = helpers.get_valid_proposer_slashing(state) # set proposer withdrawable_epoch in past - current_epoch = get_current_epoch(state) + current_epoch = spec.get_current_epoch(state) proposer_index = proposer_slashing.proposer_index state.validator_registry[proposer_index].withdrawable_epoch = current_epoch - 1 - pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) + pre_state, post_state = run_proposer_slashing_processing(spec, helpers, state, proposer_slashing, False) return pre_state, proposer_slashing, post_state diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py index cb657bb9de..197898ec0f 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py @@ -1,40 +1,27 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec - -from eth2spec.phase0.spec import ( - get_active_validator_indices, - get_beacon_proposer_index, - get_current_epoch, - process_transfer, -) -from tests.phase0.helpers import ( - get_valid_transfer, - next_epoch, -) - # mark entire file as 'transfers' pytestmark = pytest.mark.transfers -def run_transfer_processing(state, transfer, valid=True): +def run_transfer_processing(spec, helpers, state, transfer, valid=True): """ - Run ``process_transfer`` returning the pre and post state. + Run ``spec.process_transfer`` returning the pre and post state. If ``valid == False``, run expecting ``AssertionError`` """ post_state = deepcopy(state) if not valid: with pytest.raises(AssertionError): - process_transfer(post_state, transfer) + spec.process_transfer(post_state, transfer) return state, None - process_transfer(post_state, transfer) + spec.process_transfer(post_state, transfer) - proposer_index = get_beacon_proposer_index(state) + proposer_index = spec.get_beacon_proposer_index(state) pre_transfer_sender_balance = state.balances[transfer.sender] pre_transfer_recipient_balance = state.balances[transfer.recipient] pre_transfer_proposer_balance = state.balances[proposer_index] @@ -47,95 +34,95 @@ def run_transfer_processing(state, transfer, valid=True): return state, post_state -def test_success_non_activated(state): - transfer = get_valid_transfer(state) +def test_success_non_activated(spec, helpers, state): + transfer = helpers.get_valid_transfer(state) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - pre_state, post_state = run_transfer_processing(state, transfer) + pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer) return pre_state, transfer, post_state -def test_success_withdrawable(state): - next_epoch(state) +def test_success_withdrawable(spec, helpers, state): + helpers.next_epoch(state) - transfer = get_valid_transfer(state) + transfer = helpers.get_valid_transfer(state) # withdrawable_epoch in past so can transfer - state.validator_registry[transfer.sender].withdrawable_epoch = get_current_epoch(state) - 1 + state.validator_registry[transfer.sender].withdrawable_epoch = spec.get_current_epoch(state) - 1 - pre_state, post_state = run_transfer_processing(state, transfer) + pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer) return pre_state, transfer, post_state -def test_success_active_above_max_effective(state): - sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] +def test_success_active_above_max_effective(spec, helpers, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE // 32 state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + amount - transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) + transfer = helpers.get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) - pre_state, post_state = run_transfer_processing(state, transfer) + pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer) return pre_state, transfer, post_state -def test_active_but_transfer_past_effective_balance(state): - sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] +def test_active_but_transfer_past_effective_balance(spec, helpers, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE // 32 state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE - transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) + transfer = helpers.get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) - pre_state, post_state = run_transfer_processing(state, transfer, False) + pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer, False) return pre_state, transfer, post_state -def test_incorrect_slot(state): - transfer = get_valid_transfer(state, slot=state.slot+1) +def test_incorrect_slot(spec, helpers, state): + transfer = helpers.get_valid_transfer(state, slot=state.slot+1) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - pre_state, post_state = run_transfer_processing(state, transfer, False) + pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer, False) return pre_state, transfer, post_state -def test_insufficient_balance(state): - sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] +def test_insufficient_balance(spec, helpers, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE - transfer = get_valid_transfer(state, sender_index=sender_index, amount=amount + 1, fee=0) + transfer = helpers.get_valid_transfer(state, sender_index=sender_index, amount=amount + 1, fee=0) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - pre_state, post_state = run_transfer_processing(state, transfer, False) + pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer, False) return pre_state, transfer, post_state -def test_no_dust(state): - sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] +def test_no_dust(spec, helpers, state): + sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] balance = state.balances[sender_index] - transfer = get_valid_transfer(state, sender_index=sender_index, amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, fee=0) + transfer = helpers.get_valid_transfer(state, sender_index=sender_index, amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, fee=0) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - pre_state, post_state = run_transfer_processing(state, transfer, False) + pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer, False) return pre_state, transfer, post_state -def test_invalid_pubkey(state): - transfer = get_valid_transfer(state) +def test_invalid_pubkey(spec, helpers, state): + transfer = helpers.get_valid_transfer(state) state.validator_registry[transfer.sender].withdrawal_credentials = spec.ZERO_HASH # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - pre_state, post_state = run_transfer_processing(state, transfer, False) + pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer, False) return pre_state, transfer, post_state diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_voluntary_exit.py b/test_libs/pyspec/tests/phase0/block_processing/test_voluntary_exit.py index 63be4c8f09..d9e643b559 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_voluntary_exit.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_voluntary_exit.py @@ -1,37 +1,24 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec - -from eth2spec.phase0.spec import ( - get_active_validator_indices, - get_churn_limit, - get_current_epoch, - process_voluntary_exit, -) -from tests.phase0.helpers import ( - build_voluntary_exit, - pubkey_to_privkey, -) - # mark entire file as 'voluntary_exits' pytestmark = pytest.mark.voluntary_exits -def run_voluntary_exit_processing(state, voluntary_exit, valid=True): +def run_voluntary_exit_processing(spec, helpers, state, voluntary_exit, valid=True): """ - Run ``process_voluntary_exit`` returning the pre and post state. + Run ``spec.process_voluntary_exit`` returning the pre and post state. If ``valid == False``, run expecting ``AssertionError`` """ post_state = deepcopy(state) if not valid: with pytest.raises(AssertionError): - process_voluntary_exit(post_state, voluntary_exit) + spec.process_voluntary_exit(post_state, voluntary_exit) return state, None - process_voluntary_exit(post_state, voluntary_exit) + spec.process_voluntary_exit(post_state, voluntary_exit) validator_index = voluntary_exit.validator_index assert state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH @@ -40,56 +27,56 @@ def run_voluntary_exit_processing(state, voluntary_exit, valid=True): return state, post_state -def test_success(state): +def test_success(spec, helpers, state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - current_epoch = get_current_epoch(state) - validator_index = get_active_validator_indices(state, current_epoch)[0] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + current_epoch = spec.get_current_epoch(state) + validator_index = spec.get_active_validator_indices(state, current_epoch)[0] + privkey = helpers.pubkey_to_privkey[state.validator_registry[validator_index].pubkey] - voluntary_exit = build_voluntary_exit( + voluntary_exit = helpers.build_voluntary_exit( state, current_epoch, validator_index, privkey, ) - pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit) + pre_state, post_state = run_voluntary_exit_processing(spec, helpers, state, voluntary_exit) return pre_state, voluntary_exit, post_state -def test_success_exit_queue(state): +def test_success_exit_queue(spec, helpers, state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - current_epoch = get_current_epoch(state) + current_epoch = spec.get_current_epoch(state) # exit `MAX_EXITS_PER_EPOCH` - initial_indices = get_active_validator_indices(state, current_epoch)[:get_churn_limit(state)] + initial_indices = spec.get_active_validator_indices(state, current_epoch)[:spec.get_churn_limit(state)] post_state = state for index in initial_indices: - privkey = pubkey_to_privkey[state.validator_registry[index].pubkey] - voluntary_exit = build_voluntary_exit( + privkey = helpers.pubkey_to_privkey[state.validator_registry[index].pubkey] + voluntary_exit = helpers.build_voluntary_exit( state, current_epoch, index, privkey, ) - pre_state, post_state = run_voluntary_exit_processing(post_state, voluntary_exit) + pre_state, post_state = run_voluntary_exit_processing(spec, helpers, post_state, voluntary_exit) # exit an additional validator - validator_index = get_active_validator_indices(state, current_epoch)[-1] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] - voluntary_exit = build_voluntary_exit( + validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] + privkey = helpers.pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + voluntary_exit = helpers.build_voluntary_exit( state, current_epoch, validator_index, privkey, ) - pre_state, post_state = run_voluntary_exit_processing(post_state, voluntary_exit) + pre_state, post_state = run_voluntary_exit_processing(spec, helpers, post_state, voluntary_exit) assert ( post_state.validator_registry[validator_index].exit_epoch == @@ -99,55 +86,55 @@ def test_success_exit_queue(state): return pre_state, voluntary_exit, post_state -def test_validator_not_active(state): - current_epoch = get_current_epoch(state) - validator_index = get_active_validator_indices(state, current_epoch)[0] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] +def test_validator_not_active(spec, helpers, state): + current_epoch = spec.get_current_epoch(state) + validator_index = spec.get_active_validator_indices(state, current_epoch)[0] + privkey = helpers.pubkey_to_privkey[state.validator_registry[validator_index].pubkey] state.validator_registry[validator_index].activation_epoch = spec.FAR_FUTURE_EPOCH # # build and test voluntary exit # - voluntary_exit = build_voluntary_exit( + voluntary_exit = helpers.build_voluntary_exit( state, current_epoch, validator_index, privkey, ) - pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit, False) + pre_state, post_state = run_voluntary_exit_processing(spec, helpers, state, voluntary_exit, False) return pre_state, voluntary_exit, post_state -def test_validator_already_exited(state): +def test_validator_already_exited(spec, helpers, state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow validator able to exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - current_epoch = get_current_epoch(state) - validator_index = get_active_validator_indices(state, current_epoch)[0] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] + current_epoch = spec.get_current_epoch(state) + validator_index = spec.get_active_validator_indices(state, current_epoch)[0] + privkey = helpers.pubkey_to_privkey[state.validator_registry[validator_index].pubkey] # but validator already has exited state.validator_registry[validator_index].exit_epoch = current_epoch + 2 - voluntary_exit = build_voluntary_exit( + voluntary_exit = helpers.build_voluntary_exit( state, current_epoch, validator_index, privkey, ) - pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit, False) + pre_state, post_state = run_voluntary_exit_processing(spec, helpers, state, voluntary_exit, False) return pre_state, voluntary_exit, post_state -def test_validator_not_active_long_enough(state): - current_epoch = get_current_epoch(state) - validator_index = get_active_validator_indices(state, current_epoch)[0] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] +def test_validator_not_active_long_enough(spec, helpers, state): + current_epoch = spec.get_current_epoch(state) + validator_index = spec.get_active_validator_indices(state, current_epoch)[0] + privkey = helpers.pubkey_to_privkey[state.validator_registry[validator_index].pubkey] - voluntary_exit = build_voluntary_exit( + voluntary_exit = helpers.build_voluntary_exit( state, current_epoch, validator_index, @@ -159,5 +146,5 @@ def test_validator_not_active_long_enough(state): spec.PERSISTENT_COMMITTEE_PERIOD ) - pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit, False) + pre_state, post_state = run_voluntary_exit_processing(spec, helpers, state, voluntary_exit, False) return pre_state, voluntary_exit, post_state diff --git a/test_libs/pyspec/tests/phase0/conftest.py b/test_libs/pyspec/tests/phase0/conftest.py index a3ea0a00c1..25b540aa0f 100644 --- a/test_libs/pyspec/tests/phase0/conftest.py +++ b/test_libs/pyspec/tests/phase0/conftest.py @@ -1,11 +1,9 @@ import pytest -from eth2spec.phase0 import spec +from eth2spec.phase0 import spec as _spec from preset_loader import loader -from tests.phase0.helpers import ( - create_genesis_state, -) +from tests.phase0 import helpers as _helpers def pytest_addoption(parser): @@ -17,11 +15,11 @@ def pytest_addoption(parser): def config(request): config_name = request.config.getoption("--config") presets = loader.load_presets('../../configs/', config_name) - spec.apply_constants_preset(presets) + _spec.apply_constants_preset(presets) @pytest.fixture def num_validators(config): - return spec.SLOTS_PER_EPOCH * 8 + return _spec.SLOTS_PER_EPOCH * 8 @pytest.fixture def deposit_data_leaves(): @@ -29,4 +27,13 @@ def deposit_data_leaves(): @pytest.fixture def state(num_validators, deposit_data_leaves): - return create_genesis_state(num_validators, deposit_data_leaves) + return _helpers.create_genesis_state(num_validators, deposit_data_leaves) + +@pytest.fixture +def spec(): + return _spec + +@pytest.fixture +def helpers(): + return _helpers + diff --git a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py index 6bd9b5d89e..6e4ad2dd6a 100644 --- a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py @@ -1,48 +1,29 @@ from copy import deepcopy import pytest -import eth2spec.phase0.spec as spec - -from eth2spec.phase0.spec import ( - process_slot, - get_crosslink_deltas, - process_crosslinks, - state_transition, -) -from tests.phase0.helpers import ( - add_attestation_to_state, - build_empty_block_for_next_slot, - fill_aggregate_attestation, - get_crosslink_committee, - get_valid_attestation, - next_epoch, - next_slot, - set_bitfield_bit, -) - # mark entire file as 'crosslinks' pytestmark = pytest.mark.crosslinks -def run_process_crosslinks(state, valid=True): +def run_process_crosslinks(spec, helpers, state, valid=True): # transition state to slot before state transition slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = build_empty_block_for_next_slot(state) + block = helpers.build_empty_block_for_next_slot(state) block.slot = slot - state_transition(state, block) + spec.state_transition(state, block) # cache state before epoch transition - process_slot(state) + spec.process_slot(state) post_state = deepcopy(state) - process_crosslinks(post_state) + spec.process_crosslinks(post_state) return state, post_state -def test_no_attestations(state): - pre_state, post_state = run_process_crosslinks(state) +def test_no_attestations(spec, helpers, state): + pre_state, post_state = run_process_crosslinks(spec, helpers, state) for shard in range(spec.SHARD_COUNT): assert post_state.previous_crosslinks[shard] == post_state.current_crosslinks[shard] @@ -50,17 +31,17 @@ def test_no_attestations(state): return pre_state, post_state -def test_single_crosslink_update_from_current_epoch(state): - next_epoch(state) +def test_single_crosslink_update_from_current_epoch(spec, helpers, state): + helpers.next_epoch(state) - attestation = get_valid_attestation(state) + attestation = helpers.get_valid_attestation(state) - fill_aggregate_attestation(state, attestation) - add_attestation_to_state(state, attestation, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + helpers.fill_aggregate_attestation(state, attestation) + helpers.add_attestation_to_state(state, attestation, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) assert len(state.current_epoch_attestations) == 1 - pre_state, post_state = run_process_crosslinks(state) + pre_state, post_state = run_process_crosslinks(spec, helpers, state) shard = attestation.data.crosslink.shard assert post_state.previous_crosslinks[shard] != post_state.current_crosslinks[shard] @@ -69,65 +50,65 @@ def test_single_crosslink_update_from_current_epoch(state): return pre_state, post_state -def test_single_crosslink_update_from_previous_epoch(state): - next_epoch(state) +def test_single_crosslink_update_from_previous_epoch(spec, helpers, state): + helpers.next_epoch(state) - attestation = get_valid_attestation(state) + attestation = helpers.get_valid_attestation(state) - fill_aggregate_attestation(state, attestation) - add_attestation_to_state(state, attestation, state.slot + spec.SLOTS_PER_EPOCH) + helpers.fill_aggregate_attestation(state, attestation) + helpers.add_attestation_to_state(state, attestation, state.slot + spec.SLOTS_PER_EPOCH) assert len(state.previous_epoch_attestations) == 1 - pre_state, post_state = run_process_crosslinks(state) - crosslink_deltas = get_crosslink_deltas(state) + pre_state, post_state = run_process_crosslinks(spec, helpers, state) + crosslink_deltas = spec.get_crosslink_deltas(state) shard = attestation.data.crosslink.shard assert post_state.previous_crosslinks[shard] != post_state.current_crosslinks[shard] assert pre_state.current_crosslinks[shard] != post_state.current_crosslinks[shard] # ensure rewarded - for index in get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.crosslink.shard): + for index in helpers.get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.crosslink.shard): assert crosslink_deltas[0][index] > 0 assert crosslink_deltas[1][index] == 0 return pre_state, post_state -def test_double_late_crosslink(state): - next_epoch(state) +def test_double_late_crosslink(spec, helpers, state): + helpers.next_epoch(state) state.slot += 4 - attestation_1 = get_valid_attestation(state) - fill_aggregate_attestation(state, attestation_1) + attestation_1 = helpers.get_valid_attestation(state) + helpers.fill_aggregate_attestation(state, attestation_1) # add attestation_1 in the next epoch - next_epoch(state) - add_attestation_to_state(state, attestation_1, state.slot + 1) + helpers.next_epoch(state) + helpers.add_attestation_to_state(state, attestation_1, state.slot + 1) for slot in range(spec.SLOTS_PER_EPOCH): - attestation_2 = get_valid_attestation(state) + attestation_2 = helpers.get_valid_attestation(state) if attestation_2.data.crosslink.shard == attestation_1.data.crosslink.shard: break - next_slot(state) - fill_aggregate_attestation(state, attestation_2) + helpers.next_slot(state) + helpers.fill_aggregate_attestation(state, attestation_2) # add attestation_2 in the next epoch after attestation_1 has # already updated the relevant crosslink - next_epoch(state) - add_attestation_to_state(state, attestation_2, state.slot + 1) + helpers.next_epoch(state) + helpers.add_attestation_to_state(state, attestation_2, state.slot + 1) assert len(state.previous_epoch_attestations) == 1 assert len(state.current_epoch_attestations) == 0 - pre_state, post_state = run_process_crosslinks(state) - crosslink_deltas = get_crosslink_deltas(state) + pre_state, post_state = run_process_crosslinks(spec, helpers, state) + crosslink_deltas = spec.get_crosslink_deltas(state) shard = attestation_2.data.crosslink.shard # ensure that the current crosslinks were not updated by the second attestation assert post_state.previous_crosslinks[shard] == post_state.current_crosslinks[shard] # ensure no reward, only penalties for the failed crosslink - for index in get_crosslink_committee(state, attestation_2.data.target_epoch, attestation_2.data.crosslink.shard): + for index in helpers.get_crosslink_committee(state, attestation_2.data.target_epoch, attestation_2.data.crosslink.shard): assert crosslink_deltas[0][index] == 0 assert crosslink_deltas[1][index] > 0 diff --git a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py index 54379de7dc..8b5ebf9b4d 100644 --- a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py @@ -1,51 +1,41 @@ from copy import deepcopy - import pytest -import eth2spec.phase0.spec as spec - -from eth2spec.phase0.spec import ( - get_current_epoch, - is_active_validator, -) -from tests.phase0.helpers import ( - next_epoch, -) # mark entire file as 'state' pytestmark = pytest.mark.state -def test_activation(state): +def test_activation(spec, helpers, state): index = 0 - assert is_active_validator(state.validator_registry[index], get_current_epoch(state)) + assert spec.is_active_validator(state.validator_registry[index], spec.get_current_epoch(state)) # Mock a new deposit state.validator_registry[index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH state.validator_registry[index].activation_epoch = spec.FAR_FUTURE_EPOCH state.validator_registry[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE - assert not is_active_validator(state.validator_registry[index], get_current_epoch(state)) + assert not spec.is_active_validator(state.validator_registry[index], spec.get_current_epoch(state)) pre_state = deepcopy(state) blocks = [] for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): - block = next_epoch(state) + block = helpers.next_epoch(state) blocks.append(block) assert state.validator_registry[index].activation_eligibility_epoch != spec.FAR_FUTURE_EPOCH assert state.validator_registry[index].activation_epoch != spec.FAR_FUTURE_EPOCH - assert is_active_validator( + assert spec.is_active_validator( state.validator_registry[index], - get_current_epoch(state), + spec.get_current_epoch(state), ) return pre_state, blocks, state -def test_ejection(state): +def test_ejection(spec, helpers, state): index = 0 - assert is_active_validator(state.validator_registry[index], get_current_epoch(state)) + assert spec.is_active_validator(state.validator_registry[index], spec.get_current_epoch(state)) assert state.validator_registry[index].exit_epoch == spec.FAR_FUTURE_EPOCH # Mock an ejection @@ -55,13 +45,13 @@ def test_ejection(state): blocks = [] for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): - block = next_epoch(state) + block = helpers.next_epoch(state) blocks.append(block) assert state.validator_registry[index].exit_epoch != spec.FAR_FUTURE_EPOCH - assert not is_active_validator( + assert not spec.is_active_validator( state.validator_registry[index], - get_current_epoch(state), + spec.get_current_epoch(state), ) return pre_state, blocks, state diff --git a/test_libs/pyspec/tests/phase0/test_finality.py b/test_libs/pyspec/tests/phase0/test_finality.py index 49569f220c..e30e125f9a 100644 --- a/test_libs/pyspec/tests/phase0/test_finality.py +++ b/test_libs/pyspec/tests/phase0/test_finality.py @@ -2,21 +2,6 @@ import pytest -import eth2spec.phase0.spec as spec - -from eth2spec.phase0.spec import ( - state_transition, -) - -from .helpers import ( - build_empty_block_for_next_slot, - fill_aggregate_attestation, - get_current_epoch, - get_epoch_start_slot, - get_valid_attestation, - next_epoch, -) - # mark entire file as 'state' pytestmark = pytest.mark.state @@ -48,24 +33,24 @@ def check_finality(state, assert state.finalized_root == prev_state.finalized_root -def next_epoch_with_attestations(state, +def next_epoch_with_attestations(spec, helpers, state, fill_cur_epoch, fill_prev_epoch): post_state = deepcopy(state) blocks = [] for _ in range(spec.SLOTS_PER_EPOCH): - block = build_empty_block_for_next_slot(post_state) + block = helpers.build_empty_block_for_next_slot(post_state) if fill_cur_epoch: slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - if slot_to_attest >= get_epoch_start_slot(get_current_epoch(post_state)): - cur_attestation = get_valid_attestation(post_state, slot_to_attest) - fill_aggregate_attestation(post_state, cur_attestation) + if slot_to_attest >= helpers.get_epoch_start_slot(helpers.get_current_epoch(post_state)): + cur_attestation = helpers.get_valid_attestation(post_state, slot_to_attest) + helpers.fill_aggregate_attestation(post_state, cur_attestation) block.body.attestations.append(cur_attestation) if fill_prev_epoch: slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 - prev_attestation = get_valid_attestation(post_state, slot_to_attest) - fill_aggregate_attestation(post_state, prev_attestation) + prev_attestation = helpers.get_valid_attestation(post_state, slot_to_attest) + helpers.fill_aggregate_attestation(post_state, prev_attestation) block.body.attestations.append(prev_attestation) spec.state_transition(post_state, block) @@ -74,12 +59,12 @@ def next_epoch_with_attestations(state, return state, blocks, post_state -def test_finality_rule_4(state): +def test_finality_rule_4(spec, helpers, state): test_state = deepcopy(state) blocks = [] for epoch in range(4): - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) + prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, True, False) blocks += new_blocks # justification/finalization skipped at GENESIS_EPOCH @@ -99,17 +84,17 @@ def test_finality_rule_4(state): return state, blocks, test_state -def test_finality_rule_1(state): +def test_finality_rule_1(spec, helpers, state): # get past first two epochs that finality does not run on - next_epoch(state) - next_epoch(state) + helpers.next_epoch(state) + helpers.next_epoch(state) pre_state = deepcopy(state) test_state = deepcopy(state) blocks = [] for epoch in range(3): - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, True) + prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, False, True) blocks += new_blocks if epoch == 0: @@ -125,10 +110,10 @@ def test_finality_rule_1(state): return pre_state, blocks, test_state -def test_finality_rule_2(state): +def test_finality_rule_2(spec, helpers, state): # get past first two epochs that finality does not run on - next_epoch(state) - next_epoch(state) + helpers.next_epoch(state) + helpers.next_epoch(state) pre_state = deepcopy(state) test_state = deepcopy(state) @@ -136,13 +121,13 @@ def test_finality_rule_2(state): blocks = [] for epoch in range(3): if epoch == 0: - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) + prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, True, False) check_finality(test_state, prev_state, True, False, False) elif epoch == 1: - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, False) + prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, False, False) check_finality(test_state, prev_state, False, True, False) elif epoch == 2: - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, True) + prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, False, True) # finalized by rule 2 check_finality(test_state, prev_state, True, False, True) assert test_state.finalized_epoch == prev_state.previous_justified_epoch @@ -153,43 +138,43 @@ def test_finality_rule_2(state): return pre_state, blocks, test_state -def test_finality_rule_3(state): +def test_finality_rule_3(spec, helpers, state): """ Test scenario described here https://github.com/ethereum/eth2.0-specs/issues/611#issuecomment-463612892 """ # get past first two epochs that finality does not run on - next_epoch(state) - next_epoch(state) + helpers.next_epoch(state) + helpers.next_epoch(state) pre_state = deepcopy(state) test_state = deepcopy(state) blocks = [] - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) + prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, True, False) blocks += new_blocks check_finality(test_state, prev_state, True, False, False) # In epoch N, JE is set to N, prev JE is set to N-1 - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) + prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, True, False) blocks += new_blocks check_finality(test_state, prev_state, True, True, True) # In epoch N+1, JE is N, prev JE is N-1, and not enough messages get in to do anything - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, False) + prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, False, False) blocks += new_blocks check_finality(test_state, prev_state, False, True, False) # In epoch N+2, JE is N, prev JE is N, and enough messages from the previous epoch get in to justify N+1. # N+1 now becomes the JE. Not enough messages from epoch N+2 itself get in to justify N+2 - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, True) + prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, False, True) blocks += new_blocks # rule 2 check_finality(test_state, prev_state, True, False, True) # In epoch N+3, LJE is N+1, prev LJE is N, and enough messages get in to justify epochs N+2 and N+3. - prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, True) + prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, True, True) blocks += new_blocks # rule 3 check_finality(test_state, prev_state, True, True, True) diff --git a/test_libs/pyspec/tests/phase0/test_sanity.py b/test_libs/pyspec/tests/phase0/test_sanity.py index 7f3ee48e1e..ad206881da 100644 --- a/test_libs/pyspec/tests/phase0/test_sanity.py +++ b/test_libs/pyspec/tests/phase0/test_sanity.py @@ -3,128 +3,95 @@ import pytest from py_ecc import bls -import eth2spec.phase0.spec as spec from eth2spec.utils.minimal_ssz import signing_root -from eth2spec.phase0.spec import ( - # constants - ZERO_HASH, - SLOTS_PER_HISTORICAL_ROOT, - # SSZ - Deposit, - Transfer, - VoluntaryExit, - # functions - get_active_validator_indices, - get_beacon_proposer_index, - get_block_root_at_slot, - get_current_epoch, - get_domain, - process_slot, - verify_merkle_branch, - state_transition, - hash, -) + from eth2spec.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, get_merkle_proof, get_merkle_root, ) -from .helpers import ( - advance_slot, - get_balance, - build_deposit_data, - build_empty_block_for_next_slot, - fill_aggregate_attestation, - get_state_root, - get_valid_attestation, - get_valid_attester_slashing, - get_valid_proposer_slashing, - next_slot, - privkeys, - pubkeys, -) # mark entire file as 'sanity' pytestmark = pytest.mark.sanity -def test_slot_transition(state): +def test_slot_transition(spec, helpers, state): test_state = deepcopy(state) - process_slot(test_state) - advance_slot(test_state) + spec.process_slot(test_state) + helpers.advance_slot(test_state) assert test_state.slot == state.slot + 1 - assert get_state_root(test_state, state.slot) == state.hash_tree_root() + assert helpers.get_state_root(test_state, state.slot) == state.hash_tree_root() return test_state -def test_empty_block_transition(state): +def test_empty_block_transition(spec, helpers, state): test_state = deepcopy(state) - block = build_empty_block_for_next_slot(test_state) - state_transition(test_state, block) + block = helpers.build_empty_block_for_next_slot(test_state) + spec.state_transition(test_state, block) assert len(test_state.eth1_data_votes) == len(state.eth1_data_votes) + 1 - assert get_block_root_at_slot(test_state, state.slot) == block.parent_root + assert spec.get_block_root_at_slot(test_state, state.slot) == block.parent_root return state, [block], test_state -def test_skipped_slots(state): +def test_skipped_slots(spec, helpers, state): test_state = deepcopy(state) - block = build_empty_block_for_next_slot(test_state) + block = helpers.build_empty_block_for_next_slot(test_state) block.slot += 3 - state_transition(test_state, block) + spec.state_transition(test_state, block) assert test_state.slot == block.slot for slot in range(state.slot, test_state.slot): - assert get_block_root_at_slot(test_state, slot) == block.parent_root + assert spec.get_block_root_at_slot(test_state, slot) == block.parent_root return state, [block], test_state -def test_empty_epoch_transition(state): +def test_empty_epoch_transition(spec, helpers, state): test_state = deepcopy(state) - block = build_empty_block_for_next_slot(test_state) + block = helpers.build_empty_block_for_next_slot(test_state) block.slot += spec.SLOTS_PER_EPOCH - state_transition(test_state, block) + spec.state_transition(test_state, block) assert test_state.slot == block.slot for slot in range(state.slot, test_state.slot): - assert get_block_root_at_slot(test_state, slot) == block.parent_root + assert spec.get_block_root_at_slot(test_state, slot) == block.parent_root return state, [block], test_state -def test_empty_epoch_transition_not_finalizing(state): +def test_empty_epoch_transition_not_finalizing(spec, helpers, state): test_state = deepcopy(state) - block = build_empty_block_for_next_slot(test_state) + block = helpers.build_empty_block_for_next_slot(test_state) block.slot += spec.SLOTS_PER_EPOCH * 5 - state_transition(test_state, block) + spec.state_transition(test_state, block) assert test_state.slot == block.slot - assert test_state.finalized_epoch < get_current_epoch(test_state) - 4 + assert test_state.finalized_epoch < spec.get_current_epoch(test_state) - 4 for index in range(len(test_state.validator_registry)): - assert get_balance(test_state, index) < get_balance(state, index) + assert helpers.get_balance(test_state, index) < helpers.get_balance(state, index) return state, [block], test_state -def test_proposer_slashing(state): +def test_proposer_slashing(spec, helpers, state): test_state = deepcopy(state) - proposer_slashing = get_valid_proposer_slashing(state) + proposer_slashing = helpers.get_valid_proposer_slashing(state) validator_index = proposer_slashing.proposer_index # # Add to state via block transition # - block = build_empty_block_for_next_slot(test_state) + block = helpers.build_empty_block_for_next_slot(test_state) block.body.proposer_slashings.append(proposer_slashing) - state_transition(test_state, block) + spec.state_transition(test_state, block) assert not state.validator_registry[validator_index].slashed @@ -133,22 +100,22 @@ def test_proposer_slashing(state): assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH # lost whistleblower reward - assert get_balance(test_state, validator_index) < get_balance(state, validator_index) + assert helpers.get_balance(test_state, validator_index) < helpers.get_balance(state, validator_index) return state, [block], test_state -def test_attester_slashing(state): +def test_attester_slashing(spec, helpers, state): test_state = deepcopy(state) - attester_slashing = get_valid_attester_slashing(state) + attester_slashing = helpers.get_valid_attester_slashing(state) validator_index = attester_slashing.attestation_1.custody_bit_0_indices[0] # # Add to state via block transition # - block = build_empty_block_for_next_slot(test_state) + block = helpers.build_empty_block_for_next_slot(test_state) block.body.attester_slashings.append(attester_slashing) - state_transition(test_state, block) + spec.state_transition(test_state, block) assert not state.validator_registry[validator_index].slashed @@ -157,35 +124,35 @@ def test_attester_slashing(state): assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH # lost whistleblower reward - assert get_balance(test_state, validator_index) < get_balance(state, validator_index) + assert helpers.get_balance(test_state, validator_index) < helpers.get_balance(state, validator_index) - proposer_index = get_beacon_proposer_index(test_state) + proposer_index = spec.get_beacon_proposer_index(test_state) # gained whistleblower reward assert ( - get_balance(test_state, proposer_index) > - get_balance(state, proposer_index) + helpers.get_balance(test_state, proposer_index) > + helpers.get_balance(state, proposer_index) ) return state, [block], test_state -def test_deposit_in_block(state): +def test_deposit_in_block(spec, helpers, state): pre_state = deepcopy(state) - test_deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) + test_deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) index = len(test_deposit_data_leaves) - pubkey = pubkeys[index] - privkey = privkeys[index] - deposit_data = build_deposit_data(pre_state, pubkey, privkey, spec.MAX_EFFECTIVE_BALANCE) + pubkey = helpers.pubkeys[index] + privkey = helpers.privkeys[index] + deposit_data = helpers.build_deposit_data(pre_state, pubkey, privkey, spec.MAX_EFFECTIVE_BALANCE) item = deposit_data.hash_tree_root() test_deposit_data_leaves.append(item) tree = calc_merkle_tree_from_leaves(tuple(test_deposit_data_leaves)) root = get_merkle_root((tuple(test_deposit_data_leaves))) proof = list(get_merkle_proof(tree, item_index=index)) - assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) + assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) - deposit = Deposit( + deposit = spec.Deposit( proof=list(proof), index=index, data=deposit_data, @@ -194,27 +161,27 @@ def test_deposit_in_block(state): pre_state.latest_eth1_data.deposit_root = root pre_state.latest_eth1_data.deposit_count = len(test_deposit_data_leaves) post_state = deepcopy(pre_state) - block = build_empty_block_for_next_slot(post_state) + block = helpers.build_empty_block_for_next_slot(post_state) block.body.deposits.append(deposit) - state_transition(post_state, block) + spec.state_transition(post_state, block) assert len(post_state.validator_registry) == len(state.validator_registry) + 1 assert len(post_state.balances) == len(state.balances) + 1 - assert get_balance(post_state, index) == spec.MAX_EFFECTIVE_BALANCE - assert post_state.validator_registry[index].pubkey == pubkeys[index] + assert helpers.get_balance(post_state, index) == spec.MAX_EFFECTIVE_BALANCE + assert post_state.validator_registry[index].pubkey == helpers.pubkeys[index] return pre_state, [block], post_state -def test_deposit_top_up(state): +def test_deposit_top_up(spec, helpers, state): pre_state = deepcopy(state) - test_deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) + test_deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) validator_index = 0 amount = spec.MAX_EFFECTIVE_BALANCE // 4 - pubkey = pubkeys[validator_index] - privkey = privkeys[validator_index] - deposit_data = build_deposit_data(pre_state, pubkey, privkey, amount) + pubkey = helpers.pubkeys[validator_index] + privkey = helpers.privkeys[validator_index] + deposit_data = helpers.build_deposit_data(pre_state, pubkey, privkey, amount) merkle_index = len(test_deposit_data_leaves) item = deposit_data.hash_tree_root() @@ -222,9 +189,9 @@ def test_deposit_top_up(state): tree = calc_merkle_tree_from_leaves(tuple(test_deposit_data_leaves)) root = get_merkle_root((tuple(test_deposit_data_leaves))) proof = list(get_merkle_proof(tree, item_index=merkle_index)) - assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, merkle_index, root) + assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, merkle_index, root) - deposit = Deposit( + deposit = spec.Deposit( proof=list(proof), index=merkle_index, data=deposit_data, @@ -232,31 +199,31 @@ def test_deposit_top_up(state): pre_state.latest_eth1_data.deposit_root = root pre_state.latest_eth1_data.deposit_count = len(test_deposit_data_leaves) - block = build_empty_block_for_next_slot(pre_state) + block = helpers.build_empty_block_for_next_slot(pre_state) block.body.deposits.append(deposit) - pre_balance = get_balance(pre_state, validator_index) + pre_balance = helpers.get_balance(pre_state, validator_index) post_state = deepcopy(pre_state) - state_transition(post_state, block) + spec.state_transition(post_state, block) assert len(post_state.validator_registry) == len(pre_state.validator_registry) assert len(post_state.balances) == len(pre_state.balances) - assert get_balance(post_state, validator_index) == pre_balance + amount + assert helpers.get_balance(post_state, validator_index) == pre_balance + amount return pre_state, [block], post_state -def test_attestation(state): +def test_attestation(spec, helpers, state): state.slot = spec.SLOTS_PER_EPOCH test_state = deepcopy(state) - attestation = get_valid_attestation(state) + attestation = helpers.get_valid_attestation(state) # # Add to state via block transition # - attestation_block = build_empty_block_for_next_slot(test_state) + attestation_block = helpers.build_empty_block_for_next_slot(test_state) attestation_block.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation_block.body.attestations.append(attestation) - state_transition(test_state, attestation_block) + spec.state_transition(test_state, attestation_block) assert len(test_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1 @@ -266,9 +233,9 @@ def test_attestation(state): # pre_current_epoch_attestations = deepcopy(test_state.current_epoch_attestations) - epoch_block = build_empty_block_for_next_slot(test_state) + epoch_block = helpers.build_empty_block_for_next_slot(test_state) epoch_block.slot += spec.SLOTS_PER_EPOCH - state_transition(test_state, epoch_block) + spec.state_transition(test_state, epoch_block) assert len(test_state.current_epoch_attestations) == 0 assert test_state.previous_epoch_attestations == pre_current_epoch_attestations @@ -276,11 +243,11 @@ def test_attestation(state): return state, [attestation_block, epoch_block], test_state -def test_voluntary_exit(state): +def test_voluntary_exit(spec, helpers, state): pre_state = deepcopy(state) - validator_index = get_active_validator_indices( + validator_index = spec.get_active_validator_indices( pre_state, - get_current_epoch(pre_state) + spec.get_current_epoch(pre_state) )[-1] # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit @@ -288,14 +255,14 @@ def test_voluntary_exit(state): post_state = deepcopy(pre_state) - voluntary_exit = VoluntaryExit( - epoch=get_current_epoch(pre_state), + voluntary_exit = spec.VoluntaryExit( + epoch=spec.get_current_epoch(pre_state), validator_index=validator_index, ) voluntary_exit.signature = bls.sign( message_hash=signing_root(voluntary_exit), - privkey=privkeys[validator_index], - domain=get_domain( + privkey=helpers.privkeys[validator_index], + domain=spec.get_domain( state=pre_state, domain_type=spec.DOMAIN_VOLUNTARY_EXIT, ) @@ -304,37 +271,37 @@ def test_voluntary_exit(state): # # Add to state via block transition # - initiate_exit_block = build_empty_block_for_next_slot(post_state) + initiate_exit_block = helpers.build_empty_block_for_next_slot(post_state) initiate_exit_block.body.voluntary_exits.append(voluntary_exit) - state_transition(post_state, initiate_exit_block) + spec.state_transition(post_state, initiate_exit_block) assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH # # Process within epoch transition # - exit_block = build_empty_block_for_next_slot(post_state) + exit_block = helpers.build_empty_block_for_next_slot(post_state) exit_block.slot += spec.SLOTS_PER_EPOCH - state_transition(post_state, exit_block) + spec.state_transition(post_state, exit_block) assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH return pre_state, [initiate_exit_block, exit_block], post_state -def test_transfer(state): +def test_transfer(spec, helpers, state): # overwrite default 0 to test spec.apply_constants_preset({"MAX_TRANSFERS": 1}) pre_state = deepcopy(state) - current_epoch = get_current_epoch(pre_state) - sender_index = get_active_validator_indices(pre_state, current_epoch)[-1] - recipient_index = get_active_validator_indices(pre_state, current_epoch)[0] - transfer_pubkey = pubkeys[-1] - transfer_privkey = privkeys[-1] - amount = get_balance(pre_state, sender_index) - pre_transfer_recipient_balance = get_balance(pre_state, recipient_index) - transfer = Transfer( + current_epoch = spec.get_current_epoch(pre_state) + sender_index = spec.get_active_validator_indices(pre_state, current_epoch)[-1] + recipient_index = spec.get_active_validator_indices(pre_state, current_epoch)[0] + transfer_pubkey = helpers.pubkeys[-1] + transfer_privkey = helpers.privkeys[-1] + amount = helpers.get_balance(pre_state, sender_index) + pre_transfer_recipient_balance = helpers.get_balance(pre_state, recipient_index) + transfer = spec.Transfer( sender=sender_index, recipient=recipient_index, amount=amount, @@ -345,7 +312,7 @@ def test_transfer(state): transfer.signature = bls.sign( message_hash=signing_root(transfer), privkey=transfer_privkey, - domain=get_domain( + domain=spec.get_domain( state=pre_state, domain_type=spec.DOMAIN_TRANSFER, ) @@ -353,7 +320,7 @@ def test_transfer(state): # ensure withdrawal_credentials reproducable pre_state.validator_registry[sender_index].withdrawal_credentials = ( - spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(transfer_pubkey)[1:] + spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer_pubkey)[1:] ) # un-activate so validator can transfer pre_state.validator_registry[sender_index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH @@ -362,21 +329,21 @@ def test_transfer(state): # # Add to state via block transition # - block = build_empty_block_for_next_slot(post_state) + block = helpers.build_empty_block_for_next_slot(post_state) block.body.transfers.append(transfer) - state_transition(post_state, block) + spec.state_transition(post_state, block) - sender_balance = get_balance(post_state, sender_index) - recipient_balance = get_balance(post_state, recipient_index) + sender_balance = helpers.get_balance(post_state, sender_index) + recipient_balance = helpers.get_balance(post_state, recipient_index) assert sender_balance == 0 assert recipient_balance == pre_transfer_recipient_balance + amount return pre_state, [block], post_state -def test_balance_driven_status_transitions(state): - current_epoch = get_current_epoch(state) - validator_index = get_active_validator_indices(state, current_epoch)[-1] +def test_balance_driven_status_transitions(spec, helpers, state): + current_epoch = spec.get_current_epoch(state) + validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] assert state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH @@ -387,32 +354,32 @@ def test_balance_driven_status_transitions(state): # # trigger epoch transition # - block = build_empty_block_for_next_slot(post_state) + block = helpers.build_empty_block_for_next_slot(post_state) block.slot += spec.SLOTS_PER_EPOCH - state_transition(post_state, block) + spec.state_transition(post_state, block) assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH return state, [block], post_state -def test_historical_batch(state): +def test_historical_batch(spec, helpers, state): state.slot += spec.SLOTS_PER_HISTORICAL_ROOT - (state.slot % spec.SLOTS_PER_HISTORICAL_ROOT) - 1 post_state = deepcopy(state) - block = build_empty_block_for_next_slot(post_state) + block = helpers.build_empty_block_for_next_slot(post_state) - state_transition(post_state, block) + spec.state_transition(post_state, block) assert post_state.slot == block.slot - assert get_current_epoch(post_state) % (spec.SLOTS_PER_HISTORICAL_ROOT // spec.SLOTS_PER_EPOCH) == 0 + assert spec.get_current_epoch(post_state) % (spec.SLOTS_PER_HISTORICAL_ROOT // spec.SLOTS_PER_EPOCH) == 0 assert len(post_state.historical_roots) == len(state.historical_roots) + 1 return state, [block], post_state -def test_eth1_data_votes(state): +def test_eth1_data_votes(spec, helpers, state): post_state = deepcopy(state) expected_votes = 0 @@ -420,14 +387,14 @@ def test_eth1_data_votes(state): blocks = [] for _ in range(spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1): - block = build_empty_block_for_next_slot(post_state) - state_transition(post_state, block) + block = helpers.build_empty_block_for_next_slot(post_state) + spec.state_transition(post_state, block) expected_votes += 1 assert len(post_state.eth1_data_votes) == expected_votes blocks.append(block) - block = build_empty_block_for_next_slot(post_state) - state_transition(post_state, block) + block = helpers.build_empty_block_for_next_slot(post_state) + spec.state_transition(post_state, block) blocks.append(block) assert post_state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 From b0747703a77c36d821b6bf049aa8abc8e1603603 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:23:26 +0800 Subject: [PATCH 115/308] Make CI job `lint` require `test` --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3fe6643b74..d2b284e255 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ jobs: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} paths: - ~/specs-repo - install_test: + install_env: docker: - image: circleci/python:3.6 working_directory: ~/specs-repo @@ -64,7 +64,7 @@ jobs: reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements - command: make install_test + command: make install_test && make install_lint - save_cached_venv: venv_name: v1-pyspec reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' @@ -96,18 +96,18 @@ jobs: reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter - command: make install_lint && make lint + command: make lint workflows: version: 2.1 test_spec: jobs: - checkout_specs - - lint: - requires: - - checkout_specs - - install_test: + - install_env: requires: - checkout_specs - test: requires: - - install_test + - install_env + - lint: + requires: + - test From 1b3dfa67813e435761a71a3cc1c5e552efc043dd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:34:49 +0800 Subject: [PATCH 116/308] kick the cache --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d2b284e255..b9f09d9ccb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,13 +60,13 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements command: make install_test && make install_lint - save_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv test: @@ -77,7 +77,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run py-tests @@ -92,7 +92,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec + venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter From 32f76641e34afb703355ab8b8ac93fafa0d59478 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:41:48 +0800 Subject: [PATCH 117/308] Add build_pyspec job --- .circleci/config.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b9f09d9ccb..a0700a004d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,6 +69,19 @@ jobs: venv_name: v1-pyspec-02 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv + build_pyspec: + docker: + - image: circleci/python:3.6 + working_directory: ~/specs-repo + steps: + - restore_cache: + key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} + - restore_cached_venv: + venv_name: v1-pyspec-02 + reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + - run: + name: Build pyspec + command: make pyspec test: docker: - image: circleci/python:3.6 @@ -105,9 +118,13 @@ workflows: - install_env: requires: - checkout_specs + - build_pyspec: + requires: + - checkout_specs - test: requires: - install_env + - build_pyspec - lint: requires: - - test + - build_pyspec From ee4fdf5d06c909b4d7bc6f1dff3bce5a253b54e7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 17:45:09 +0800 Subject: [PATCH 118/308] Kick the cache again --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a0700a004d..972f0e3514 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,13 +60,13 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements command: make install_test && make install_lint - save_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv build_pyspec: @@ -77,7 +77,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Build pyspec @@ -90,7 +90,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run py-tests @@ -105,7 +105,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-02 + venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter From b1711b4ff1399461d0832ae7206ecf06b1312160 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 16 May 2019 12:10:08 +0200 Subject: [PATCH 119/308] Implements parameterised phase1 tests --- .gitignore | 1 + Makefile | 3 +- .../test_phase0_block_processing.py | 7 ++ ...est_process_early_derived_secret_reveal.py | 72 ++++++++----------- test_libs/pyspec/tests/phase1/conftest.py | 23 ++++-- .../phase1/epoch_processing/test_phase0.py | 2 + test_libs/pyspec/tests/phase1/helpers.py | 16 +---- .../pyspec/tests/phase1/helpers_phase1.py | 52 -------------- .../pyspec/tests/phase1/test_finality.py | 25 +------ test_libs/pyspec/tests/phase1/test_sanity.py | 66 +---------------- 10 files changed, 60 insertions(+), 207 deletions(-) create mode 100644 test_libs/pyspec/tests/phase1/block_processing_phase1/test_phase0_block_processing.py create mode 100644 test_libs/pyspec/tests/phase1/epoch_processing/test_phase0.py delete mode 100644 test_libs/pyspec/tests/phase1/helpers_phase1.py diff --git a/.gitignore b/.gitignore index 3dd86fc805..c6b39955f0 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ eth2.0-spec-tests/ # Dynamically built from Markdown spec test_libs/pyspec/eth2spec/phase0/spec.py +test_libs/pyspec/eth2spec/phase1/spec.py diff --git a/Makefile b/Makefile index 49460bdf11..c3ec212808 100644 --- a/Makefile +++ b/Makefile @@ -38,8 +38,7 @@ gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS) install_test: cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt; -test: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest . +test: test_phase0 test_phase1 test_phase0: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest tests/phase0 diff --git a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_phase0_block_processing.py b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_phase0_block_processing.py new file mode 100644 index 0000000000..246ba163c5 --- /dev/null +++ b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_phase0_block_processing.py @@ -0,0 +1,7 @@ +from tests.phase0.block_processing.test_process_attestation import * +from tests.phase0.block_processing.test_process_attester_slashing import * +from tests.phase0.block_processing.test_process_block_header import * +from tests.phase0.block_processing.test_process_deposit import * +from tests.phase0.block_processing.test_process_proposer_slashing import * +from tests.phase0.block_processing.test_process_transfer import * +from tests.phase0.block_processing.test_voluntary_exit import * diff --git a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py index 6661e883fb..e3cf30ef3c 100644 --- a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py @@ -1,22 +1,10 @@ from copy import deepcopy import pytest -import eth2spec.phase1.spec as spec -from eth2spec.phase1.spec import ( - get_current_epoch, - process_early_derived_secret_reveal, - RANDAO_PENALTY_EPOCHS, - CUSTODY_PERIOD_TO_RANDAO_PADDING, - EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS, -) -from tests.phase1.helpers import ( - get_valid_early_derived_secret_reveal, -) - #mark entire file as 'randao_key_reveals' pytestmark = pytest.mark.randao_key_reveals -def run_early_derived_secret_reveal_processing(state, randao_key_reveal, valid=True): +def terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal, valid=True): """ Run ``process_randao_key_reveal`` returning the pre and post state. If ``valid == False``, run expecting ``AssertionError`` @@ -25,14 +13,14 @@ def run_early_derived_secret_reveal_processing(state, randao_key_reveal, valid=T if not valid: with pytest.raises(AssertionError): - process_early_derived_secret_reveal(post_state, randao_key_reveal) + spec.process_early_derived_secret_reveal(post_state, randao_key_reveal) return state, None - process_early_derived_secret_reveal(post_state, randao_key_reveal) + spec.process_early_derived_secret_reveal(post_state, randao_key_reveal) slashed_validator = post_state.validator_registry[randao_key_reveal.revealed_index] - if randao_key_reveal.epoch >= get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING: + if randao_key_reveal.epoch >= spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING: assert slashed_validator.slashed assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH @@ -46,63 +34,63 @@ def run_early_derived_secret_reveal_processing(state, randao_key_reveal, valid=T return state, post_state -def test_success(state): - randao_key_reveal = get_valid_early_derived_secret_reveal(state) +def test_success(spec, helpers, state): + randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state) - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal) + pre_state, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal) return pre_state, randao_key_reveal, post_state -def test_reveal_from_current_epoch(state): - randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state)) +def test_reveal_from_current_epoch(spec, helpers, state): + randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state)) - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + pre_state, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal, False) return pre_state, randao_key_reveal, post_state @pytest.mark.skip(reason="Not currently possible as we are testing at epoch 0") -def test_reveal_from_past_epoch(state): - randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) - 1) +def test_reveal_from_past_epoch(spec, helpers, state): + randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) - 1) - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + pre_state, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal, False) return pre_state, randao_key_reveal, post_state -def test_reveal_with_custody_padding(state): - randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING) - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) +def test_reveal_with_custody_padding(spec, helpers, state): + randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING) + pre_state, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal, True) return pre_state, randao_key_reveal, post_state -def test_reveal_with_custody_padding_minus_one(state): - randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + CUSTODY_PERIOD_TO_RANDAO_PADDING - 1) - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) +def test_reveal_with_custody_padding_minus_one(spec, helpers, state): + randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING - 1) + pre_state, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal, True) return pre_state, randao_key_reveal, post_state -def test_double_reveal(state): +def test_double_reveal(spec, helpers, state): - randao_key_reveal1 = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + RANDAO_PENALTY_EPOCHS + 1) - pre_state, intermediate_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal1) + randao_key_reveal1 = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) + spec.RANDAO_PENALTY_EPOCHS + 1) + pre_state, intermediate_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal1) - randao_key_reveal2 = get_valid_early_derived_secret_reveal(intermediate_state, get_current_epoch(pre_state) + RANDAO_PENALTY_EPOCHS + 1) - _, post_state = run_early_derived_secret_reveal_processing(intermediate_state, randao_key_reveal2, False) + randao_key_reveal2 = helpers.get_valid_early_derived_secret_reveal(intermediate_state, spec.get_current_epoch(pre_state) + spec.RANDAO_PENALTY_EPOCHS + 1) + _, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, intermediate_state, randao_key_reveal2, False) return pre_state, [randao_key_reveal1, randao_key_reveal2], post_state -def test_revealer_is_slashed(state): - randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state)) +def test_revealer_is_slashed(spec, helpers, state): + randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state)) state.validator_registry[randao_key_reveal.revealed_index].slashed = True - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + pre_state, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal, False) return pre_state, randao_key_reveal, post_state -def test_far_future_epoch(state): - randao_key_reveal = get_valid_early_derived_secret_reveal(state, get_current_epoch(state) + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS) +def test_far_future_epoch(spec, helpers, state): + randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) + spec.EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS) - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + pre_state, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal, False) return pre_state, randao_key_reveal, post_state diff --git a/test_libs/pyspec/tests/phase1/conftest.py b/test_libs/pyspec/tests/phase1/conftest.py index c73f25c1b1..4af3451a5a 100644 --- a/test_libs/pyspec/tests/phase1/conftest.py +++ b/test_libs/pyspec/tests/phase1/conftest.py @@ -1,15 +1,12 @@ import pytest -from eth2spec.phase1 import spec +from eth2spec.phase1 import spec as _spec from preset_loader import loader -from tests.phase0.helpers import ( - create_genesis_state, -) +from tests.phase1 import helpers as _helpers from tests.phase0.conftest import ( pytest_addoption, - num_validators, deposit_data_leaves, ) @@ -18,9 +15,21 @@ def config(request): config_name = request.config.getoption("--config") presets = loader.load_presets('../../configs/', config_name) - spec.apply_constants_preset(presets) + _spec.apply_constants_preset(presets) + +@pytest.fixture +def num_validators(config): + return _spec.SLOTS_PER_EPOCH * 8 #This is redefined so that the BeaconState is the new SSZ Object @pytest.fixture def state(num_validators, deposit_data_leaves): - return create_genesis_state(num_validators, deposit_data_leaves) + return _helpers.create_genesis_state(num_validators, deposit_data_leaves) + +@pytest.fixture +def spec(): + return _spec + +@pytest.fixture +def helpers(): + return _helpers diff --git a/test_libs/pyspec/tests/phase1/epoch_processing/test_phase0.py b/test_libs/pyspec/tests/phase1/epoch_processing/test_phase0.py new file mode 100644 index 0000000000..ec30125600 --- /dev/null +++ b/test_libs/pyspec/tests/phase1/epoch_processing/test_phase0.py @@ -0,0 +1,2 @@ +from tests.phase0.epoch_processing.test_process_crosslinks import * +from tests.phase0.epoch_processing.test_process_registry_updates import * diff --git a/test_libs/pyspec/tests/phase1/helpers.py b/test_libs/pyspec/tests/phase1/helpers.py index 8520ce9d75..b562666177 100644 --- a/test_libs/pyspec/tests/phase1/helpers.py +++ b/test_libs/pyspec/tests/phase1/helpers.py @@ -3,7 +3,6 @@ import eth2spec.phase1.spec as spec from eth2spec.phase1.spec import ( # constants - ZERO_HASH, CUSTODY_PERIOD_TO_RANDAO_PADDING, # SSZ EarlyDerivedSecretReveal, @@ -14,20 +13,7 @@ hash_tree_root, ) -from tests.phase0.helpers import ( - advance_slot, - get_balance, - build_deposit_data, - build_empty_block_for_next_slot, - fill_aggregate_attestation, - get_state_root, - get_valid_attestation, - get_valid_attester_slashing, - get_valid_proposer_slashing, - next_slot, - privkeys, - pubkeys, -) +from tests.phase0.helpers import * def get_valid_early_derived_secret_reveal(state, epoch=None): current_epoch = get_current_epoch(state) diff --git a/test_libs/pyspec/tests/phase1/helpers_phase1.py b/test_libs/pyspec/tests/phase1/helpers_phase1.py deleted file mode 100644 index 5b67e9aa65..0000000000 --- a/test_libs/pyspec/tests/phase1/helpers_phase1.py +++ /dev/null @@ -1,52 +0,0 @@ -from py_ecc import bls - -import eth2spec.phase1.spec as spec -from eth2spec.phase1.spec import ( - # constants - ZERO_HASH, - CUSTODY_PERIOD_TO_RANDAO_PADDING, - # SSZ - EarlyDerivedSecretReveal, - # functions - get_active_validator_indices, - get_current_epoch, - get_domain, - hash_tree_root, -) - -from .helpers import privkeys - -def get_valid_early_derived_secret_reveal(state, epoch=None): - current_epoch = get_current_epoch(state) - revealed_index = get_active_validator_indices(state, current_epoch)[-1] - masker_index = get_active_validator_indices(state, current_epoch)[0] - - if epoch is None: - epoch = current_epoch + CUSTODY_PERIOD_TO_RANDAO_PADDING - - reveal = bls.sign( - message_hash=hash_tree_root(epoch), - privkey=privkeys[revealed_index], - domain=get_domain( - state=state, - domain_type=spec.DOMAIN_RANDAO, - message_epoch=epoch, - ), - ) - mask = bls.sign( - message_hash=hash_tree_root(epoch), - privkey=privkeys[masker_index], - domain=get_domain( - state=state, - domain_type=spec.DOMAIN_RANDAO, - message_epoch=epoch, - ), - ) - - return EarlyDerivedSecretReveal( - revealed_index=revealed_index, - epoch=epoch, - reveal=reveal, - masker_index=masker_index, - mask=mask, - ) diff --git a/test_libs/pyspec/tests/phase1/test_finality.py b/test_libs/pyspec/tests/phase1/test_finality.py index 4b57c94935..1e4e0374a2 100644 --- a/test_libs/pyspec/tests/phase1/test_finality.py +++ b/test_libs/pyspec/tests/phase1/test_finality.py @@ -1,27 +1,4 @@ -from copy import deepcopy - import pytest -import eth2spec.phase1.spec as spec - -from eth2spec.phase1.spec import ( - state_transition, -) - -from tests.phase0.helpers import ( - build_empty_block_for_next_slot, - fill_aggregate_attestation, - get_current_epoch, - get_epoch_start_slot, - get_valid_attestation, - next_epoch, -) -from tests.phase0.test_finality import ( - pytestmark, - check_finality, - test_finality_rule_1, - test_finality_rule_2, - test_finality_rule_3, - test_finality_rule_4, -) +from tests.phase0.test_finality import * diff --git a/test_libs/pyspec/tests/phase1/test_sanity.py b/test_libs/pyspec/tests/phase1/test_sanity.py index 45a08bd0e3..a9b4d71908 100644 --- a/test_libs/pyspec/tests/phase1/test_sanity.py +++ b/test_libs/pyspec/tests/phase1/test_sanity.py @@ -1,68 +1,4 @@ -from copy import deepcopy - import pytest -from py_ecc import bls -import eth2spec.phase1.spec as spec - -from eth2spec.utils.minimal_ssz import signing_root -from eth2spec.phase1.spec import ( - # constants - ZERO_HASH, - SLOTS_PER_HISTORICAL_ROOT, - # SSZ - Deposit, - Transfer, - VoluntaryExit, - # functions - get_active_validator_indices, - get_beacon_proposer_index, - get_block_root_at_slot, - get_current_epoch, - get_domain, - process_slot, - verify_merkle_branch, - state_transition, - hash, -) -from eth2spec.utils.merkle_minimal import ( - calc_merkle_tree_from_leaves, - get_merkle_proof, - get_merkle_root, -) -from .helpers import ( - advance_slot, - get_balance, - build_deposit_data, - build_empty_block_for_next_slot, - fill_aggregate_attestation, - get_state_root, - get_valid_attestation, - get_valid_attester_slashing, - get_valid_proposer_slashing, - next_slot, - privkeys, - pubkeys, -) - - -# mark entire file as 'sanity' -pytestmark = pytest.mark.sanity -from tests.phase0.test_sanity import ( - test_slot_transition, - test_empty_block_transition, - test_skipped_slots, - test_empty_epoch_transition, - test_empty_epoch_transition_not_finalizing, - test_proposer_slashing, - test_attester_slashing, - test_deposit_in_block, - test_deposit_top_up, - test_attestation, - test_voluntary_exit, - test_transfer, - test_balance_driven_status_transitions, - test_historical_batch, - test_eth1_data_votes, -) +from tests.phase0.test_sanity import * From be4c792fc301b7ee254b61466720e40e155b1601 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 16 May 2019 18:20:20 +0800 Subject: [PATCH 120/308] Change it back to serial workflow --- .circleci/config.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 972f0e3514..f3c5f6a81c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,19 +69,6 @@ jobs: venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv - build_pyspec: - docker: - - image: circleci/python:3.6 - working_directory: ~/specs-repo - steps: - - restore_cache: - key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_cached_venv: - venv_name: v1-pyspec-03 - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - - run: - name: Build pyspec - command: make pyspec test: docker: - image: circleci/python:3.6 @@ -118,13 +105,9 @@ workflows: - install_env: requires: - checkout_specs - - build_pyspec: - requires: - - checkout_specs - test: requires: - install_env - - build_pyspec - lint: requires: - - build_pyspec + - test From 3e085947b7416d82503c7fc8e8674ca002798ed8 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 16 May 2019 12:38:50 +0200 Subject: [PATCH 121/308] Linted :) --- .../eth2spec/phase1/state_transition.py | 19 ++++++------------- .../pyspec/eth2spec/utils/minimal_ssz.py | 13 ++++++++----- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/test_libs/pyspec/eth2spec/phase1/state_transition.py b/test_libs/pyspec/eth2spec/phase1/state_transition.py index 7e6411287d..87b2e1c12f 100644 --- a/test_libs/pyspec/eth2spec/phase1/state_transition.py +++ b/test_libs/pyspec/eth2spec/phase1/state_transition.py @@ -1,11 +1,5 @@ from . import spec -from typing import ( - Any, - Callable, - List -) - from .spec import ( BeaconState, BeaconBlock, @@ -13,18 +7,14 @@ ) from eth2spec.phase0.state_transition import ( - expected_deposit_count, process_operation_type, process_operations as process_operations_phase0, - process_block as process_block_phase0, - process_epoch_transition as process_epoch_transition_phase0, - state_transition_to as state_transition_to_phase0, - state_transition as state_transition_phase0 ) + def process_operations(state: BeaconState, block: BeaconBlock) -> None: process_operations_phase0(state, block) - + process_operation_type( state, block.body.custody_key_reveals, @@ -39,6 +29,7 @@ def process_operations(state: BeaconState, block: BeaconBlock) -> None: spec.process_early_derived_secret_reveal, ) + def process_block(state: BeaconState, block: BeaconBlock, verify_state_root: bool=False) -> None: @@ -50,6 +41,7 @@ def process_block(state: BeaconState, if verify_state_root: spec.verify_block_state_root(state, block) + def process_epoch_transition(state: BeaconState) -> None: spec.process_justification_and_finalization(state) spec.process_crosslinks(state) @@ -62,6 +54,7 @@ def process_epoch_transition(state: BeaconState) -> None: spec.process_final_updates(state) spec.after_process_final_updates(state) + def state_transition_to(state: BeaconState, up_to: Slot) -> BeaconState: while state.slot < up_to: spec.cache_state(state) @@ -74,4 +67,4 @@ def state_transition(state: BeaconState, block: BeaconBlock, verify_state_root: bool=False) -> BeaconState: state_transition_to(state, block.slot) - process_block(state, block, verify_state_root) \ No newline at end of file + process_block(state, block, verify_state_root) diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py index 8a210f41ba..57d9579d94 100644 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py @@ -8,12 +8,13 @@ cached_typedefs = {} + def SSZType(name, fields): class SSZObject(): - if name != None: + if name is not None: __name__ = name __qualname__ = name - + def __init__(self, **kwargs): for f, t in fields.items(): if f not in kwargs: @@ -32,7 +33,7 @@ def __str__(self): for field in self.fields: output.append(f'{field}: {repr(getattr(self, field))},') return "\n".join(output) - + def __repr__(self): return name + "(**{\n " + str(self).replace("\n", "\n ") + "\n})" @@ -43,17 +44,19 @@ def hash_tree_root(self): return hash_tree_root(self, self.__class__) SSZObject.fields = fields - - if name != None: + + if name is not None: cached_typedefs[name] = SSZObject return SSZObject + def SSZTypeExtension(original_type, new_fields): typedef = cached_typedefs[original_type] typedef.fields.update(new_fields) return typedef + class Vector(): def __init__(self, items): self.items = items From 49d708e44bba654c779b000358ec41618756a7f8 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 16 May 2019 12:51:36 +0200 Subject: [PATCH 122/308] Ok, lets at least pass tests :/ --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c3ec212808..2ddc82e68a 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,9 @@ test_phase1: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest tests/phase1 citest: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml . + cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; \ + python -m pytest --junitxml=test-reports/eth2spec/test_results_phase0.xml tests/phase0; \ + python -m pytest --junitxml=test-reports/eth2spec/test_results_phase0.xml tests/phase1 install_lint: cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.5.0 From 14ea7d9f40dfd0cb485e09f752a783549d2b3661 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 16 May 2019 16:36:35 +0200 Subject: [PATCH 123/308] new spec buildign stratergy --- Makefile | 6 +- scripts/{phase0 => }/__init__.py | 0 scripts/build_spec.py | 196 ++++++++++++++++++++++++ scripts/{phase0 => }/function_puller.py | 0 scripts/phase0/build_spec.py | 143 ----------------- 5 files changed, 199 insertions(+), 146 deletions(-) rename scripts/{phase0 => }/__init__.py (100%) create mode 100644 scripts/build_spec.py rename scripts/{phase0 => }/function_puller.py (100%) delete mode 100644 scripts/phase0/build_spec.py diff --git a/Makefile b/Makefile index 2ddc82e68a..a262ac5e1a 100644 --- a/Makefile +++ b/Makefile @@ -67,11 +67,11 @@ phase0: $(PY_SPEC_PHASE_0_TARGETS) # "make phase1" to create pyspec for phase1 phase1: $(PY_SPEC_PHASE_1_TARGETS) -$(PY_SPEC_DIR)/eth2spec/phase0/spec.py: $(PY_SPEC_PHASE_0_DEPS) - python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@ +$(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) + python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) - python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_custody-game.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $@ CURRENT_DIR = ${CURDIR} diff --git a/scripts/phase0/__init__.py b/scripts/__init__.py similarity index 100% rename from scripts/phase0/__init__.py rename to scripts/__init__.py diff --git a/scripts/build_spec.py b/scripts/build_spec.py new file mode 100644 index 0000000000..c77a8b82f2 --- /dev/null +++ b/scripts/build_spec.py @@ -0,0 +1,196 @@ +import sys +import re +import function_puller +from argparse import ArgumentParser +from typing import Tuple, List + + +def split_retain_delimiter(regex_pattern: str, text: str) -> List[str]: + ''' + Splits a string based on regex, but down not remove the matched text + ''' + find_pattern = r'%s.*?(?=%s|$)' % (regex_pattern, regex_pattern) + return re.findall(find_pattern, text, re.DOTALL) + + +def inserter(oldfile: str, newfile: str) -> Tuple[str, str]: + ''' + Searches for occurrences of @LabelName in oldfile and replaces them with instances of code wraped as follows: + # begin insert @LabelName + def foo(bar): + return bar + # end insert @LabelName + ''' + new_insert_objects = re.split(r"(# begin insert |# end insert @[a-zA-Z0-9_]*\n)", newfile) + # Retrieve label from insert objects + def get_labeled_object(labeled_text): + label = re.match(r"@[a-zA-Z0-9_]*\n", labeled_text) + if label is not None: + label = label.group(0) + labeled_text = re.sub(label, '', labeled_text) + return {'label': label, 'text': labeled_text} + new_insert_objects = map(get_labeled_object, new_insert_objects) + # Find and replace labels + newfile = "" + for item in new_insert_objects: + if item['label'] is not None: + oldfile, insertions = re.subn('# %s' % item['label'], item['text'], oldfile) + if insertions == 0: + newfile.join('# begin insert %s/n%s# end insert %s' % (item['label'], item['text'], item['label'])) + elif re.match(r"(# begin insert |# end insert )", item['text']) is None: + newfile += item['text'] + return oldfile, newfile + + +def merger(oldfile:str, newfile:str) -> str: + ''' + Seeks out functions and objects in new and old files. + Replaces old objects with new ones if they exist. + ''' + old_objects = split_retain_delimiter('\n[a-zA-Z]', oldfile) + new_objects = split_retain_delimiter('\n[a-zA-Z]', newfile) + object_regex = r"\n[#@a-zA-Z_0-9]+[\sa-zA-Z_0-9]*[(){}=:'" "]*" + old_object_tuples = list(map(lambda x: [re.match(object_regex, x).group(0),x], old_objects)) + for new_item in new_objects: + found_old = False + for old_item in old_object_tuples: + if old_item[0] == re.match(object_regex, new_item).group(0): + print(old_item[0]) + old_item[1] = new_item + found_old = True + break + if not found_old: + old_object_tuples += [[re.match(object_regex, new_item).group(0), new_item]] + return ''.join(elem for elem in map(lambda x: x[1], old_object_tuples)) + + +def build_phase0_spec(sourcefile, outfile=None): + code_lines = [] + code_lines.append(""" + +from typing import ( + Any, + Dict, + List, + NewType, + Tuple, +) +from eth2spec.utils.minimal_ssz import ( + SSZType, + hash_tree_root, + signing_root, +) +from eth2spec.utils.bls_stub import ( + bls_aggregate_pubkeys, + bls_verify, + bls_verify_multiple, +) +from eth2spec.utils.hash_function import hash + + +# stub, will get overwritten by real var +SLOTS_PER_EPOCH = 64 + +Slot = NewType('Slot', int) # uint64 +Epoch = NewType('Epoch', int) # uint64 +Shard = NewType('Shard', int) # uint64 +ValidatorIndex = NewType('ValidatorIndex', int) # uint64 +Gwei = NewType('Gwei', int) # uint64 +Bytes32 = NewType('Bytes32', bytes) # bytes32 +BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 +BLSSignature = NewType('BLSSignature', bytes) # bytes96 +Store = None +""") + + code_lines += function_puller.get_spec(sourcefile) + + code_lines.append(""" +# Monkey patch validator compute committee code +_compute_committee = compute_committee +committee_cache = {} + + +def compute_committee(indices: List[ValidatorIndex], seed: Bytes32, index: int, count: int) -> List[ValidatorIndex]: + param_hash = (hash_tree_root(indices), seed, index, count) + + if param_hash in committee_cache: + return committee_cache[param_hash] + else: + ret = _compute_committee(indices, seed, index, count) + committee_cache[param_hash] = ret + return ret + + +# Monkey patch hash cache +_hash = hash +hash_cache = {} + + +def hash(x): + if x in hash_cache: + return hash_cache[x] + else: + ret = _hash(x) + hash_cache[x] = ret + return ret + + +# Access to overwrite spec constants based on configuration +def apply_constants_preset(preset: Dict[str, Any]): + global_vars = globals() + for k, v in preset.items(): + global_vars[k] = v + + # Deal with derived constants + global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT) + + # Initialize SSZ types again, to account for changed lengths + init_SSZ_types() +""") + + if outfile is not None: + with open(outfile, 'w') as out: + out.write("\n".join(code_lines)) + else: + return "\n".join(code_lines) + + +def build_phase1_spec(phase0_sourcefile, phase1_sourcefile, outfile=None): + phase0_code = build_phase0_spec(phase0_sourcefile) + phase1_code = build_phase0_spec(phase1_sourcefile) + phase0_code, phase1_code = inserter(phase0_code, phase1_code) + phase1_code = merger(phase0_code, phase1_code) + + if outfile is not None: + with open(outfile, 'w') as out: + out.write(phase1_code) + else: + return phase1_code + + +if __name__ == '__main__': + description = ''' +Build the specs from the md docs. +If building phase 0: + 1st argument is input spec.md + 2nd argument is output spec.py + +If building phase 1: + 1st argument is input spec_phase0.md + 2nd argument is input spec_phase1.md + 3rd argument is output spec.py +''' + parser = ArgumentParser(description=description) + parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #") + parser.add_argument(dest="files", help="Input and output files", nargs="+") + + args = parser.parse_args() + if args.phase == 0: + build_phase0_spec(*args.files) + elif args.phase == 1: + if len(args.files) == 3: + build_phase1_spec(*args.files) + else: + print(" Phase 1 requires an output as well as 2 input files (phase0.md and phase1.md)") + else: + print("Invalid phase: {0}".format(args.phase)) diff --git a/scripts/phase0/function_puller.py b/scripts/function_puller.py similarity index 100% rename from scripts/phase0/function_puller.py rename to scripts/function_puller.py diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py deleted file mode 100644 index 4faf8cbf64..0000000000 --- a/scripts/phase0/build_spec.py +++ /dev/null @@ -1,143 +0,0 @@ -import sys -import function_puller -from optparse import OptionParser - -def build_phase0_spec(sourcefile, outfile): - code_lines = [] - code_lines.append(""" - -from typing import ( - Any, - Dict, - List, - NewType, - Tuple, -) -from eth2spec.utils.minimal_ssz import ( - SSZType, - SSZTypeExtension, - hash_tree_root, - signing_root, -) -from eth2spec.utils.bls_stub import ( - bls_aggregate_pubkeys, - bls_verify, - bls_verify_multiple, -) -from eth2spec.utils.hash_function import hash - - -# stub, will get overwritten by real var -SLOTS_PER_EPOCH = 64 - -Slot = NewType('Slot', int) # uint64 -Epoch = NewType('Epoch', int) # uint64 -Shard = NewType('Shard', int) # uint64 -ValidatorIndex = NewType('ValidatorIndex', int) # uint64 -Gwei = NewType('Gwei', int) # uint64 -Bytes32 = NewType('Bytes32', bytes) # bytes32 -BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 -BLSSignature = NewType('BLSSignature', bytes) # bytes96 -Store = None -""") - - code_lines += function_puller.get_spec(sourcefile) - - code_lines.append(""" -# Monkey patch validator compute committee code -_compute_committee = compute_committee -committee_cache = {} - - -def compute_committee(indices: List[ValidatorIndex], seed: Bytes32, index: int, count: int) -> List[ValidatorIndex]: - param_hash = (hash_tree_root(indices), seed, index, count) - - if param_hash in committee_cache: - return committee_cache[param_hash] - else: - ret = _compute_committee(indices, seed, index, count) - committee_cache[param_hash] = ret - return ret - - -# Monkey patch hash cache -_hash = hash -hash_cache = {} - - -def hash(x): - if x in hash_cache: - return hash_cache[x] - else: - ret = _hash(x) - hash_cache[x] = ret - return ret - - -# Access to overwrite spec constants based on configuration -def apply_constants_preset(preset: Dict[str, Any]): - global_vars = globals() - for k, v in preset.items(): - global_vars[k] = v - - # Deal with derived constants - global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT) - - # Initialize SSZ types again, to account for changed lengths - init_SSZ_types() -""") - - with open(outfile, 'w') as out: - out.write("\n".join(code_lines)) - - -def build_phase1_spec(sourcefile, outfile): - code_lines = [] - code_lines.append(""" -from eth2spec.phase0.spec import * -from eth2spec.phase0.spec import apply_constants_preset as apply_constants_preset_phase0 - -""") - - code_lines += function_puller.get_spec(sourcefile) - - code_lines.append(""" -# Access to overwrite spec constants based on configuration -def apply_constants_preset(preset: Dict[str, Any]): - - apply_constants_preset_phase0(preset) - - global_vars = globals() - for k, v in preset.items(): - global_vars[k] = v - - # Deal with derived constants - global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT) - - # Initialize SSZ types again, to account for changed lengths - init_SSZ_types() -""") - - with open(outfile, 'w') as out: - out.write("\n".join(code_lines)) - - -if __name__ == '__main__': - parser = OptionParser() - parser.add_option("-p", "--phase", dest="phase", type="int", default=0, - help="Build for phase #") - - (options, args) = parser.parse_args() - - if len(args) < 2: - parser.print_help() - - if options.phase == 0: - build_phase0_spec(args[0], args[1]) - print(args[0]) - print(args[1]) - elif options.phase == 1: - build_phase1_spec(args[0], args[1]) - else: - print("Invalid phase: {0}".format(options["phase"])) - From f6f7d53735ee456264cae0fe4a0dc5883f4164d6 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 16 May 2019 17:14:07 +0200 Subject: [PATCH 124/308] simplify function_puller --- scripts/function_puller.py | 20 ++------- scripts/phase0/function_puller.py | 75 +++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 17 deletions(-) create mode 100644 scripts/phase0/function_puller.py diff --git a/scripts/function_puller.py b/scripts/function_puller.py index 418eb05b11..787c2ff06c 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -2,26 +2,16 @@ from typing import List -def get_spec(file_name: str, phase:int = 0) -> List[str]: +def get_spec(file_name: str) -> List[str]: code_lines = [] pulling_from = None current_name = None current_typedef = None - is_update_section = False - update_section_depth = None type_defs = [] for linenum, line in enumerate(open(file_name).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': current_name = line[line[:-1].rfind('`') + 1: -1] - if pulling_from is None and len(line) > 0 and line[0] == '#' and line.endswith('updates'): - is_update_section = True - update_section_depth = max(i for i in range(10) if line.startswith('#' * i)) - elif pulling_from is None and len(line) > 0 and line[0] == '#' and is_update_section: - section_depth = max(i for i in range(10) if line.startswith('#' * i)) - if section_depth <= update_section_depth: - is_update_section = False - update_section_depth = None if line[:9] == '```python': assert pulling_from is None pulling_from = linenum + 1 @@ -38,12 +28,8 @@ def get_spec(file_name: str, phase:int = 0) -> List[str]: current_typedef = None else: if pulling_from == linenum and line == '{': - if is_update_section: - code_lines.append('%s = SSZTypeExtension("%s", {' % (current_name, current_name)) - current_typedef = ['global_vars["%s"] = SSZTypeExtension("%s", {' % (current_name, current_name)] - else: - code_lines.append('%s = SSZType("%s", {' % (current_name, current_name)) - current_typedef = ['global_vars["%s"] = SSZType("%s", {' % (current_name, current_name)] + code_lines.append('%s = SSZType({' % current_name) + current_typedef = ['global_vars["%s"] = SSZType({' % current_name] elif pulling_from is not None: # Add some whitespace between functions if line[:3] == 'def': diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py new file mode 100644 index 0000000000..750f195904 --- /dev/null +++ b/scripts/phase0/function_puller.py @@ -0,0 +1,75 @@ +import sys +from typing import List + + +def get_spec(file_name: str) -> List[str]: + code_lines = [] + pulling_from = None + current_name = None + current_typedef = None + type_defs = [] + for linenum, line in enumerate(open(sys.argv[1]).readlines()): + line = line.rstrip() + if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': + current_name = line[line[:-1].rfind('`') + 1: -1] + if line[:9] == '```python': + assert pulling_from is None + pulling_from = linenum + 1 + elif line[:3] == '```': + if pulling_from is None: + pulling_from = linenum + else: + if current_typedef is not None: + assert code_lines[-1] == '}' + code_lines[-1] = '})' + current_typedef[-1] = '})' + type_defs.append((current_name, current_typedef)) + pulling_from = None + current_typedef = None + else: + if pulling_from == linenum and line == '{': + code_lines.append('%s = SSZType({' % current_name) + current_typedef = ['global_vars["%s"] = SSZType({' % current_name] + elif pulling_from is not None: + # Add some whitespace between functions + if line[:3] == 'def': + code_lines.append('') + code_lines.append('') + code_lines.append(line) + # Remember type def lines + if current_typedef is not None: + current_typedef.append(line) + elif pulling_from is None and len(line) > 0 and line[0] == '|': + row = line[1:].split('|') + if len(row) >= 2: + for i in range(2): + row[i] = row[i].strip().strip('`') + if '`' in row[i]: + row[i] = row[i][:row[i].find('`')] + eligible = True + if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_': + eligible = False + for c in row[0]: + if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': + eligible = False + if eligible: + code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890'))) + # Build type-def re-initialization + code_lines.append('\n') + code_lines.append('def init_SSZ_types():') + code_lines.append(' global_vars = globals()') + for ssz_type_name, ssz_type in type_defs: + code_lines.append('') + for type_line in ssz_type: + if len(type_line) > 0: + code_lines.append(' ' + type_line) + code_lines.append('\n') + code_lines.append('ssz_types = [\n') + for (ssz_type_name, _) in type_defs: + code_lines.append(f' {ssz_type_name},\n') + code_lines.append(']') + code_lines.append('\n') + code_lines.append('def get_ssz_type_by_name(name: str) -> SSZType:') + code_lines.append(' return globals()[name]') + code_lines.append('') + return code_lines From 9f2d410a1737973a54f12ddf38832145ef4e9dcd Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 16 May 2019 17:15:45 +0200 Subject: [PATCH 125/308] removes old files --- scripts/phase0/function_puller.py | 75 ------------------------------- 1 file changed, 75 deletions(-) delete mode 100644 scripts/phase0/function_puller.py diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py deleted file mode 100644 index 750f195904..0000000000 --- a/scripts/phase0/function_puller.py +++ /dev/null @@ -1,75 +0,0 @@ -import sys -from typing import List - - -def get_spec(file_name: str) -> List[str]: - code_lines = [] - pulling_from = None - current_name = None - current_typedef = None - type_defs = [] - for linenum, line in enumerate(open(sys.argv[1]).readlines()): - line = line.rstrip() - if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': - current_name = line[line[:-1].rfind('`') + 1: -1] - if line[:9] == '```python': - assert pulling_from is None - pulling_from = linenum + 1 - elif line[:3] == '```': - if pulling_from is None: - pulling_from = linenum - else: - if current_typedef is not None: - assert code_lines[-1] == '}' - code_lines[-1] = '})' - current_typedef[-1] = '})' - type_defs.append((current_name, current_typedef)) - pulling_from = None - current_typedef = None - else: - if pulling_from == linenum and line == '{': - code_lines.append('%s = SSZType({' % current_name) - current_typedef = ['global_vars["%s"] = SSZType({' % current_name] - elif pulling_from is not None: - # Add some whitespace between functions - if line[:3] == 'def': - code_lines.append('') - code_lines.append('') - code_lines.append(line) - # Remember type def lines - if current_typedef is not None: - current_typedef.append(line) - elif pulling_from is None and len(line) > 0 and line[0] == '|': - row = line[1:].split('|') - if len(row) >= 2: - for i in range(2): - row[i] = row[i].strip().strip('`') - if '`' in row[i]: - row[i] = row[i][:row[i].find('`')] - eligible = True - if row[0][0] not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_': - eligible = False - for c in row[0]: - if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': - eligible = False - if eligible: - code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890'))) - # Build type-def re-initialization - code_lines.append('\n') - code_lines.append('def init_SSZ_types():') - code_lines.append(' global_vars = globals()') - for ssz_type_name, ssz_type in type_defs: - code_lines.append('') - for type_line in ssz_type: - if len(type_line) > 0: - code_lines.append(' ' + type_line) - code_lines.append('\n') - code_lines.append('ssz_types = [\n') - for (ssz_type_name, _) in type_defs: - code_lines.append(f' {ssz_type_name},\n') - code_lines.append(']') - code_lines.append('\n') - code_lines.append('def get_ssz_type_by_name(name: str) -> SSZType:') - code_lines.append(' return globals()[name]') - code_lines.append('') - return code_lines From 306a613c8a1dcb516fb0901c92e320aff3d2c5aa Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 16 May 2019 21:01:32 +0200 Subject: [PATCH 126/308] Simplify minimal SSZ again --- scripts/build_spec.py | 18 ++++++++------- .../pyspec/eth2spec/utils/minimal_ssz.py | 23 +++---------------- 2 files changed, 13 insertions(+), 28 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index c77a8b82f2..705808d9ff 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -27,7 +27,7 @@ def get_labeled_object(labeled_text): label = re.match(r"@[a-zA-Z0-9_]*\n", labeled_text) if label is not None: label = label.group(0) - labeled_text = re.sub(label, '', labeled_text) + labeled_text = re.sub(label, '', labeled_text) return {'label': label, 'text': labeled_text} new_insert_objects = map(get_labeled_object, new_insert_objects) # Find and replace labels @@ -53,9 +53,8 @@ def merger(oldfile:str, newfile:str) -> str: old_object_tuples = list(map(lambda x: [re.match(object_regex, x).group(0),x], old_objects)) for new_item in new_objects: found_old = False - for old_item in old_object_tuples: + for old_item in old_object_tuples: if old_item[0] == re.match(object_regex, new_item).group(0): - print(old_item[0]) old_item[1] = new_item found_old = True break @@ -67,7 +66,6 @@ def merger(oldfile:str, newfile:str) -> str: def build_phase0_spec(sourcefile, outfile=None): code_lines = [] code_lines.append(""" - from typing import ( Any, Dict, @@ -75,16 +73,19 @@ def build_phase0_spec(sourcefile, outfile=None): NewType, Tuple, ) + from eth2spec.utils.minimal_ssz import ( SSZType, hash_tree_root, signing_root, ) + from eth2spec.utils.bls_stub import ( bls_aggregate_pubkeys, bls_verify, bls_verify_multiple, ) + from eth2spec.utils.hash_function import hash @@ -146,6 +147,7 @@ def apply_constants_preset(preset: Dict[str, Any]): # Initialize SSZ types again, to account for changed lengths init_SSZ_types() + """) if outfile is not None: @@ -160,7 +162,7 @@ def build_phase1_spec(phase0_sourcefile, phase1_sourcefile, outfile=None): phase1_code = build_phase0_spec(phase1_sourcefile) phase0_code, phase1_code = inserter(phase0_code, phase1_code) phase1_code = merger(phase0_code, phase1_code) - + if outfile is not None: with open(outfile, 'w') as out: out.write(phase1_code) @@ -172,12 +174,12 @@ def build_phase1_spec(phase0_sourcefile, phase1_sourcefile, outfile=None): description = ''' Build the specs from the md docs. If building phase 0: - 1st argument is input spec.md + 1st argument is input spec.md 2nd argument is output spec.py If building phase 1: - 1st argument is input spec_phase0.md - 2nd argument is input spec_phase1.md + 1st argument is input spec_phase0.md + 2nd argument is input spec_phase1.md 3rd argument is output spec.py ''' parser = ArgumentParser(description=description) diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py index 57d9579d94..f7784b6de6 100644 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py @@ -9,12 +9,8 @@ cached_typedefs = {} -def SSZType(name, fields): +def SSZType(fields): class SSZObject(): - if name is not None: - __name__ = name - __qualname__ = name - def __init__(self, **kwargs): for f, t in fields.items(): if f not in kwargs: @@ -31,12 +27,9 @@ def __hash__(self): def __str__(self): output = [] for field in self.fields: - output.append(f'{field}: {repr(getattr(self, field))},') + output.append(f'{field}: {getattr(self, field)}') return "\n".join(output) - def __repr__(self): - return name + "(**{\n " + str(self).replace("\n", "\n ") + "\n})" - def serialize(self): return serialize_value(self, self.__class__) @@ -44,19 +37,9 @@ def hash_tree_root(self): return hash_tree_root(self, self.__class__) SSZObject.fields = fields - - if name is not None: - cached_typedefs[name] = SSZObject - return SSZObject -def SSZTypeExtension(original_type, new_fields): - typedef = cached_typedefs[original_type] - typedef.fields.update(new_fields) - return typedef - - class Vector(): def __init__(self, items): self.items = items @@ -334,7 +317,7 @@ def truncate(container): key: container.fields[key] for key in field_keys[:-1] } - truncated_class = SSZType(None, truncated_fields) + truncated_class = SSZType(truncated_fields) kwargs = { field: getattr(container, field) for field in field_keys[:-1] From 7c8f83d5e84d04274eb61e3faea9f5d09b727c38 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 16 May 2019 23:52:40 +0200 Subject: [PATCH 127/308] partial combination of phase0 and phase1 implemented --- scripts/build_spec.py | 39 +++++++++++++++++++++++--------------- scripts/function_puller.py | 5 ----- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 705808d9ff..a18107528a 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -5,12 +5,14 @@ from typing import Tuple, List -def split_retain_delimiter(regex_pattern: str, text: str) -> List[str]: +def split_and_label(regex_pattern: str, text: str) -> List[str]: ''' - Splits a string based on regex, but down not remove the matched text + Splits a string based on regex, but down not remove the matched text. + It subsequently labels the matches with their match ''' - find_pattern = r'%s.*?(?=%s|$)' % (regex_pattern, regex_pattern) - return re.findall(find_pattern, text, re.DOTALL) + find_pattern = r'''%s.*?(?=%s|$)''' % (regex_pattern, regex_pattern) + matches = re.findall(find_pattern, text, re.DOTALL) + return list(map(lambda x: [re.match(regex_pattern, x).group(0), x], matches)) def inserter(oldfile: str, newfile: str) -> Tuple[str, str]: @@ -21,10 +23,10 @@ def foo(bar): return bar # end insert @LabelName ''' - new_insert_objects = re.split(r"(# begin insert |# end insert @[a-zA-Z0-9_]*\n)", newfile) + new_insert_objects = re.split(r"(# begin insert |# end insert @[\w\d_]*\n)", newfile) # Retrieve label from insert objects def get_labeled_object(labeled_text): - label = re.match(r"@[a-zA-Z0-9_]*\n", labeled_text) + label = re.match(r"@[\w\d_]*\n", labeled_text) if label is not None: label = label.group(0) labeled_text = re.sub(label, '', labeled_text) @@ -47,20 +49,27 @@ def merger(oldfile:str, newfile:str) -> str: Seeks out functions and objects in new and old files. Replaces old objects with new ones if they exist. ''' - old_objects = split_retain_delimiter('\n[a-zA-Z]', oldfile) - new_objects = split_retain_delimiter('\n[a-zA-Z]', newfile) - object_regex = r"\n[#@a-zA-Z_0-9]+[\sa-zA-Z_0-9]*[(){}=:'" "]*" - old_object_tuples = list(map(lambda x: [re.match(object_regex, x).group(0),x], old_objects)) + object_regex = r'''(?:\n[@\w]+[\s\w]*[='" "\.\w]*)|(?:\s{4}global_vars\["\w+"\])''' + ssz_object_regex = r'''(?:\w+|\s{4}global_vars\["\w+"\]) = SSZType\(\{\n''' + old_objects = split_and_label(object_regex, oldfile) + new_objects = split_and_label(object_regex, newfile) for new_item in new_objects: found_old = False - for old_item in old_object_tuples: - if old_item[0] == re.match(object_regex, new_item).group(0): - old_item[1] = new_item + for index, old_item in enumerate(old_objects): + if old_item[0] == new_item[0]: + ssz_object_match = re.match(ssz_object_regex, new_item[1]) + if ssz_object_match is not None: + new_item[1] = re.sub(ssz_object_regex, '', new_item[1]) + old_item[1] = re.sub(r'\n\w*\}\)', '', old_item[1]) + old_item[1] += new_item[1] + else: + old_item[1] = new_item[1] found_old = True + old_objects[index] = old_item break if not found_old: - old_object_tuples += [[re.match(object_regex, new_item).group(0), new_item]] - return ''.join(elem for elem in map(lambda x: x[1], old_object_tuples)) + old_objects.append(new_item) + return ''.join(elem for elem in map(lambda x: x[1], old_objects)) def build_phase0_spec(sourcefile, outfile=None): diff --git a/scripts/function_puller.py b/scripts/function_puller.py index 787c2ff06c..97bc628217 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -64,11 +64,6 @@ def get_spec(file_name: str) -> List[str]: if len(type_line) > 0: code_lines.append(' ' + type_line) code_lines.append('\n') - code_lines.append('ssz_types = [\n') - for (ssz_type_name, _) in type_defs: - code_lines.append(f' {ssz_type_name},\n') - code_lines.append(']') - code_lines.append('\n') code_lines.append('def get_ssz_type_by_name(name: str) -> SSZType:') code_lines.append(' return globals()[name]') code_lines.append('') From 7f6896cca3931c0e4a6b352bfd0ba3fa3dca3212 Mon Sep 17 00:00:00 2001 From: Matthew Slipper Date: Thu, 16 May 2019 16:43:10 -0700 Subject: [PATCH 128/308] Update to use union type --- specs/networking/rpc-interface.md | 34 ++++++++++--------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/specs/networking/rpc-interface.md b/specs/networking/rpc-interface.md index ca6008a406..48ee9333b0 100644 --- a/specs/networking/rpc-interface.md +++ b/specs/networking/rpc-interface.md @@ -39,36 +39,24 @@ To facilitate RPC-over-`libp2p`, a single protocol name is used: `/eth/serenity/ Remote method calls are wrapped in a "request" structure: ``` - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ id (uint64) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| method_id (uint16) | body_len (uint32) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | body (body_len bytes) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +( + id: uint64 + method_id: uint16 + body: (message_body...) +) ``` and their corresponding responses are wrapped in a "response" structure: ``` - 0 1 2 3 - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | -+ id (uint64) + -| | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| response_code (uint16) | result_len (uint32) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -| | result (result_len bytes) | -+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +( + id: uint64 + response_code: uint16 + result: bytes +) ``` -Note that the above structures are NOT encoded as SSZ but rather as sequences of bytes according to the packet diagrams above. This is because SSZ does not support structures without an explicit schema. Since the `body` and `result` fields depend on the value of `method_id` and `response_code`, a schema for the above structure cannot be known beforehand. +A union type is used to determine the contents of the `body` field in the request structure. Each "body" entry in the RPC calls below corresponds to one subtype in the `body` type union. The details of the RPC-Over-`libp2p` protocol are similar to [JSON-RPC 2.0](https://www.jsonrpc.org/specification). Specifically: From 05f9dc7baa356402a5a52b2bc0f910340ce75c52 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 17 May 2019 05:59:01 -0400 Subject: [PATCH 129/308] Fix #1090 Avoid signed integer --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ee37d92175..aa8cf57ca1 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -887,7 +887,7 @@ def get_shuffled_index(index: ValidatorIndex, index_count: int, seed: Bytes32) - # See the 'generalized domain' algorithm on page 3 for round in range(SHUFFLE_ROUND_COUNT): pivot = bytes_to_int(hash(seed + int_to_bytes(round, length=1))[0:8]) % index_count - flip = (pivot - index) % index_count + flip = (pivot + index_count - index) % index_count position = max(index, flip) source = hash(seed + int_to_bytes(round, length=1) + int_to_bytes(position // 256, length=4)) byte = source[(position % 256) // 8] From f19188816b2bac23b41222181be1bde31fdc79d1 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 17 May 2019 06:07:38 -0400 Subject: [PATCH 130/308] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index aa8cf57ca1..622eabaf8f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1362,19 +1362,19 @@ def process_justification_and_finalization(state: BeaconState) -> None: # Process finalizations bitfield = state.justification_bitfield # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch == current_epoch - 3: + if (bitfield >> 1) % 8 == 0b111 and old_previous_justified_epoch + 3 == current_epoch: state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch == current_epoch - 2: + if (bitfield >> 1) % 4 == 0b11 and old_previous_justified_epoch + 2 == current_epoch: state.finalized_epoch = old_previous_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch == current_epoch - 2: + if (bitfield >> 0) % 8 == 0b111 and old_current_justified_epoch + 2 == current_epoch: state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch == current_epoch - 1: + if (bitfield >> 0) % 4 == 0b11 and old_current_justified_epoch + 1 == current_epoch: state.finalized_epoch = old_current_justified_epoch state.finalized_root = get_block_root(state, state.finalized_epoch) ``` From 694b31b934d13ffbddf17574fa7f04b824d67c98 Mon Sep 17 00:00:00 2001 From: Justin Date: Fri, 17 May 2019 06:11:39 -0400 Subject: [PATCH 131/308] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 622eabaf8f..b75f14153a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -647,7 +647,7 @@ def get_previous_epoch(state: BeaconState) -> Epoch: Return the current epoch if it's genesis epoch. """ current_epoch = get_current_epoch(state) - return (current_epoch - 1) if current_epoch > GENESIS_EPOCH else current_epoch + return GENESIS_EPOCH if current_epoch == GENESIS_EPOCH else current_epoch - 1 ``` ### `get_current_epoch` From 37aca60fae778268841a3e64253b85bb616b61ed Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 17 May 2019 10:55:07 -0400 Subject: [PATCH 132/308] pr feedback --- specs/validator/0_beacon-chain-validator.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 5c92c99534..46eb551732 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -165,11 +165,11 @@ def is_proposer(state: BeaconState, return get_beacon_proposer_index(state) == validator_index ``` -*Note*: To see if a validator is assigned to proposer during the slot, the validator must run an empty slot transition from the previous state to the current slot using `process_slots(state, current_slot)`. +*Note*: To see if a validator is assigned to propose during the slot, the beacon state must be in the epoch in question. At the epoch boundaries, the validator must run an epoch transition into the epoch to successfully check the proposal assignment of the first slot. ### Lookahead -The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must be checked during the slot in question. +The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must be checked during the epoch in question. `get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in Phase 1+). @@ -181,7 +181,7 @@ A validator has two primary responsibilities to the beacon chain: [proposing blo ### Block proposal -A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator is to create, sign, and broadcast a `block` that is a child of `parent` and that executes a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). +A validator is expected to propose a [`BeaconBlock`](../core/0_beacon-chain.md#beaconblock) at the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. To propose, the validator selects the `BeaconBlock`, `parent`, that in their view of the fork choice is the head of the chain during `slot - 1`. The validator creates, signs, and broadcasts a `block` that is a child of `parent` that satisfies a valid [beacon chain state transition](../core/0_beacon-chain.md#beacon-chain-state-transition-function). There is one proposer per slot, so if there are N active validators any individual validator will on average be assigned to propose once per N slots (e.g. at 312500 validators = 10 million ETH, that's once per ~3 weeks). From acb7444184b533de6c72fc65be1b0f7c506414c3 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 17 May 2019 10:58:02 -0400 Subject: [PATCH 133/308] pr feedback --- specs/validator/0_beacon-chain-validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 46eb551732..302deeae12 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -157,7 +157,7 @@ def get_committee_assignment( return committee, shard, slot ``` -A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run during the slot in question. Proposer selection is only stable within the context of the current epoch. +A validator can use the following function to see if they are supposed to propose during their assigned committee slot. This function can only be run with a `state` of the slot in question. Proposer selection is only stable within the context of the current epoch. ```python def is_proposer(state: BeaconState, @@ -171,7 +171,7 @@ def is_proposer(state: BeaconState, The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. Note that this lookahead does not apply to proposing, which must be checked during the epoch in question. -`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments which involves noting at which future slot one will have to attest and also which shard one should begin syncing (in Phase 1+). +`get_committee_assignment` should be called at the start of each epoch to get the assignment for the next epoch (`current_epoch + 1`). A validator should plan for future assignments by noting at which future slot they will have to attest and also which shard they should begin syncing (in Phase 1+). Specifically, a validator should call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. From e1d973d5467e25fe496cc67ba3800aab096d1c54 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 17 May 2019 11:01:18 -0400 Subject: [PATCH 134/308] Update specs/validator/0_beacon-chain-validator.md Co-Authored-By: Hsiao-Wei Wang --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 302deeae12..9e4fa7960b 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -148,7 +148,7 @@ def get_committee_assignment( committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH epoch_start_slot = get_epoch_start_slot(epoch) - for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH) + for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH): slot_start_shard = get_epoch_start_shard(state, epoch) + committees_per_slot * (slot % SLOTS_PER_EPOCH) for i in range(committees_per_slot): shard = (slot_start_shard + i) % SHARD_COUNT From 174e1e4dbef7605f1e4131f66d3fd0f52276b6a4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 17 May 2019 11:04:05 -0400 Subject: [PATCH 135/308] pr feedback --- specs/validator/0_beacon-chain-validator.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 302deeae12..0940d592a6 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -149,7 +149,8 @@ def get_committee_assignment( committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH epoch_start_slot = get_epoch_start_slot(epoch) for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH) - slot_start_shard = get_epoch_start_shard(state, epoch) + committees_per_slot * (slot % SLOTS_PER_EPOCH) + offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) + slot_start_shard = (get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT for i in range(committees_per_slot): shard = (slot_start_shard + i) % SHARD_COUNT committee = get_crosslink_committee(state, epoch, shard) From 24edca3456166c11ebce912d3397c1618b79a3dc Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 17 May 2019 13:52:23 -0400 Subject: [PATCH 136/308] Fix to make Danny and hww happy --- specs/core/1_custody-game.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index e03e54ed03..e28b30794d 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -689,9 +689,7 @@ Append this to `process_final_updates(state)`: ) for index, validator in enumerate(state.validator_registry): if index not in validator_indices_in_records: - if validator.exit_epoch == FAR_FUTURE_EPOCH: - validator.withdrawable_epoch = FAR_FUTURE_EPOCH - else: + if validator.exit_epoch != FAR_FUTURE_EPOCH and validator.withdrawable_epoch == FAR_FUTURE_EPOCH: validator.withdrawable_epoch = validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY ``` From 0e2d9e49636e531cd59c182bd93b158d24764e1c Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 18 May 2019 10:42:04 +0200 Subject: [PATCH 137/308] Function_puller returns objects --- scripts/build_spec.py | 219 ++++++++++++++++++++++--------------- scripts/function_puller.py | 71 +++++------- 2 files changed, 157 insertions(+), 133 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index a18107528a..cdcbc2456a 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -5,77 +5,7 @@ from typing import Tuple, List -def split_and_label(regex_pattern: str, text: str) -> List[str]: - ''' - Splits a string based on regex, but down not remove the matched text. - It subsequently labels the matches with their match - ''' - find_pattern = r'''%s.*?(?=%s|$)''' % (regex_pattern, regex_pattern) - matches = re.findall(find_pattern, text, re.DOTALL) - return list(map(lambda x: [re.match(regex_pattern, x).group(0), x], matches)) - - -def inserter(oldfile: str, newfile: str) -> Tuple[str, str]: - ''' - Searches for occurrences of @LabelName in oldfile and replaces them with instances of code wraped as follows: - # begin insert @LabelName - def foo(bar): - return bar - # end insert @LabelName - ''' - new_insert_objects = re.split(r"(# begin insert |# end insert @[\w\d_]*\n)", newfile) - # Retrieve label from insert objects - def get_labeled_object(labeled_text): - label = re.match(r"@[\w\d_]*\n", labeled_text) - if label is not None: - label = label.group(0) - labeled_text = re.sub(label, '', labeled_text) - return {'label': label, 'text': labeled_text} - new_insert_objects = map(get_labeled_object, new_insert_objects) - # Find and replace labels - newfile = "" - for item in new_insert_objects: - if item['label'] is not None: - oldfile, insertions = re.subn('# %s' % item['label'], item['text'], oldfile) - if insertions == 0: - newfile.join('# begin insert %s/n%s# end insert %s' % (item['label'], item['text'], item['label'])) - elif re.match(r"(# begin insert |# end insert )", item['text']) is None: - newfile += item['text'] - return oldfile, newfile - - -def merger(oldfile:str, newfile:str) -> str: - ''' - Seeks out functions and objects in new and old files. - Replaces old objects with new ones if they exist. - ''' - object_regex = r'''(?:\n[@\w]+[\s\w]*[='" "\.\w]*)|(?:\s{4}global_vars\["\w+"\])''' - ssz_object_regex = r'''(?:\w+|\s{4}global_vars\["\w+"\]) = SSZType\(\{\n''' - old_objects = split_and_label(object_regex, oldfile) - new_objects = split_and_label(object_regex, newfile) - for new_item in new_objects: - found_old = False - for index, old_item in enumerate(old_objects): - if old_item[0] == new_item[0]: - ssz_object_match = re.match(ssz_object_regex, new_item[1]) - if ssz_object_match is not None: - new_item[1] = re.sub(ssz_object_regex, '', new_item[1]) - old_item[1] = re.sub(r'\n\w*\}\)', '', old_item[1]) - old_item[1] += new_item[1] - else: - old_item[1] = new_item[1] - found_old = True - old_objects[index] = old_item - break - if not found_old: - old_objects.append(new_item) - return ''.join(elem for elem in map(lambda x: x[1], old_objects)) - - -def build_phase0_spec(sourcefile, outfile=None): - code_lines = [] - code_lines.append(""" -from typing import ( +IMPORTS = '''from typing import ( Any, Dict, List, @@ -96,11 +26,8 @@ def build_phase0_spec(sourcefile, outfile=None): ) from eth2spec.utils.hash_function import hash - - -# stub, will get overwritten by real var -SLOTS_PER_EPOCH = 64 - +''' +NEW_TYPE_DEFINITIONS = ''' Slot = NewType('Slot', int) # uint64 Epoch = NewType('Epoch', int) # uint64 Shard = NewType('Shard', int) # uint64 @@ -110,11 +37,8 @@ def build_phase0_spec(sourcefile, outfile=None): BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 BLSSignature = NewType('BLSSignature', bytes) # bytes96 Store = None -""") - - code_lines += function_puller.get_spec(sourcefile) - - code_lines.append(""" +''' +SUNDRY_FUNCTIONS = ''' # Monkey patch validator compute committee code _compute_committee = compute_committee committee_cache = {} @@ -156,27 +80,140 @@ def apply_constants_preset(preset: Dict[str, Any]): # Initialize SSZ types again, to account for changed lengths init_SSZ_types() +''' + +def split_and_label(regex_pattern: str, text: str) -> List[str]: + ''' + Splits a string based on regex, but down not remove the matched text. + It subsequently labels the matches with their match + ''' + find_pattern = r'''%s.*?(?=%s|$)''' % (regex_pattern, regex_pattern) + matches = re.findall(find_pattern, text, re.DOTALL) + return list(map(lambda x: [re.match(regex_pattern, x).group(0), x], matches)) -""") +def inserter(oldfile: str, newfile: str) -> Tuple[str, str]: + ''' + Searches for occurrences of @LabelName in oldfile and replaces them with instances of code wraped as follows: + # begin insert @LabelName + def foo(bar): + return bar + # end insert @LabelName + ''' + new_insert_objects = re.split(r"(# begin insert |# end insert @[\w\d_]*\n)", newfile) + # Retrieve label from insert objects + def get_labeled_object(labeled_text): + label = re.match(r"@[\w\d_]*\n", labeled_text) + if label is not None: + label = label.group(0) + labeled_text = re.sub(label, '', labeled_text) + return {'label': label, 'text': labeled_text} + new_insert_objects = map(get_labeled_object, new_insert_objects) + # Find and replace labels + newfile = "" + for item in new_insert_objects: + if item['label'] is not None: + oldfile, insertions = re.subn('# %s' % item['label'], item['text'], oldfile) + if insertions == 0: + newfile.join('# begin insert %s/n%s# end insert %s' % (item['label'], item['text'], item['label'])) + elif re.match(r"(# begin insert |# end insert )", item['text']) is None: + newfile += item['text'] + return oldfile, newfile + + +def merger(oldfile:str, newfile:str) -> str: + ''' + Seeks out functions and objects in new and old files. + Replaces old objects with new ones if they exist. + ''' + object_regex = r'''(?:\n[@\w]+[\s\w]*[='" "\.\w]*)|(?:\s{4}global_vars\["\w+"\])''' + ssz_object_regex = r'''(?:\w+|\s{4}global_vars\["\w+"\]) = SSZType\(\{\n''' + old_objects = split_and_label(object_regex, oldfile) + new_objects = split_and_label(object_regex, newfile) + for new_item in new_objects: + found_old = False + for index, old_item in enumerate(old_objects): + if old_item[0] == new_item[0]: + ssz_object_match = re.match(ssz_object_regex, new_item[1]) + if ssz_object_match is not None: + new_item[1] = re.sub(ssz_object_regex, '', new_item[1]) + old_item[1] = re.sub(r'\n\w*\}\)', '', old_item[1]) + old_item[1] += new_item[1] + else: + old_item[1] = new_item[1] + found_old = True + old_objects[index] = old_item + break + if not found_old: + old_objects.append(new_item) + return ''.join(elem for elem in map(lambda x: x[1], old_objects)) + + +def objects_to_spec(functions, constants, ssz_objects): + functions_spec = '\n\n'.join(functions.values()) + constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]),constants)) + ssz_objects_instantiation_spec = '\n'.join(map(lambda x: '%s = SSZType(%s)' % (x, ssz_objects[x][:-1]), ssz_objects)) + ssz_objects_reinitialization_spec = '\n'.join( + map(lambda x: ' global_vars[%s] = SSZType(%s })' % (x, re.sub('( ){4}', ' '*8, ssz_objects[x][:-2])), ssz_objects)) + ssz_objects_reinitialization_spec = ( + 'def init_SSZ_types():\n global_vars = globals()\n' + + ssz_objects_reinitialization_spec + ) + return ( + IMPORTS + + '\n' + NEW_TYPE_DEFINITIONS + + '\n' + constants_spec + + '\n' + ssz_objects_instantiation_spec + + '\n\n\n' + functions_spec + + '\n' + SUNDRY_FUNCTIONS + + '\n\n' + ssz_objects_reinitialization_spec + + '\n' + ) + +def combine_functions(old_funcitons, new_functions): + for key, value in new_functions.items(): + old_funcitons[key] = value + # TODO: Add insert functionality + return old_funcitons + + +def combine_constants(old_constants, new_constants): + for key, value in new_constants.items(): + old_constants[key] = value + return old_constants + +def combine_ssz_objects(old_objects, new_objects): + remove_encasing = lambda x: x[1:-1] + old_objects = map(remove_encasing, old_objects) + new_objects = map(remove_encasing, new_objects) + for key, value in new_objects.items(): + old_objects[key] += value + reapply_encasing = lambda x: '{%s}' %x + return map(reapply_encasing, old_objects) + + +def build_phase0_spec(sourcefile, outfile=None): + functions, constants, ssz_objects = function_puller.get_spec(sourcefile) + spec = objects_to_spec(functions, constants, ssz_objects) if outfile is not None: with open(outfile, 'w') as out: - out.write("\n".join(code_lines)) + out.write(spec) else: - return "\n".join(code_lines) + return spec def build_phase1_spec(phase0_sourcefile, phase1_sourcefile, outfile=None): - phase0_code = build_phase0_spec(phase0_sourcefile) - phase1_code = build_phase0_spec(phase1_sourcefile) - phase0_code, phase1_code = inserter(phase0_code, phase1_code) - phase1_code = merger(phase0_code, phase1_code) - + phase0_functions, phase0_constants, phase0_ssz_objects = function_puller.get_spec(phase0_sourcefile) + phase1_functions, phase1_constants, phase1_ssz_objects = function_puller.get_spec(phase1_sourcefile) + functions = combine_functions(phase0_functions, phase1_functions) + constants = combine_constants(phase0_constants, phase1_constants) + ssz_objects = combine_functions(phase0_ssz_objects, phase1_ssz_objects) + spec = objects_to_spec(functions, constants, ssz_objects) if outfile is not None: with open(outfile, 'w') as out: - out.write(phase1_code) + out.write(spec) else: - return phase1_code + return spec if __name__ == '__main__': diff --git a/scripts/function_puller.py b/scripts/function_puller.py index 97bc628217..720f075023 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -1,13 +1,21 @@ import sys +import re from typing import List +from collections import defaultdict -def get_spec(file_name: str) -> List[str]: +FUNCTION_REGEX = r'^def [\w_]*' + + +def get_spec(file_name: str): code_lines = [] - pulling_from = None - current_name = None - current_typedef = None - type_defs = [] + pulling_from = None # line number of start of latest object + current_name = None # most recent section title + functions = defaultdict(str) + constants = {} + ssz_objects = defaultdict(str) + function_matcher = re.compile(FUNCTION_REGEX) + # type_defs = [] for linenum, line in enumerate(open(file_name).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': @@ -16,29 +24,21 @@ def get_spec(file_name: str) -> List[str]: assert pulling_from is None pulling_from = linenum + 1 elif line[:3] == '```': - if pulling_from is None: - pulling_from = linenum - else: - if current_typedef is not None: - assert code_lines[-1] == '}' - code_lines[-1] = '})' - current_typedef[-1] = '})' - type_defs.append((current_name, current_typedef)) - pulling_from = None - current_typedef = None + pulling_from = None else: - if pulling_from == linenum and line == '{': - code_lines.append('%s = SSZType({' % current_name) - current_typedef = ['global_vars["%s"] = SSZType({' % current_name] - elif pulling_from is not None: - # Add some whitespace between functions - if line[:3] == 'def': - code_lines.append('') - code_lines.append('') - code_lines.append(line) - # Remember type def lines - if current_typedef is not None: - current_typedef.append(line) + # # Handle SSZObjects + # if pulling_from == linenum and line == '{': + # code_lines.append('%s = SSZType({' % current_name) + # Handle function definitions + if pulling_from is not None: + match = function_matcher.match(line) + if match is not None: + current_name = match.group(0) + if function_matcher.match(current_name) is None: + ssz_objects[current_name] += line + '\n' + else: + functions[current_name] += line + '\n' + # Handle constant table entries elif pulling_from is None and len(line) > 0 and line[0] == '|': row = line[1:].split('|') if len(row) >= 2: @@ -53,18 +53,5 @@ def get_spec(file_name: str) -> List[str]: if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': eligible = False if eligible: - code_lines.append(row[0] + ' = ' + (row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890'))) - # Build type-def re-initialization - code_lines.append('\n') - code_lines.append('def init_SSZ_types():') - code_lines.append(' global_vars = globals()') - for ssz_type_name, ssz_type in type_defs: - code_lines.append('') - for type_line in ssz_type: - if len(type_line) > 0: - code_lines.append(' ' + type_line) - code_lines.append('\n') - code_lines.append('def get_ssz_type_by_name(name: str) -> SSZType:') - code_lines.append(' return globals()[name]') - code_lines.append('') - return code_lines + constants[row[0]] = row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890') + return functions, constants, ssz_objects From 85c16544566f82be57c5905ccf3b7b48e2888d1d Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 19 May 2019 09:33:01 -0400 Subject: [PATCH 138/308] Crosslinks store start and end epoch Solves #1034 --- specs/core/0_beacon-chain.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b75f14153a..643985e85c 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -281,8 +281,9 @@ The types are defined topologically to aid in facilitating an executable version { # Shard number 'shard': 'uint64', - # Epoch number - 'epoch': 'uint64', + # Crosslinking data from epochs [start....end-1] + 'start_epoch': 'uint64', + 'end_epoch': 'uint64', # Root of the previous crosslink 'parent_root': 'bytes32', # Root of the crosslinked shard data since the previous crosslink @@ -1728,7 +1729,8 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Check FFG data, crosslink data, and signature assert ffg_data == (data.source_epoch, data.source_root, data.target_epoch) - assert data.crosslink.epoch == min(data.target_epoch, parent_crosslink.epoch + MAX_EPOCHS_PER_CROSSLINK) + assert data.crosslink.start_epoch == parent_crosslink.end_epoch + assert data.crosslink.end_epoch == min(data.target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK) assert data.crosslink.parent_root == hash_tree_root(parent_crosslink) assert data.crosslink.data_root == ZERO_HASH # [to be removed in phase 1] validate_indexed_attestation(state, convert_to_indexed(state, attestation)) From a2108741e804046b1ba721676482091a27f0e0fb Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 19 May 2019 15:47:59 -0600 Subject: [PATCH 139/308] fix tests with new starT_epoch and end_epoch in Crosslink --- test_libs/pyspec/eth2spec/utils/bls_stub.py | 2 +- .../tests/block_processing/test_process_attestation.py | 2 +- test_libs/pyspec/tests/helpers.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/bls_stub.py b/test_libs/pyspec/eth2spec/utils/bls_stub.py index 108c4ef710..ae97de175b 100644 --- a/test_libs/pyspec/eth2spec/utils/bls_stub.py +++ b/test_libs/pyspec/eth2spec/utils/bls_stub.py @@ -9,4 +9,4 @@ def bls_verify_multiple(pubkeys, message_hashes, signature, domain): def bls_aggregate_pubkeys(pubkeys): - return b'\x42' * 96 + return b'\x42' * 48 diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index 708d68dca8..6851561e94 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -124,7 +124,7 @@ def test_bad_previous_crosslink(state): for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): next_slot(state) - state.current_crosslinks[attestation.data.crosslink.shard].epoch += 10 + state.current_crosslinks[attestation.data.crosslink.shard].end_epoch += 10 pre_state, post_state = run_attestation_processing(state, attestation, False) diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 5ddb2dc154..7af210f85a 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -177,6 +177,7 @@ def build_attestation_data(state, slot, shard): justified_block_root = state.current_justified_root crosslinks = state.current_crosslinks if slot_to_epoch(slot) == get_current_epoch(state) else state.previous_crosslinks + parent_crosslink = crosslinks[shard] return AttestationData( beacon_block_root=block_root, source_epoch=justified_epoch, @@ -185,9 +186,10 @@ def build_attestation_data(state, slot, shard): target_root=epoch_boundary_root, crosslink=Crosslink( shard=shard, - epoch=min(slot_to_epoch(slot), crosslinks[shard].epoch + MAX_EPOCHS_PER_CROSSLINK), + start_epoch=parent_crosslink.end_epoch, + end_epoch=min(slot_to_epoch(slot), parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK), data_root=spec.ZERO_HASH, - parent_root=hash_tree_root(crosslinks[shard]), + parent_root=hash_tree_root(parent_crosslink), ), ) From 62f8d19ffc2f80763e7dfef5a66a19782080d5c4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 19 May 2019 16:06:10 -0600 Subject: [PATCH 140/308] add some attestation crosslink tests --- .../test_process_attestation.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index 6851561e94..97eddb902d 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -64,6 +64,22 @@ def test_success_prevous_epoch(state): return pre_state, attestation, post_state +def test_success_since_max_epochs_per_crosslink(state): + for _ in range(spec.MAX_EPOCHS_PER_CROSSLINK + 2): + next_epoch(state) + + attestation = get_valid_attestation(state) + data = attestation.data + assert data.crosslink.end_epoch - data.crosslink.start_epoch == spec.MAX_EPOCHS_PER_CROSSLINK + + for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): + next_slot(state) + + pre_state, post_state = run_attestation_processing(state, attestation) + + return pre_state, attestation, post_state + + def test_before_inclusion_delay(state): attestation = get_valid_attestation(state) # do not increment slot to allow for inclusion delay @@ -131,6 +147,32 @@ def test_bad_previous_crosslink(state): return pre_state, attestation, post_state +def test_bad_crosslink_start_epoch(state): + next_epoch(state) + attestation = get_valid_attestation(state) + for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): + next_slot(state) + + attestation.data.crosslink.start_epoch += 1 + + pre_state, post_state = run_attestation_processing(state, attestation, False) + + return pre_state, attestation, post_state + + +def test_bad_crosslink_end_epoch(state): + next_epoch(state) + attestation = get_valid_attestation(state) + for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): + next_slot(state) + + attestation.data.crosslink.end_epoch += 1 + + pre_state, post_state = run_attestation_processing(state, attestation, False) + + return pre_state, attestation, post_state + + def test_non_empty_custody_bitfield(state): attestation = get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY From 4c5e0548833bd3b4d92c22806afce48a1b4c1ab3 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sun, 19 May 2019 16:11:39 -0600 Subject: [PATCH 141/308] fix previous crosslink root test --- .../pyspec/tests/block_processing/test_process_attestation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/tests/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/block_processing/test_process_attestation.py index 97eddb902d..763178717a 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/block_processing/test_process_attestation.py @@ -140,7 +140,7 @@ def test_bad_previous_crosslink(state): for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): next_slot(state) - state.current_crosslinks[attestation.data.crosslink.shard].end_epoch += 10 + attestation.data.crosslink.parent_root = b'\x27' * 32 pre_state, post_state = run_attestation_processing(state, attestation, False) From c14452bcf445761926da99a1259f0a792268bdff Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sun, 19 May 2019 19:44:12 -0400 Subject: [PATCH 142/308] Updated get_custody_chunk_count Co-requisite with #1097 --- specs/core/1_custody-game.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 0e8fcf6e9a..7965e25f96 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -255,8 +255,8 @@ The `empty` function accepts and SSZ type as input and returns an object of that ```python def get_custody_chunk_count(attestation: Attestation) -> int: - crosslink_start_epoch = attestation.data.latest_crosslink.epoch - crosslink_end_epoch = slot_to_epoch(attestation.data.slot) + crosslink_start_epoch = attestation.data.latest_crosslink.start_epoch + crosslink_end_epoch = attestation.data.latest_crosslink.end_epoch crosslink_crosslink_length = min(MAX_EPOCHS_PER_CROSSLINK, end_epoch - start_epoch) chunks_per_epoch = 2 * BYTES_PER_SHARD_BLOCK * SLOTS_PER_EPOCH // BYTES_PER_CUSTODY_CHUNK return crosslink_crosslink_length * chunks_per_epoch From a68aa82b894153eca18f290d306623bf4118357b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 20 May 2019 11:39:13 +0800 Subject: [PATCH 143/308] Update validator guide --- specs/validator/0_beacon-chain-validator.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 0940d592a6..d05f25ef29 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -306,10 +306,12 @@ Set `attestation_data.beacon_block_root = signing_root(head_block)`. ##### Crosslink vote -Construct `attestation_data.crosslink` via the following +Construct `attestation_data.crosslink` via the following. * Set `attestation_data.crosslink.shard = shard` where `shard` is the shard associated with the validator's committee. -* Set `attestation_data.crosslink.epoch = min(attestation_data.target_epoch, head_state.current_crosslinks[shard].epoch + MAX_EPOCHS_PER_CROSSLINK)`. +* Let `parent_crosslink = head_state.current_crosslinks[shard]`. +* Set `attestation_data.crosslink.start_epoch = parent_crosslink.end_epoch`. +* Set `attestation_data.crosslink.end_epoch = min(attestation_data.target_epoch, parent_crosslink.end_epoch + MAX_EPOCHS_PER_CROSSLINK)`. * Set `attestation_data.crosslink.parent_root = hash_tree_root(head_state.current_crosslinks[shard])`. * Set `attestation_data.crosslink.data_root = ZERO_HASH`. *Note*: This is a stub for Phase 0. From 73c9d126dee0a4dafab4bbf00197ea56a2a3da94 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 20 May 2019 13:49:06 +1000 Subject: [PATCH 144/308] Updated API spec with suggestions by @hwwhww. - Corrected to use American English instead of Australian - Fixed spelling mistake with indices - Changed tag to 'MinimalSet' and 'OptionalSet' - Added a response to the list of components - Renamed 'block_production' to 'block_proposal' --- .../validator/0_beacon-node-validator-api.md | 2 +- specs/validator/beacon_node_oapi.yaml | 38 ++++++++++--------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/specs/validator/0_beacon-node-validator-api.md b/specs/validator/0_beacon-node-validator-api.md index 5c597aa75c..ec0300e9a9 100644 --- a/specs/validator/0_beacon-node-validator-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -11,7 +11,7 @@ The API is a REST interface, accessed via HTTP, designed for use as a local comm ### Background The beacon node maintains the state of the beacon chain by communicating with other beacon nodes in the Ethereum Serenity network. Conceptually, it does not maintain keypairs that participate with the beacon chain. -The validator client is a conceptually separate entity which utilises private keys to perform validator related tasks on the beacon chain, which we call validator "duties". These duties includes the production of beacon blocks and signing of attestations. +The validator client is a conceptually separate entity which utilizes private keys to perform validator related tasks on the beacon chain, which we call validator "duties". These duties include the production of beacon blocks and signing of attestations. Since it is recommended to separate these concerns in the client implementations, we must clearly define the communication between them. diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 0c2937f3df..e0e3da7d25 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -7,15 +7,15 @@ info: name: "Apache 2.0" url: "https://www.apache.org/licenses/LICENSE-2.0.html" tags: - - name: Necessary for validator + - name: MinimalSet description: The minimal set of endpoints to enable a working validator implementation. - - name: Optional + - name: OptionalSet description: Extra endpoints which are nice-to-haves. paths: /node/version: get: tags: - - Necessary for validator + - MinimalSet summary: "Get version string of the running beacon node." description: "Requests that the beacon node identify information about its implementation in a format similar to a [HTTP User-Agent](https://tools.ietf.org/html/rfc7231#section-5.5.3) field." responses: @@ -30,7 +30,7 @@ paths: /node/genesis_time: get: tags: - - Necessary for validator + - MinimalSet summary: "Get the genesis_time parameter from beacon node configuration." description: "Requests the genesis_time parameter from the beacon node, which should be consistent across all beacon nodes that follow the same beacon chain." responses: @@ -46,7 +46,7 @@ paths: /node/syncing: get: tags: - - Necessary for validator + - MinimalSet summary: "Poll to see if the the beacon node is syncing." description: "Requests the beacon node to describe if it's currently syncing or not, and if it is, what block it is up to. This is modelled after the Eth1.0 JSON-RPC eth_syncing call.." responses: @@ -67,7 +67,7 @@ paths: /node/fork: get: tags: - - Optional + - OptionalSet summary: "Get fork information from running beacon node." description: "Requests the beacon node to provide which fork version it is currently on." responses: @@ -83,16 +83,16 @@ paths: chain_id: type: integer format: uint64 - description: "Sometimes called the network id, this number discerns the active chain for the BeaconNode. Analagous to Eth1.0 JSON-RPC net_version." + description: "Sometimes called the network id, this number discerns the active chain for the beacon node. Analagous to Eth1.0 JSON-RPC net_version." 500: $ref: '#/components/responses/InternalError' /validator/duties: get: tags: - - Necessary for validator + - MinimalSet summary: "Get validator duties for the requested validators." - description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected beacon node is properly synchronised." + description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected beacon node is properly synchronized." parameters: - name: validator_pubkeys in: query @@ -122,7 +122,7 @@ paths: /validator/block: get: tags: - - Necessary for validator + - MinimalSet summary: "Produce a new block, without signature." description: "Requests a beacon node to produce a valid block, which can then be signed by a validator." parameters: @@ -155,7 +155,7 @@ paths: $ref: '#/components/responses/CurrentlySyncing' post: tags: - - Necessary for validator + - MinimalSet summary: "Publish a signed block." description: "Instructs the beacon node to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." parameters: @@ -178,7 +178,7 @@ paths: /validator/attestation: get: tags: - - Necessary for validator + - MinimalSet summary: "Produce an attestation, without signature." description: "Requests that the beacon node produce an IndexedAttestation, with a blank signature field, which the validator will then sign." parameters: @@ -209,7 +209,7 @@ paths: $ref: '#/components/responses/CurrentlySyncing' post: tags: - - Necessary for validator + - MinimalSet summary: "Published a signed attestation." description: "Instructs the beacon node to publish a newly signed IndexedAttestation object, to be incorporated into the beacon chain." parameters: @@ -263,7 +263,7 @@ components: type: integer format: uint64 description: "The shard in which the validator must attest." - block_production_slot: + block_proposal_slot: type: integer format: uint64 nullable: true @@ -537,15 +537,15 @@ components: type: object description: "The [`IndexedAttestation`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#indexedattestation) object from the Eth2.0 spec." properties: - custody_bit_0_indicies: + custody_bit_0_indices: type: array - description: "Validator indicies for 0 bits." + description: "Validator indices for 0 bits." items: type: integer format: uint64 - custody_bit_1_indicies: + custody_bit_1_indices: type: array - description: "Validator indicies for 1 bits." + description: "Validator indices for 1 bits." items: type: integer format: uint64 @@ -611,3 +611,5 @@ components: description: "Beacon node internal error." CurrentlySyncing: description: "Beacon node is currently syncing, try again later." + NotFound: + description: "The requested API endpoint does not exist." From 490cf9e347c5229127af94e0bfa431d3a0f38b16 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 20 May 2019 13:56:25 +1000 Subject: [PATCH 145/308] Updated version number to reflect small changes. --- specs/validator/beacon_node_oapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index e0e3da7d25..e995e9c84e 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -2,7 +2,7 @@ openapi: "3.0.2" info: title: "Minimal Beacon Node API for Validator" description: "A minimal API specification for the beacon node, which enables a validator to connect and perform its obligations on the Ethereum 2.0 phase 0 beacon chain." - version: "0.1" + version: "0.1.1" license: name: "Apache 2.0" url: "https://www.apache.org/licenses/LICENSE-2.0.html" From 8e67dec7e40a241ace49efd1e02114c2830ddb7a Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 20 May 2019 16:43:21 +1000 Subject: [PATCH 146/308] Fixed misinterpretation of the proof array in the Deposit object, bumped version. --- specs/validator/beacon_node_oapi.yaml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index e995e9c84e..900b8eefa8 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -2,7 +2,7 @@ openapi: "3.0.2" info: title: "Minimal Beacon Node API for Validator" description: "A minimal API specification for the beacon node, which enables a validator to connect and perform its obligations on the Ethereum 2.0 phase 0 beacon chain." - version: "0.1.1" + version: "0.1.2" license: name: "Apache 2.0" url: "https://www.apache.org/licenses/LICENSE-2.0.html" @@ -421,17 +421,11 @@ components: type: array description: "Branch in the deposit tree." items: - oneOf: - - type: string - format: byte - pattern: "^0x[a-fA-F0-9]{64}$" - - type: integer - format: uint64 - enum: [32] - example: 32 - description: "The DEPOSIT_CONTRACT_TREE_DEPTH value." - minItems: 2 - maxItems: 2 + type: string + format: byte + pattern: "^0x[a-fA-F0-9]{64}$" + minItems: 32 + maxItems: 32 index: type: integer format: uint64 From 725dcf27f584fed5b2df86d66c05c02cb952a247 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 20 May 2019 09:47:44 +0200 Subject: [PATCH 147/308] Hella bug-fixes and PEP8 --- specs/core/1_custody-game.md | 68 +++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index ba828dd2c9..df4f495bec 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -273,7 +273,7 @@ The `empty` function accepts and SSZ type as input and returns an object of that def get_custody_chunk_count(attestation: Attestation) -> int: crosslink_start_epoch = attestation.data.latest_crosslink.epoch crosslink_end_epoch = slot_to_epoch(attestation.data.slot) - crosslink_crosslink_length = min(MAX_EPOCHS_PER_CROSSLINK, end_epoch - start_epoch) + crosslink_crosslink_length = min(MAX_EPOCHS_PER_CROSSLINK, crosslink_end_epoch - crosslink_start_epoch) chunks_per_epoch = 2 * BYTES_PER_SHARD_BLOCK * SLOTS_PER_EPOCH // BYTES_PER_CUSTODY_CHUNK return crosslink_crosslink_length * chunks_per_epoch ``` @@ -283,7 +283,7 @@ def get_custody_chunk_count(attestation: Attestation) -> int: ```python def get_custody_chunk_bit(key: BLSSignature, chunk: bytes) -> bool: # TODO: Replace with something MPC-friendly, e.g. the Legendre symbol - return get_bitfield_bit(hash(challenge.responder_key + chunk), 0) + return get_bitfield_bit(hash(key + chunk), 0) ``` ### `get_chunk_bits_root` @@ -348,8 +348,7 @@ For each `reveal` in `block.body.custody_key_reveals`, run the following functio ```python def process_custody_key_reveal(state: BeaconState, - reveal: CustodyKeyReveal) -> None: - + reveal: CustodyKeyReveal) -> None: """ Process ``CustodyKeyReveal`` operation. Note that this function mutates ``state``. @@ -377,15 +376,18 @@ def process_custody_key_reveal(state: BeaconState, # Decrement max reveal lateness if response is timely if revealer.next_custody_reveal_period == get_validators_custody_reveal_period(state, reveal.revealer_index) - 2: - revealer.max_reveal_lateness -= MAX_REVEAL_LATENESS_DECREMENT - revealer.max_reveal_lateness = max(revealed_validator.max_reveal_lateness, get_validators_custody_reveal_period(state, reveal.revealed_index) - revealer.next_custody_reveal_period) + revealer.max_reveal_lateness -= MAX_REVEAL_LATENESS_DECREMENT + revealer.max_reveal_lateness = max( + revealer.max_reveal_lateness, + get_validators_custody_reveal_period(state, reveal.revealed_index) - revealer.next_custody_reveal_period + ) # Process reveal revealer.next_custody_reveal_period += 1 # Reward Block Preposer proposer_index = get_beacon_proposer_index(state) - increase_balance(state, proposer_index, base_reward(state, index) // MINOR_REWARD_QUOTIENT) + increase_balance(state, proposer_index, get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT) ``` ##### Early derived secret reveals @@ -396,7 +398,7 @@ For each `reveal` in `block.body.early_derived_secret_reveals`, run the followin ```python def process_early_derived_secret_reveal(state: BeaconState, - reveal: EarlyDerivedSecretReveal) -> None: + reveal: EarlyDerivedSecretReveal) -> None: """ Process ``EarlyDerivedSecretReveal`` operation. Note that this function mutates ``state``. @@ -404,11 +406,12 @@ def process_early_derived_secret_reveal(state: BeaconState, revealed_validator = state.validator_registry[reveal.revealed_index] masker = state.validator_registry[reveal.masker_index] + derived_secret_location = reveal.epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS assert reveal.epoch >= get_current_epoch(state) + RANDAO_PENALTY_EPOCHS assert reveal.epoch < get_current_epoch(state) + EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS assert revealed_validator.slashed is False - assert reveal.revealed_index not in state.exposed_derived_secrets[reveal.epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] + assert reveal.revealed_index not in state.exposed_derived_secrets[derived_secret_location] # Verify signature correctness masker = state.validator_registry[reveal.masker_index] @@ -440,12 +443,16 @@ def process_early_derived_secret_reveal(state: BeaconState, # Calculate penalty max_proposer_slot_reward = ( - get_base_reward(state, reveal.revealed_index) * - SLOTS_PER_EPOCH // - len(get_active_validator_indices(state, get_current_epoch(state))) // - PROPOSER_REWARD_QUOTIENT + get_base_reward(state, reveal.revealed_index) + * SLOTS_PER_EPOCH + // len(get_active_validator_indices(state, get_current_epoch(state))) + // PROPOSER_REWARD_QUOTIENT + ) + penalty = ( + max_proposer_slot_reward + * EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE + * (len(state.exposed_derived_secrets[derived_secret_location]) + 1) ) - penalty = max_proposer_slot_reward * EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE * (len(state.exposed_derived_secrets[reveal.epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS]) + 1) # Apply penalty proposer_index = get_beacon_proposer_index(state) @@ -457,8 +464,7 @@ def process_early_derived_secret_reveal(state: BeaconState, decrease_balance(state, reveal.revealed_index, penalty) # Mark this derived secret as exposed so validator cannot be punished repeatedly - state.exposed_derived_secrets[reveal.epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS].append(reveal.revealed_index) - + state.exposed_derived_secrets[derived_secret_location].append(reveal.revealed_index) ``` #### Chunk challenges @@ -471,13 +477,13 @@ For each `challenge` in `block.body.custody_chunk_challenges`, run the following def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge) -> None: # Verify the attestation - assert verify_indexed_attestation(state, convert_to_indexed(state, challenge.attestation)) + assert validate_indexed_attestation(state, convert_to_indexed(state, challenge.attestation)) # Verify it is not too late to challenge assert slot_to_epoch(challenge.attestation.data.slot) >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY responder = state.validator_registry[challenge.responder_index] assert responder.exit_epoch >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY # Verify the responder participated in the attestation - attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) + attesters = get_attesting_indices(state, challenge.attestation.data, challenge.attestation.aggregation_bitfield) assert challenge.responder_index in attesters # Verify the challenge is not a duplicate for record in state.custody_chunk_challenge_records: @@ -526,14 +532,14 @@ def process_bit_challenge(state: BeaconState, assert is_slashable_validator(challenger, get_current_epoch(state)) # Verify the attestation - assert verify_indexed_attestation(state, convert_to_indexed(state, challenge.attestation)) + assert validate_indexed_attestation(state, convert_to_indexed(state, challenge.attestation)) # Verify the attestation is eligible for challenging responder = state.validator_registry[challenge.responder_index] assert (slot_to_epoch(challenge.attestation.data.slot) + responder.max_reveal_lateness <= get_validators_custody_reveal_period(state, challenge.responder_index)) # Verify the responder participated in the attestation - attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) + attesters = get_attesting_indices(state, challenge.attestation.data, challenge.attestation.aggregation_bitfield) assert challenge.responder_index in attesters # A validator can be the challenger or responder for at most one challenge at a time @@ -546,7 +552,7 @@ def process_bit_challenge(state: BeaconState, get_validators_custody_reveal_period( state=state, index=challenge.responder_index, - epoch=slot_to_epoch(attestation.data.slot)), + epoch=slot_to_epoch(challenge.attestation.data.slot)), challenge.responder_index ) assert bls_verify( @@ -564,7 +570,7 @@ def process_bit_challenge(state: BeaconState, chunk_count = get_custody_chunk_count(challenge.attestation) assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit - custody_bit = get_bitfield_bit(attestation.custody_bitfield, attesters.index(responder_index)) + custody_bit = get_bitfield_bit(challenge.attestation.custody_bitfield, attesters.index(challenge.responder_index)) assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) # Add new bit challenge record new_record = CustodyBitChallengeRecord( @@ -593,11 +599,13 @@ For each `response` in `block.body.custody_responses`, run the following functio ```python def process_custody_response(state: BeaconState, response: CustodyResponse) -> None: - chunk_challenge = next((record for record in state.custody_chunk_challenge_records if record.challenge_index == response.challenge_index), None) + chunk_challenge = next((record for record in state.custody_chunk_challenge_records + if record.challenge_index == response.challenge_index), None) if chunk_challenge is not None: return process_chunk_challenge_response(state, response, chunk_challenge) - bit_challenge = next((record for record in state.custody_bit_challenge_records if record.challenge_index == response.challenge_index), None) + bit_challenge = next((record for record in state.custody_bit_challenge_records + if record.challenge_index == response.challenge_index), None) if bit_challenge is not None: return process_bit_challenge_response(state, response, bit_challenge) @@ -625,7 +633,7 @@ def process_chunk_challenge_response(state: BeaconState, records[records.index(challenge)] = CustodyChunkChallengeRecord() # Reward the proposer proposer_index = get_beacon_proposer_index(state) - increase_balance(state, proposer_index, base_reward(state, index) // MINOR_REWARD_QUOTIENT) + increase_balance(state, proposer_index, get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT) ``` ```python @@ -651,7 +659,8 @@ def process_bit_challenge_response(state: BeaconState, root=challenge.chunk_bits_merkle_root ) # Verify the chunk bit does not match the challenge chunk bit - assert get_custody_chunk_bit(challenge.responder_key, response.chunk) != get_bitfield_bit(challenge.chunk_bits_leaf, response.chunk_index % 256) + assert (get_custody_chunk_bit(challenge.responder_key, response.chunk) + != get_bitfield_bit(challenge.chunk_bits_leaf, response.chunk_index % 256)) # Clear the challenge records = state.custody_bit_challenge_records records[records.index(challenge)] = CustodyBitChallengeRecord() @@ -670,7 +679,7 @@ def process_reveal_deadlines(state: BeaconState) -> None: for index, validator in enumerate(state.validator_registry): if (validator.next_custody_reveal_period + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) < - get_validators_custody_reveal_period(state, index)): + get_validators_custody_reveal_period(state, index)): slash_validator(state, index) ``` @@ -712,12 +721,13 @@ def eligible(state: BeaconState, index: ValidatorIndex) -> bool: if len([record for record in state.custody_bit_challenge_records if record.responder_index == index]) > 0: return False # Cannot exit if you have not revealed all of your custody keys - elif validator.next_custody_reveal_period <= get_validators_custody_reveal_period(state, index, validator.exit_epoch): + elif (validator.next_custody_reveal_period + <= get_validators_custody_reveal_period(state, index, validator.exit_epoch)): return False # Cannot exit if you already have elif validator.withdrawable_epoch < FAR_FUTURE_EPOCH: return False # Return minimum time else: - return current_epoch >= validator.exit_epoch + MIN_VALIDATOR_WITHDRAWAL_EPOCHS + return get_current_epoch(state) >= validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY ``` From 2018dd83f5a324472623db15a159be2882ffe8fa Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 20 May 2019 09:29:09 +0100 Subject: [PATCH 148/308] Update 1_custody-game.md --- specs/core/1_custody-game.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 7965e25f96..f91fe81de0 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -254,12 +254,10 @@ The `empty` function accepts and SSZ type as input and returns an object of that ### `get_crosslink_chunk_count` ```python -def get_custody_chunk_count(attestation: Attestation) -> int: - crosslink_start_epoch = attestation.data.latest_crosslink.start_epoch - crosslink_end_epoch = attestation.data.latest_crosslink.end_epoch - crosslink_crosslink_length = min(MAX_EPOCHS_PER_CROSSLINK, end_epoch - start_epoch) +def get_custody_chunk_count(crosslink: Crosslink) -> int: + crosslink_length = min(MAX_EPOCHS_PER_CROSSLINK, crosslink.end_epoch - crosslink.start_epoch) chunks_per_epoch = 2 * BYTES_PER_SHARD_BLOCK * SLOTS_PER_EPOCH // BYTES_PER_CUSTODY_CHUNK - return crosslink_crosslink_length * chunks_per_epoch + return crosslink_length * chunks_per_epoch ``` ### `get_custody_chunk_bit` @@ -470,7 +468,7 @@ def process_chunk_challenge(state: BeaconState, record.chunk_index != challenge.chunk_index ) # Verify depth - depth = math.log2(next_power_of_two(get_custody_chunk_count(challenge.attestation))) + depth = math.log2(next_power_of_two(get_custody_chunk_count(challenge.attestation.data.crosslink))) assert challenge.chunk_index < 2**depth # Add new chunk challenge record new_record = CustodyChunkChallengeRecord( @@ -544,7 +542,7 @@ def process_bit_challenge(state: BeaconState, ) # Verify the chunk count - chunk_count = get_custody_chunk_count(challenge.attestation) + chunk_count = get_custody_chunk_count(challenge.attestation.data.crosslink) assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit custody_bit = get_bitfield_bit(attestation.custody_bitfield, attesters.index(responder_index)) From cfed4bb41ce30fb25b6ae2f36dc3439d4ec24256 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 20 May 2019 10:50:07 +0200 Subject: [PATCH 149/308] Adds missing functions --- scripts/build_spec.py | 103 ++++++------------ scripts/function_puller.py | 4 - specs/core/1_custody-game.md | 20 ++-- .../pyspec/eth2spec/utils/minimal_ssz.py | 10 ++ 4 files changed, 51 insertions(+), 86 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index cdcbc2456a..f244b91e1a 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -5,7 +5,7 @@ from typing import Tuple, List -IMPORTS = '''from typing import ( +PHASE0_IMPORTS = '''from typing import ( Any, Dict, List, @@ -27,6 +27,31 @@ from eth2spec.utils.hash_function import hash ''' +PHASE1_IMPORTS = '''from typing import ( + Any, + Dict, + List, + NewType, + Tuple, +) + +from eth2spec.utils.minimal_ssz import ( + SSZType, + hash_tree_root, + signing_root, + type_of, + empty, +) + +from eth2spec.utils.bls_stub import ( + bls_aggregate_pubkeys, + bls_verify, + bls_verify_multiple, +) + +from eth2spec.utils.hash_function import hash +import math +''' NEW_TYPE_DEFINITIONS = ''' Slot = NewType('Slot', int) # uint64 Epoch = NewType('Epoch', int) # uint64 @@ -82,74 +107,8 @@ def apply_constants_preset(preset: Dict[str, Any]): init_SSZ_types() ''' -def split_and_label(regex_pattern: str, text: str) -> List[str]: - ''' - Splits a string based on regex, but down not remove the matched text. - It subsequently labels the matches with their match - ''' - find_pattern = r'''%s.*?(?=%s|$)''' % (regex_pattern, regex_pattern) - matches = re.findall(find_pattern, text, re.DOTALL) - return list(map(lambda x: [re.match(regex_pattern, x).group(0), x], matches)) - - -def inserter(oldfile: str, newfile: str) -> Tuple[str, str]: - ''' - Searches for occurrences of @LabelName in oldfile and replaces them with instances of code wraped as follows: - # begin insert @LabelName - def foo(bar): - return bar - # end insert @LabelName - ''' - new_insert_objects = re.split(r"(# begin insert |# end insert @[\w\d_]*\n)", newfile) - # Retrieve label from insert objects - def get_labeled_object(labeled_text): - label = re.match(r"@[\w\d_]*\n", labeled_text) - if label is not None: - label = label.group(0) - labeled_text = re.sub(label, '', labeled_text) - return {'label': label, 'text': labeled_text} - new_insert_objects = map(get_labeled_object, new_insert_objects) - # Find and replace labels - newfile = "" - for item in new_insert_objects: - if item['label'] is not None: - oldfile, insertions = re.subn('# %s' % item['label'], item['text'], oldfile) - if insertions == 0: - newfile.join('# begin insert %s/n%s# end insert %s' % (item['label'], item['text'], item['label'])) - elif re.match(r"(# begin insert |# end insert )", item['text']) is None: - newfile += item['text'] - return oldfile, newfile - - -def merger(oldfile:str, newfile:str) -> str: - ''' - Seeks out functions and objects in new and old files. - Replaces old objects with new ones if they exist. - ''' - object_regex = r'''(?:\n[@\w]+[\s\w]*[='" "\.\w]*)|(?:\s{4}global_vars\["\w+"\])''' - ssz_object_regex = r'''(?:\w+|\s{4}global_vars\["\w+"\]) = SSZType\(\{\n''' - old_objects = split_and_label(object_regex, oldfile) - new_objects = split_and_label(object_regex, newfile) - for new_item in new_objects: - found_old = False - for index, old_item in enumerate(old_objects): - if old_item[0] == new_item[0]: - ssz_object_match = re.match(ssz_object_regex, new_item[1]) - if ssz_object_match is not None: - new_item[1] = re.sub(ssz_object_regex, '', new_item[1]) - old_item[1] = re.sub(r'\n\w*\}\)', '', old_item[1]) - old_item[1] += new_item[1] - else: - old_item[1] = new_item[1] - found_old = True - old_objects[index] = old_item - break - if not found_old: - old_objects.append(new_item) - return ''.join(elem for elem in map(lambda x: x[1], old_objects)) - - -def objects_to_spec(functions, constants, ssz_objects): + +def objects_to_spec(functions, constants, ssz_objects, imports): functions_spec = '\n\n'.join(functions.values()) constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]),constants)) ssz_objects_instantiation_spec = '\n'.join(map(lambda x: '%s = SSZType(%s)' % (x, ssz_objects[x][:-1]), ssz_objects)) @@ -160,7 +119,7 @@ def objects_to_spec(functions, constants, ssz_objects): + ssz_objects_reinitialization_spec ) return ( - IMPORTS + imports + '\n' + NEW_TYPE_DEFINITIONS + '\n' + constants_spec + '\n' + ssz_objects_instantiation_spec @@ -194,7 +153,7 @@ def combine_ssz_objects(old_objects, new_objects): def build_phase0_spec(sourcefile, outfile=None): functions, constants, ssz_objects = function_puller.get_spec(sourcefile) - spec = objects_to_spec(functions, constants, ssz_objects) + spec = objects_to_spec(functions, constants, ssz_objects, PHASE0_IMPORTS) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) @@ -208,7 +167,7 @@ def build_phase1_spec(phase0_sourcefile, phase1_sourcefile, outfile=None): functions = combine_functions(phase0_functions, phase1_functions) constants = combine_constants(phase0_constants, phase1_constants) ssz_objects = combine_functions(phase0_ssz_objects, phase1_ssz_objects) - spec = objects_to_spec(functions, constants, ssz_objects) + spec = objects_to_spec(functions, constants, ssz_objects, PHASE1_IMPORTS) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) diff --git a/scripts/function_puller.py b/scripts/function_puller.py index 720f075023..dc0c30998f 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -15,7 +15,6 @@ def get_spec(file_name: str): constants = {} ssz_objects = defaultdict(str) function_matcher = re.compile(FUNCTION_REGEX) - # type_defs = [] for linenum, line in enumerate(open(file_name).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': @@ -26,9 +25,6 @@ def get_spec(file_name: str): elif line[:3] == '```': pulling_from = None else: - # # Handle SSZObjects - # if pulling_from == linenum and line == '{': - # code_lines.append('%s = SSZType({' % current_name) # Handle function definitions if pulling_from is not None: match = function_matcher.match(line) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index df4f495bec..adc1211600 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -31,7 +31,7 @@ - [`BeaconState`](#beaconstate) - [`BeaconBlockBody`](#beaconblockbody) - [Helpers](#helpers) - - [`typeof`](#typeof) + - [`type_of`](#type_of) - [`empty`](#empty) - [`get_crosslink_chunk_count`](#get_crosslink_chunk_count) - [`get_custody_chunk_bit`](#get_custody_chunk_bit) @@ -259,13 +259,13 @@ Add the following fields to the end of the specified container objects. Fields w ## Helpers -### `typeof` +### `type_of` -The `typeof` function accepts and SSZ object as a single input and returns the corresponding SSZ type. +The `type_of` function accepts an SSZ object as a single input and returns the corresponding SSZ type. ### `empty` -The `empty` function accepts and SSZ type as input and returns an object of that type with all fields initialized to default values. +The `empty` function accepts an SSZ type as input and returns an object of that type with all fields initialized to default values. ### `get_crosslink_chunk_count` @@ -327,7 +327,7 @@ def get_validators_custody_reveal_period(state: BeaconState, ```python def replace_empty_or_append(list: List[Any], new_element: Any) -> int: for i in range(len(list)): - if list[i] == empty(typeof(new_element)): + if list[i] == empty(type_of(new_element)): list[i] = new_element return i list.append(new_element) @@ -492,7 +492,7 @@ def process_chunk_challenge(state: BeaconState, record.chunk_index != challenge.chunk_index ) # Verify depth - depth = math.log2(next_power_of_two(get_custody_chunk_count(challenge.attestation))) + depth = math.ceil(math.log2(get_custody_chunk_count(challenge.attestation))) assert challenge.chunk_index < 2**depth # Add new chunk challenge record new_record = CustodyChunkChallengeRecord( @@ -580,7 +580,7 @@ def process_bit_challenge(state: BeaconState, deadline=get_current_epoch(state) + CUSTODY_RESPONSE_DEADLINE, data_root=challenge.attestation.data.crosslink.data_root, chunk_count=chunk_count, - chunk_bits_merkle_root=merkle_root(pad_to_power_of_2((challenge.chunk_bits))), + chunk_bits_merkle_root=hash_tree_root(challenge.chunk_bits), responder_key=challenge.responder_key, ) replace_empty_or_append(state.custody_bit_challenge_records, new_record) @@ -646,7 +646,7 @@ def process_bit_challenge_response(state: BeaconState, assert verify_merkle_branch( leaf=hash_tree_root(response.chunk), branch=response.data_branch, - depth=math.log2(next_power_of_two(challenge.chunk_count)), + depth=math.ceil(math.log2(challenge.chunk_count)), index=response.chunk_index, root=challenge.data_root, ) @@ -654,13 +654,13 @@ def process_bit_challenge_response(state: BeaconState, assert verify_merkle_branch( leaf=response.chunk_bits_leaf, branch=response.chunk_bits_branch, - depth=math.log2(next_power_of_two(challenge.chunk_count) // 256), + depth=math.ceil(math.log2(challenge.chunk_count)) >> 8, index=response.chunk_index // 256, root=challenge.chunk_bits_merkle_root ) # Verify the chunk bit does not match the challenge chunk bit assert (get_custody_chunk_bit(challenge.responder_key, response.chunk) - != get_bitfield_bit(challenge.chunk_bits_leaf, response.chunk_index % 256)) + != get_bitfield_bit(challenge.chunk_bits_leaf, response.chunk_index % 256)) # Clear the challenge records = state.custody_bit_challenge_records records[records.index(challenge)] = CustodyBitChallengeRecord() diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py index f7784b6de6..63516780e7 100644 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py @@ -58,6 +58,16 @@ def __len__(self): return self.length +def type_of(obj): + return obj.__class__ + + +def empty(obj): + for field in obj.fields: + field = get_zero_value(field) + return obj + + def is_basic(typ): # if not a string, it is a complex, and cannot be basic if not isinstance(typ, str): From 83123a33da792397101bdee263d03a25ee281e0b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 20 May 2019 17:16:20 +0800 Subject: [PATCH 150/308] Set genesis_state.latest_block_header with `body_root` of empty BeaconBlockBody (#1098) --- specs/core/0_beacon-chain.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b75f14153a..b34bbe7d1b 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1189,7 +1189,11 @@ Let `genesis_state = get_genesis_beacon_state(genesis_deposits, eth2genesis.gene ```python def get_genesis_beacon_state(deposits: List[Deposit], genesis_time: int, genesis_eth1_data: Eth1Data) -> BeaconState: - state = BeaconState(genesis_time=genesis_time, latest_eth1_data=genesis_eth1_data) + state = BeaconState( + genesis_time=genesis_time, + latest_eth1_data=genesis_eth1_data, + latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), + ) # Process genesis deposits for deposit in deposits: From c954687f583ba5a1b037d5845232447bc8b9eb52 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 20 May 2019 11:17:24 +0200 Subject: [PATCH 151/308] Corrects combination of ssz objects --- scripts/build_spec.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index f244b91e1a..749dfaee4b 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -141,14 +141,16 @@ def combine_constants(old_constants, new_constants): old_constants[key] = value return old_constants + def combine_ssz_objects(old_objects, new_objects): - remove_encasing = lambda x: x[1:-1] - old_objects = map(remove_encasing, old_objects) - new_objects = map(remove_encasing, new_objects) for key, value in new_objects.items(): + # remove leading "{" and trailing "\n}" + old_objects[key] = old_objects[key][1:-3] + # remove leading "{" + value = value[1:] old_objects[key] += value - reapply_encasing = lambda x: '{%s}' %x - return map(reapply_encasing, old_objects) + old_objects[key] = '{' + old_objects[key] + return old_objects def build_phase0_spec(sourcefile, outfile=None): @@ -166,7 +168,7 @@ def build_phase1_spec(phase0_sourcefile, phase1_sourcefile, outfile=None): phase1_functions, phase1_constants, phase1_ssz_objects = function_puller.get_spec(phase1_sourcefile) functions = combine_functions(phase0_functions, phase1_functions) constants = combine_constants(phase0_constants, phase1_constants) - ssz_objects = combine_functions(phase0_ssz_objects, phase1_ssz_objects) + ssz_objects = combine_ssz_objects(phase0_ssz_objects, phase1_ssz_objects) spec = objects_to_spec(functions, constants, ssz_objects, PHASE1_IMPORTS) if outfile is not None: with open(outfile, 'w') as out: From fac9f6c91dbf37e79a84d69a21ff20a80a74ac1d Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 20 May 2019 14:00:54 +0200 Subject: [PATCH 152/308] Automatic SSZ dependancy Ordering! --- scripts/build_spec.py | 48 ++++++++++++++++++++++++-------------- scripts/function_puller.py | 9 ++++--- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 749dfaee4b..48ef0888c3 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -52,17 +52,17 @@ from eth2spec.utils.hash_function import hash import math ''' -NEW_TYPE_DEFINITIONS = ''' -Slot = NewType('Slot', int) # uint64 -Epoch = NewType('Epoch', int) # uint64 -Shard = NewType('Shard', int) # uint64 -ValidatorIndex = NewType('ValidatorIndex', int) # uint64 -Gwei = NewType('Gwei', int) # uint64 -Bytes32 = NewType('Bytes32', bytes) # bytes32 -BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 -BLSSignature = NewType('BLSSignature', bytes) # bytes96 -Store = None -''' +NEW_TYPES = { + 'Slot': 'int', + 'Epoch': 'int', + 'Shard': 'int', + 'ValidatorIndex': 'int', + 'Gwei': 'int', + 'Bytes32': 'bytes', + 'BLSPubkey': 'bytes', + 'BLSSignature': 'bytes', + 'Store': 'None', +} SUNDRY_FUNCTIONS = ''' # Monkey patch validator compute committee code _compute_committee = compute_committee @@ -109,6 +109,7 @@ def apply_constants_preset(preset: Dict[str, Any]): def objects_to_spec(functions, constants, ssz_objects, imports): + new_type_definitions = '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in NEW_TYPES.items()]) functions_spec = '\n\n'.join(functions.values()) constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]),constants)) ssz_objects_instantiation_spec = '\n'.join(map(lambda x: '%s = SSZType(%s)' % (x, ssz_objects[x][:-1]), ssz_objects)) @@ -120,9 +121,9 @@ def objects_to_spec(functions, constants, ssz_objects, imports): ) return ( imports - + '\n' + NEW_TYPE_DEFINITIONS - + '\n' + constants_spec - + '\n' + ssz_objects_instantiation_spec + + '\n' + new_type_definitions + + '\n\n' + constants_spec + + '\n\n' + ssz_objects_instantiation_spec + '\n\n\n' + functions_spec + '\n' + SUNDRY_FUNCTIONS + '\n\n' + ssz_objects_reinitialization_spec @@ -142,14 +143,27 @@ def combine_constants(old_constants, new_constants): return old_constants +def dependency_order_ssz_objects(objects): + items = list(objects.items()) + for key, value in items: + dependencies = re.findall(r'(: [\[]*[A-Z][a-z][\w]+)', value) + dependencies = map(lambda x: re.sub(r'\W', '', x), dependencies) + for dep in dependencies: + if dep in NEW_TYPES: + continue + key_list = list(objects.keys()) + for item in [dep, key] + key_list[key_list.index(dep)+1:]: + objects[item] = objects.pop(item) + + def combine_ssz_objects(old_objects, new_objects): for key, value in new_objects.items(): # remove leading "{" and trailing "\n}" - old_objects[key] = old_objects[key][1:-3] + old_objects[key] = old_objects.get(key, '')[1:-3] # remove leading "{" value = value[1:] - old_objects[key] += value - old_objects[key] = '{' + old_objects[key] + old_objects[key] = '{' + old_objects.get(key, '') + value + dependency_order_ssz_objects(old_objects) return old_objects diff --git a/scripts/function_puller.py b/scripts/function_puller.py index dc0c30998f..29ecccfbf5 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -1,7 +1,6 @@ import sys import re from typing import List -from collections import defaultdict FUNCTION_REGEX = r'^def [\w_]*' @@ -11,9 +10,9 @@ def get_spec(file_name: str): code_lines = [] pulling_from = None # line number of start of latest object current_name = None # most recent section title - functions = defaultdict(str) + functions = {} constants = {} - ssz_objects = defaultdict(str) + ssz_objects = {} function_matcher = re.compile(FUNCTION_REGEX) for linenum, line in enumerate(open(file_name).readlines()): line = line.rstrip() @@ -31,9 +30,9 @@ def get_spec(file_name: str): if match is not None: current_name = match.group(0) if function_matcher.match(current_name) is None: - ssz_objects[current_name] += line + '\n' + ssz_objects[current_name] = ssz_objects.get(current_name, '') + line + '\n' else: - functions[current_name] += line + '\n' + functions[current_name] = functions.get(current_name, '') + line + '\n' # Handle constant table entries elif pulling_from is None and len(line) > 0 and line[0] == '|': row = line[1:].split('|') From b68c471a54ee41ade662efaca1102f364a44c503 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 20 May 2019 17:08:03 +0200 Subject: [PATCH 153/308] Moves spec to global variable --- .../test_process_attestation.py | 42 +++++++++---------- .../test_process_attester_slashing.py | 26 ++++++------ .../test_process_block_header.py | 22 +++++----- .../block_processing/test_process_deposit.py | 8 ++-- .../test_process_proposer_slashing.py | 22 +++++----- .../block_processing/test_process_transfer.py | 34 +++++++-------- .../block_processing/test_voluntary_exit.py | 24 +++++------ test_libs/pyspec/tests/phase0/conftest.py | 23 ++++------ .../test_process_crosslinks.py | 18 ++++---- .../test_process_registry_updates.py | 4 +- .../pyspec/tests/phase0/test_finality.py | 30 ++++++------- test_libs/pyspec/tests/phase0/test_sanity.py | 30 ++++++------- ...est_process_early_derived_secret_reveal.py | 36 ++++++++-------- test_libs/pyspec/tests/phase1/conftest.py | 20 ++++----- 14 files changed, 163 insertions(+), 176 deletions(-) diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_attestation.py index 5dcf17e383..faa954d53b 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_attestation.py @@ -6,7 +6,7 @@ pytestmark = pytest.mark.attestations -def run_attestation_processing(spec, helpers, state, attestation, valid=True): +def run_attestation_processing(state, attestation, valid=True): """ Run ``spec.process_attestation`` returning the pre and post state. If ``valid == False``, run expecting ``AssertionError`` @@ -29,81 +29,81 @@ def run_attestation_processing(spec, helpers, state, attestation, valid=True): return state, post_state -def test_success(spec, helpers, state): +def test_success(state): attestation = helpers.get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation) + pre_state, post_state = run_attestation_processing(state, attestation) return pre_state, attestation, post_state -def test_success_prevous_epoch(spec, helpers, state): +def test_success_prevous_epoch(state): attestation = helpers.get_valid_attestation(state) block = helpers.build_empty_block_for_next_slot(state) block.slot = state.slot + spec.SLOTS_PER_EPOCH spec.state_transition(state, block) - pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation) + pre_state, post_state = run_attestation_processing(state, attestation) return pre_state, attestation, post_state -def test_before_inclusion_delay(spec, helpers, state): +def test_before_inclusion_delay(state): attestation = helpers.get_valid_attestation(state) # do not increment slot to allow for inclusion delay - pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation, False) + pre_state, post_state = run_attestation_processing(state, attestation, False) return pre_state, attestation, post_state -def test_after_epoch_slots(spec, helpers, state): +def test_after_epoch_slots(state): attestation = helpers.get_valid_attestation(state) block = helpers.build_empty_block_for_next_slot(state) # increment past latest inclusion slot block.slot = state.slot + spec.SLOTS_PER_EPOCH + 1 spec.state_transition(state, block) - pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation, False) + pre_state, post_state = run_attestation_processing(state, attestation, False) return pre_state, attestation, post_state -def test_bad_source_epoch(spec, helpers, state): +def test_bad_source_epoch(state): attestation = helpers.get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.source_epoch += 10 - pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation, False) + pre_state, post_state = run_attestation_processing(state, attestation, False) return pre_state, attestation, post_state -def test_bad_source_root(spec, helpers, state): +def test_bad_source_root(state): attestation = helpers.get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.source_root = b'\x42' * 32 - pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation, False) + pre_state, post_state = run_attestation_processing(state, attestation, False) return pre_state, attestation, post_state -def test_non_zero_crosslink_data_root(spec, helpers, state): +def test_non_zero_crosslink_data_root(state): attestation = helpers.get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.data.crosslink.data_root = b'\x42' * 32 - pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation, False) + pre_state, post_state = run_attestation_processing(state, attestation, False) return pre_state, attestation, post_state -def test_bad_previous_crosslink(spec, helpers, state): +def test_bad_previous_crosslink(state): helpers.next_epoch(state) attestation = helpers.get_valid_attestation(state) for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): @@ -111,28 +111,28 @@ def test_bad_previous_crosslink(spec, helpers, state): state.current_crosslinks[attestation.data.crosslink.shard].epoch += 10 - pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation, False) + pre_state, post_state = run_attestation_processing(state, attestation, False) return pre_state, attestation, post_state -def test_non_empty_custody_bitfield(spec, helpers, state): +def test_non_empty_custody_bitfield(state): attestation = helpers.get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) - pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation, False) + pre_state, post_state = run_attestation_processing(state, attestation, False) return pre_state, attestation, post_state -def test_empty_aggregation_bitfield(spec, helpers, state): +def test_empty_aggregation_bitfield(state): attestation = helpers.get_valid_attestation(state) state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY attestation.aggregation_bitfield = b'\x00' * len(attestation.aggregation_bitfield) - pre_state, post_state = run_attestation_processing(spec, helpers, state, attestation) + pre_state, post_state = run_attestation_processing(state, attestation) return pre_state, attestation, post_state diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_attester_slashing.py index ed48b17cff..66617fd6dc 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_attester_slashing.py @@ -6,7 +6,7 @@ pytestmark = pytest.mark.attester_slashings -def run_attester_slashing_processing(spec, helpers, state, attester_slashing, valid=True): +def run_attester_slashing_processing(state, attester_slashing, valid=True): """ Run ``spec.process_attester_slashing`` returning the pre and post state. If ``valid == False``, run expecting ``AssertionError`` @@ -40,15 +40,15 @@ def run_attester_slashing_processing(spec, helpers, state, attester_slashing, va return state, post_state -def test_success_double(spec, helpers, state): +def test_success_double(state): attester_slashing = helpers.get_valid_attester_slashing(state) - pre_state, post_state = run_attester_slashing_processing(spec, helpers, state, attester_slashing) + pre_state, post_state = run_attester_slashing_processing(state, attester_slashing) return pre_state, attester_slashing, post_state -def test_success_surround(spec, helpers, state): +def test_success_surround(state): helpers.next_epoch(state) state.current_justified_epoch += 1 attester_slashing = helpers.get_valid_attester_slashing(state) @@ -57,32 +57,32 @@ def test_success_surround(spec, helpers, state): attester_slashing.attestation_1.data.source_epoch = attester_slashing.attestation_2.data.source_epoch - 1 attester_slashing.attestation_1.data.target_epoch = attester_slashing.attestation_2.data.target_epoch + 1 - pre_state, post_state = run_attester_slashing_processing(spec, helpers, state, attester_slashing) + pre_state, post_state = run_attester_slashing_processing(state, attester_slashing) return pre_state, attester_slashing, post_state -def test_same_data(spec, helpers, state): +def test_same_data(state): attester_slashing = helpers.get_valid_attester_slashing(state) attester_slashing.attestation_1.data = attester_slashing.attestation_2.data - pre_state, post_state = run_attester_slashing_processing(spec, helpers, state, attester_slashing, False) + pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) return pre_state, attester_slashing, post_state -def test_no_double_or_surround(spec, helpers, state): +def test_no_double_or_surround(state): attester_slashing = helpers.get_valid_attester_slashing(state) attester_slashing.attestation_1.data.target_epoch += 1 - pre_state, post_state = run_attester_slashing_processing(spec, helpers, state, attester_slashing, False) + pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) return pre_state, attester_slashing, post_state -def test_participants_already_slashed(spec, helpers, state): +def test_participants_already_slashed(state): attester_slashing = helpers.get_valid_attester_slashing(state) # set all indices to slashed @@ -91,17 +91,17 @@ def test_participants_already_slashed(spec, helpers, state): for index in validator_indices: state.validator_registry[index].slashed = True - pre_state, post_state = run_attester_slashing_processing(spec, helpers, state, attester_slashing, False) + pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) return pre_state, attester_slashing, post_state -def test_custody_bit_0_and_1(spec, helpers, state): +def test_custody_bit_0_and_1(state): attester_slashing = helpers.get_valid_attester_slashing(state) attester_slashing.attestation_1.custody_bit_1_indices = ( attester_slashing.attestation_1.custody_bit_0_indices ) - pre_state, post_state = run_attester_slashing_processing(spec, helpers, state, attester_slashing, False) + pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) return pre_state, attester_slashing, post_state diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_block_header.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_block_header.py index e4cd70f770..f7032a2839 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_block_header.py @@ -6,17 +6,17 @@ pytestmark = pytest.mark.header -def prepare_state_for_header_processing(spec, helpers, state): +def prepare_state_for_header_processing(state): spec.process_slot(state) helpers.advance_slot(state) -def run_block_header_processing(spec, helpers, state, block, valid=True): +def run_block_header_processing(state, block, valid=True): """ Run ``spec.process_block_header`` returning the pre and post state. If ``valid == False``, run expecting ``AssertionError`` """ - prepare_state_for_header_processing(spec, helpers, state) + prepare_state_for_header_processing(state) post_state = deepcopy(state) if not valid: @@ -28,29 +28,29 @@ def run_block_header_processing(spec, helpers, state, block, valid=True): return state, post_state -def test_success(spec, helpers, state): +def test_success(state): block = helpers.build_empty_block_for_next_slot(state) - pre_state, post_state = run_block_header_processing(spec, helpers, state, block) + pre_state, post_state = run_block_header_processing(state, block) return state, block, post_state -def test_invalid_slot(spec, helpers, state): +def test_invalid_slot(state): block = helpers.build_empty_block_for_next_slot(state) block.slot = state.slot + 2 # invalid slot - pre_state, post_state = run_block_header_processing(spec, helpers, state, block, valid=False) + pre_state, post_state = run_block_header_processing(state, block, valid=False) return pre_state, block, None -def test_invalid_parent_block_root(spec, helpers, state): +def test_invalid_parent_block_root(state): block = helpers.build_empty_block_for_next_slot(state) block.parent_root = b'\12' * 32 # invalid prev root - pre_state, post_state = run_block_header_processing(spec, helpers, state, block, valid=False) + pre_state, post_state = run_block_header_processing(state, block, valid=False) return pre_state, block, None -def test_proposer_slashed(spec, helpers, state): +def test_proposer_slashed(state): # use stub state to get proposer index of next slot stub_state = deepcopy(state) helpers.next_slot(stub_state) @@ -61,5 +61,5 @@ def test_proposer_slashed(spec, helpers, state): block = helpers.build_empty_block_for_next_slot(state) - pre_state, post_state = run_block_header_processing(spec, helpers, state, block, valid=False) + pre_state, post_state = run_block_header_processing(state, block, valid=False) return pre_state, block, None diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_deposit.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_deposit.py index 97d49114b6..f3abab3544 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_deposit.py @@ -6,7 +6,7 @@ pytestmark = pytest.mark.deposits -def test_success(spec, helpers, state): +def test_success(state): pre_state = deepcopy(state) # fill previous deposits with zero-hash deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) @@ -38,7 +38,7 @@ def test_success(spec, helpers, state): return pre_state, deposit, post_state -def test_success_top_up(spec, helpers, state): +def test_success_top_up(state): pre_state = deepcopy(state) deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) @@ -70,7 +70,7 @@ def test_success_top_up(spec, helpers, state): return pre_state, deposit, post_state -def test_wrong_index(spec, helpers, state): +def test_wrong_index(state): pre_state = deepcopy(state) deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) @@ -99,7 +99,7 @@ def test_wrong_index(spec, helpers, state): return pre_state, deposit, None -def test_bad_merkle_proof(spec, helpers, state): +def test_bad_merkle_proof(state): pre_state = deepcopy(state) deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_proposer_slashing.py index 6f82d6c33e..f97eb1584f 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_proposer_slashing.py @@ -6,7 +6,7 @@ pytestmark = pytest.mark.proposer_slashings -def run_proposer_slashing_processing(spec, helpers, state, proposer_slashing, valid=True): +def run_proposer_slashing_processing(state, proposer_slashing, valid=True): """ Run ``spec.process_proposer_slashing`` returning the pre and post state. If ``valid == False``, run expecting ``AssertionError`` @@ -33,48 +33,48 @@ def run_proposer_slashing_processing(spec, helpers, state, proposer_slashing, va return state, post_state -def test_success(spec, helpers, state): +def test_success(state): proposer_slashing = helpers.get_valid_proposer_slashing(state) - pre_state, post_state = run_proposer_slashing_processing(spec, helpers, state, proposer_slashing) + pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing) return pre_state, proposer_slashing, post_state -def test_epochs_are_different(spec, helpers, state): +def test_epochs_are_different(state): proposer_slashing = helpers.get_valid_proposer_slashing(state) # set slots to be in different epochs proposer_slashing.header_2.slot += spec.SLOTS_PER_EPOCH - pre_state, post_state = run_proposer_slashing_processing(spec, helpers, state, proposer_slashing, False) + pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) return pre_state, proposer_slashing, post_state -def test_headers_are_same(spec, helpers, state): +def test_headers_are_same(state): proposer_slashing = helpers.get_valid_proposer_slashing(state) # set headers to be the same proposer_slashing.header_2 = proposer_slashing.header_1 - pre_state, post_state = run_proposer_slashing_processing(spec, helpers, state, proposer_slashing, False) + pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) return pre_state, proposer_slashing, post_state -def test_proposer_is_slashed(spec, helpers, state): +def test_proposer_is_slashed(state): proposer_slashing = helpers.get_valid_proposer_slashing(state) # set proposer to slashed state.validator_registry[proposer_slashing.proposer_index].slashed = True - pre_state, post_state = run_proposer_slashing_processing(spec, helpers, state, proposer_slashing, False) + pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) return pre_state, proposer_slashing, post_state -def test_proposer_is_withdrawn(spec, helpers, state): +def test_proposer_is_withdrawn(state): proposer_slashing = helpers.get_valid_proposer_slashing(state) # set proposer withdrawable_epoch in past @@ -82,6 +82,6 @@ def test_proposer_is_withdrawn(spec, helpers, state): proposer_index = proposer_slashing.proposer_index state.validator_registry[proposer_index].withdrawable_epoch = current_epoch - 1 - pre_state, post_state = run_proposer_slashing_processing(spec, helpers, state, proposer_slashing, False) + pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) return pre_state, proposer_slashing, post_state diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py index 197898ec0f..724c696478 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py @@ -6,7 +6,7 @@ pytestmark = pytest.mark.transfers -def run_transfer_processing(spec, helpers, state, transfer, valid=True): +def run_transfer_processing(state, transfer, valid=True): """ Run ``spec.process_transfer`` returning the pre and post state. If ``valid == False``, run expecting ``AssertionError`` @@ -34,17 +34,17 @@ def run_transfer_processing(spec, helpers, state, transfer, valid=True): return state, post_state -def test_success_non_activated(spec, helpers, state): +def test_success_non_activated(state): transfer = helpers.get_valid_transfer(state) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer) + pre_state, post_state = run_transfer_processing(state, transfer) return pre_state, transfer, post_state -def test_success_withdrawable(spec, helpers, state): +def test_success_withdrawable(state): helpers.next_epoch(state) transfer = helpers.get_valid_transfer(state) @@ -52,44 +52,44 @@ def test_success_withdrawable(spec, helpers, state): # withdrawable_epoch in past so can transfer state.validator_registry[transfer.sender].withdrawable_epoch = spec.get_current_epoch(state) - 1 - pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer) + pre_state, post_state = run_transfer_processing(state, transfer) return pre_state, transfer, post_state -def test_success_active_above_max_effective(spec, helpers, state): +def test_success_active_above_max_effective(state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE // 32 state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + amount transfer = helpers.get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) - pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer) + pre_state, post_state = run_transfer_processing(state, transfer) return pre_state, transfer, post_state -def test_active_but_transfer_past_effective_balance(spec, helpers, state): +def test_active_but_transfer_past_effective_balance(state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE // 32 state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE transfer = helpers.get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) - pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer, False) + pre_state, post_state = run_transfer_processing(state, transfer, False) return pre_state, transfer, post_state -def test_incorrect_slot(spec, helpers, state): +def test_incorrect_slot(state): transfer = helpers.get_valid_transfer(state, slot=state.slot+1) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer, False) + pre_state, post_state = run_transfer_processing(state, transfer, False) return pre_state, transfer, post_state -def test_insufficient_balance(spec, helpers, state): +def test_insufficient_balance(state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] amount = spec.MAX_EFFECTIVE_BALANCE state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE @@ -98,12 +98,12 @@ def test_insufficient_balance(spec, helpers, state): # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer, False) + pre_state, post_state = run_transfer_processing(state, transfer, False) return pre_state, transfer, post_state -def test_no_dust(spec, helpers, state): +def test_no_dust(state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] balance = state.balances[sender_index] transfer = helpers.get_valid_transfer(state, sender_index=sender_index, amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, fee=0) @@ -111,18 +111,18 @@ def test_no_dust(spec, helpers, state): # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer, False) + pre_state, post_state = run_transfer_processing(state, transfer, False) return pre_state, transfer, post_state -def test_invalid_pubkey(spec, helpers, state): +def test_invalid_pubkey(state): transfer = helpers.get_valid_transfer(state) state.validator_registry[transfer.sender].withdrawal_credentials = spec.ZERO_HASH # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - pre_state, post_state = run_transfer_processing(spec, helpers, state, transfer, False) + pre_state, post_state = run_transfer_processing(state, transfer, False) return pre_state, transfer, post_state diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_voluntary_exit.py b/test_libs/pyspec/tests/phase0/block_processing/test_voluntary_exit.py index d9e643b559..a3e54f00e2 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_voluntary_exit.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_voluntary_exit.py @@ -6,7 +6,7 @@ pytestmark = pytest.mark.voluntary_exits -def run_voluntary_exit_processing(spec, helpers, state, voluntary_exit, valid=True): +def run_voluntary_exit_processing(state, voluntary_exit, valid=True): """ Run ``spec.process_voluntary_exit`` returning the pre and post state. If ``valid == False``, run expecting ``AssertionError`` @@ -27,7 +27,7 @@ def run_voluntary_exit_processing(spec, helpers, state, voluntary_exit, valid=Tr return state, post_state -def test_success(spec, helpers, state): +def test_success(state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH @@ -42,11 +42,11 @@ def test_success(spec, helpers, state): privkey, ) - pre_state, post_state = run_voluntary_exit_processing(spec, helpers, state, voluntary_exit) + pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit) return pre_state, voluntary_exit, post_state -def test_success_exit_queue(spec, helpers, state): +def test_success_exit_queue(state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH @@ -64,7 +64,7 @@ def test_success_exit_queue(spec, helpers, state): privkey, ) - pre_state, post_state = run_voluntary_exit_processing(spec, helpers, post_state, voluntary_exit) + pre_state, post_state = run_voluntary_exit_processing(post_state, voluntary_exit) # exit an additional validator validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] @@ -76,7 +76,7 @@ def test_success_exit_queue(spec, helpers, state): privkey, ) - pre_state, post_state = run_voluntary_exit_processing(spec, helpers, post_state, voluntary_exit) + pre_state, post_state = run_voluntary_exit_processing(post_state, voluntary_exit) assert ( post_state.validator_registry[validator_index].exit_epoch == @@ -86,7 +86,7 @@ def test_success_exit_queue(spec, helpers, state): return pre_state, voluntary_exit, post_state -def test_validator_not_active(spec, helpers, state): +def test_validator_not_active(state): current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] privkey = helpers.pubkey_to_privkey[state.validator_registry[validator_index].pubkey] @@ -103,11 +103,11 @@ def test_validator_not_active(spec, helpers, state): privkey, ) - pre_state, post_state = run_voluntary_exit_processing(spec, helpers, state, voluntary_exit, False) + pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit, False) return pre_state, voluntary_exit, post_state -def test_validator_already_exited(spec, helpers, state): +def test_validator_already_exited(state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow validator able to exit state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH @@ -125,11 +125,11 @@ def test_validator_already_exited(spec, helpers, state): privkey, ) - pre_state, post_state = run_voluntary_exit_processing(spec, helpers, state, voluntary_exit, False) + pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit, False) return pre_state, voluntary_exit, post_state -def test_validator_not_active_long_enough(spec, helpers, state): +def test_validator_not_active_long_enough(state): current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[0] privkey = helpers.pubkey_to_privkey[state.validator_registry[validator_index].pubkey] @@ -146,5 +146,5 @@ def test_validator_not_active_long_enough(spec, helpers, state): spec.PERSISTENT_COMMITTEE_PERIOD ) - pre_state, post_state = run_voluntary_exit_processing(spec, helpers, state, voluntary_exit, False) + pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit, False) return pre_state, voluntary_exit, post_state diff --git a/test_libs/pyspec/tests/phase0/conftest.py b/test_libs/pyspec/tests/phase0/conftest.py index 25b540aa0f..4db8f3919d 100644 --- a/test_libs/pyspec/tests/phase0/conftest.py +++ b/test_libs/pyspec/tests/phase0/conftest.py @@ -1,10 +1,10 @@ import pytest -from eth2spec.phase0 import spec as _spec +from eth2spec.phase0 import spec from preset_loader import loader -from tests.phase0 import helpers as _helpers - +from tests.phase0 import helpers +from tests.phase0 import test_finality def pytest_addoption(parser): parser.addoption( @@ -13,13 +13,15 @@ def pytest_addoption(parser): @pytest.fixture(autouse=True) def config(request): + request.function.__globals__['spec'] = spec + request.function.__globals__['helpers'] = helpers config_name = request.config.getoption("--config") presets = loader.load_presets('../../configs/', config_name) - _spec.apply_constants_preset(presets) + spec.apply_constants_preset(presets) @pytest.fixture def num_validators(config): - return _spec.SLOTS_PER_EPOCH * 8 + return spec.SLOTS_PER_EPOCH * 8 @pytest.fixture def deposit_data_leaves(): @@ -27,13 +29,4 @@ def deposit_data_leaves(): @pytest.fixture def state(num_validators, deposit_data_leaves): - return _helpers.create_genesis_state(num_validators, deposit_data_leaves) - -@pytest.fixture -def spec(): - return _spec - -@pytest.fixture -def helpers(): - return _helpers - + return helpers.create_genesis_state(num_validators, deposit_data_leaves) diff --git a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py index 6e4ad2dd6a..a3ec38989a 100644 --- a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py @@ -6,7 +6,7 @@ pytestmark = pytest.mark.crosslinks -def run_process_crosslinks(spec, helpers, state, valid=True): +def run_process_crosslinks(state, valid=True): # transition state to slot before state transition slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 block = helpers.build_empty_block_for_next_slot(state) @@ -22,8 +22,8 @@ def run_process_crosslinks(spec, helpers, state, valid=True): return state, post_state -def test_no_attestations(spec, helpers, state): - pre_state, post_state = run_process_crosslinks(spec, helpers, state) +def test_no_attestations(state): + pre_state, post_state = run_process_crosslinks(state) for shard in range(spec.SHARD_COUNT): assert post_state.previous_crosslinks[shard] == post_state.current_crosslinks[shard] @@ -31,7 +31,7 @@ def test_no_attestations(spec, helpers, state): return pre_state, post_state -def test_single_crosslink_update_from_current_epoch(spec, helpers, state): +def test_single_crosslink_update_from_current_epoch(state): helpers.next_epoch(state) attestation = helpers.get_valid_attestation(state) @@ -41,7 +41,7 @@ def test_single_crosslink_update_from_current_epoch(spec, helpers, state): assert len(state.current_epoch_attestations) == 1 - pre_state, post_state = run_process_crosslinks(spec, helpers, state) + pre_state, post_state = run_process_crosslinks(state) shard = attestation.data.crosslink.shard assert post_state.previous_crosslinks[shard] != post_state.current_crosslinks[shard] @@ -50,7 +50,7 @@ def test_single_crosslink_update_from_current_epoch(spec, helpers, state): return pre_state, post_state -def test_single_crosslink_update_from_previous_epoch(spec, helpers, state): +def test_single_crosslink_update_from_previous_epoch(state): helpers.next_epoch(state) attestation = helpers.get_valid_attestation(state) @@ -60,7 +60,7 @@ def test_single_crosslink_update_from_previous_epoch(spec, helpers, state): assert len(state.previous_epoch_attestations) == 1 - pre_state, post_state = run_process_crosslinks(spec, helpers, state) + pre_state, post_state = run_process_crosslinks(state) crosslink_deltas = spec.get_crosslink_deltas(state) shard = attestation.data.crosslink.shard @@ -74,7 +74,7 @@ def test_single_crosslink_update_from_previous_epoch(spec, helpers, state): return pre_state, post_state -def test_double_late_crosslink(spec, helpers, state): +def test_double_late_crosslink(state): helpers.next_epoch(state) state.slot += 4 @@ -100,7 +100,7 @@ def test_double_late_crosslink(spec, helpers, state): assert len(state.previous_epoch_attestations) == 1 assert len(state.current_epoch_attestations) == 0 - pre_state, post_state = run_process_crosslinks(spec, helpers, state) + pre_state, post_state = run_process_crosslinks(state) crosslink_deltas = spec.get_crosslink_deltas(state) shard = attestation_2.data.crosslink.shard diff --git a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py index 8b5ebf9b4d..962ad67040 100644 --- a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py @@ -6,7 +6,7 @@ pytestmark = pytest.mark.state -def test_activation(spec, helpers, state): +def test_activation(state): index = 0 assert spec.is_active_validator(state.validator_registry[index], spec.get_current_epoch(state)) @@ -33,7 +33,7 @@ def test_activation(spec, helpers, state): return pre_state, blocks, state -def test_ejection(spec, helpers, state): +def test_ejection(state): index = 0 assert spec.is_active_validator(state.validator_registry[index], spec.get_current_epoch(state)) assert state.validator_registry[index].exit_epoch == spec.FAR_FUTURE_EPOCH diff --git a/test_libs/pyspec/tests/phase0/test_finality.py b/test_libs/pyspec/tests/phase0/test_finality.py index e30e125f9a..be8ecaa1db 100644 --- a/test_libs/pyspec/tests/phase0/test_finality.py +++ b/test_libs/pyspec/tests/phase0/test_finality.py @@ -33,7 +33,7 @@ def check_finality(state, assert state.finalized_root == prev_state.finalized_root -def next_epoch_with_attestations(spec, helpers, state, +def next_epoch_with_attestations(state, fill_cur_epoch, fill_prev_epoch): post_state = deepcopy(state) @@ -59,12 +59,12 @@ def next_epoch_with_attestations(spec, helpers, state, return state, blocks, post_state -def test_finality_rule_4(spec, helpers, state): +def test_finality_rule_4(state): test_state = deepcopy(state) blocks = [] for epoch in range(4): - prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, True, False) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) blocks += new_blocks # justification/finalization skipped at GENESIS_EPOCH @@ -84,7 +84,7 @@ def test_finality_rule_4(spec, helpers, state): return state, blocks, test_state -def test_finality_rule_1(spec, helpers, state): +def test_finality_rule_1(state): # get past first two epochs that finality does not run on helpers.next_epoch(state) helpers.next_epoch(state) @@ -94,7 +94,7 @@ def test_finality_rule_1(spec, helpers, state): blocks = [] for epoch in range(3): - prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, False, True) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, True) blocks += new_blocks if epoch == 0: @@ -110,7 +110,7 @@ def test_finality_rule_1(spec, helpers, state): return pre_state, blocks, test_state -def test_finality_rule_2(spec, helpers, state): +def test_finality_rule_2(state): # get past first two epochs that finality does not run on helpers.next_epoch(state) helpers.next_epoch(state) @@ -121,13 +121,13 @@ def test_finality_rule_2(spec, helpers, state): blocks = [] for epoch in range(3): if epoch == 0: - prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, True, False) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) check_finality(test_state, prev_state, True, False, False) elif epoch == 1: - prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, False, False) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, False) check_finality(test_state, prev_state, False, True, False) elif epoch == 2: - prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, False, True) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, True) # finalized by rule 2 check_finality(test_state, prev_state, True, False, True) assert test_state.finalized_epoch == prev_state.previous_justified_epoch @@ -138,7 +138,7 @@ def test_finality_rule_2(spec, helpers, state): return pre_state, blocks, test_state -def test_finality_rule_3(spec, helpers, state): +def test_finality_rule_3(state): """ Test scenario described here https://github.com/ethereum/eth2.0-specs/issues/611#issuecomment-463612892 @@ -152,29 +152,29 @@ def test_finality_rule_3(spec, helpers, state): test_state = deepcopy(state) blocks = [] - prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, True, False) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) blocks += new_blocks check_finality(test_state, prev_state, True, False, False) # In epoch N, JE is set to N, prev JE is set to N-1 - prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, True, False) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, False) blocks += new_blocks check_finality(test_state, prev_state, True, True, True) # In epoch N+1, JE is N, prev JE is N-1, and not enough messages get in to do anything - prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, False, False) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, False) blocks += new_blocks check_finality(test_state, prev_state, False, True, False) # In epoch N+2, JE is N, prev JE is N, and enough messages from the previous epoch get in to justify N+1. # N+1 now becomes the JE. Not enough messages from epoch N+2 itself get in to justify N+2 - prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, False, True) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, False, True) blocks += new_blocks # rule 2 check_finality(test_state, prev_state, True, False, True) # In epoch N+3, LJE is N+1, prev LJE is N, and enough messages get in to justify epochs N+2 and N+3. - prev_state, new_blocks, test_state = next_epoch_with_attestations(spec, helpers, test_state, True, True) + prev_state, new_blocks, test_state = next_epoch_with_attestations(test_state, True, True) blocks += new_blocks # rule 3 check_finality(test_state, prev_state, True, True, True) diff --git a/test_libs/pyspec/tests/phase0/test_sanity.py b/test_libs/pyspec/tests/phase0/test_sanity.py index ad206881da..821d973866 100644 --- a/test_libs/pyspec/tests/phase0/test_sanity.py +++ b/test_libs/pyspec/tests/phase0/test_sanity.py @@ -17,7 +17,7 @@ pytestmark = pytest.mark.sanity -def test_slot_transition(spec, helpers, state): +def test_slot_transition(state): test_state = deepcopy(state) spec.process_slot(test_state) helpers.advance_slot(test_state) @@ -26,7 +26,7 @@ def test_slot_transition(spec, helpers, state): return test_state -def test_empty_block_transition(spec, helpers, state): +def test_empty_block_transition(state): test_state = deepcopy(state) block = helpers.build_empty_block_for_next_slot(test_state) @@ -38,7 +38,7 @@ def test_empty_block_transition(spec, helpers, state): return state, [block], test_state -def test_skipped_slots(spec, helpers, state): +def test_skipped_slots(state): test_state = deepcopy(state) block = helpers.build_empty_block_for_next_slot(test_state) block.slot += 3 @@ -52,7 +52,7 @@ def test_skipped_slots(spec, helpers, state): return state, [block], test_state -def test_empty_epoch_transition(spec, helpers, state): +def test_empty_epoch_transition(state): test_state = deepcopy(state) block = helpers.build_empty_block_for_next_slot(test_state) block.slot += spec.SLOTS_PER_EPOCH @@ -66,7 +66,7 @@ def test_empty_epoch_transition(spec, helpers, state): return state, [block], test_state -def test_empty_epoch_transition_not_finalizing(spec, helpers, state): +def test_empty_epoch_transition_not_finalizing(state): test_state = deepcopy(state) block = helpers.build_empty_block_for_next_slot(test_state) block.slot += spec.SLOTS_PER_EPOCH * 5 @@ -81,7 +81,7 @@ def test_empty_epoch_transition_not_finalizing(spec, helpers, state): return state, [block], test_state -def test_proposer_slashing(spec, helpers, state): +def test_proposer_slashing(state): test_state = deepcopy(state) proposer_slashing = helpers.get_valid_proposer_slashing(state) validator_index = proposer_slashing.proposer_index @@ -105,7 +105,7 @@ def test_proposer_slashing(spec, helpers, state): return state, [block], test_state -def test_attester_slashing(spec, helpers, state): +def test_attester_slashing(state): test_state = deepcopy(state) attester_slashing = helpers.get_valid_attester_slashing(state) validator_index = attester_slashing.attestation_1.custody_bit_0_indices[0] @@ -136,7 +136,7 @@ def test_attester_slashing(spec, helpers, state): return state, [block], test_state -def test_deposit_in_block(spec, helpers, state): +def test_deposit_in_block(state): pre_state = deepcopy(state) test_deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) @@ -173,7 +173,7 @@ def test_deposit_in_block(spec, helpers, state): return pre_state, [block], post_state -def test_deposit_top_up(spec, helpers, state): +def test_deposit_top_up(state): pre_state = deepcopy(state) test_deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) @@ -212,7 +212,7 @@ def test_deposit_top_up(spec, helpers, state): return pre_state, [block], post_state -def test_attestation(spec, helpers, state): +def test_attestation(state): state.slot = spec.SLOTS_PER_EPOCH test_state = deepcopy(state) attestation = helpers.get_valid_attestation(state) @@ -243,7 +243,7 @@ def test_attestation(spec, helpers, state): return state, [attestation_block, epoch_block], test_state -def test_voluntary_exit(spec, helpers, state): +def test_voluntary_exit(state): pre_state = deepcopy(state) validator_index = spec.get_active_validator_indices( pre_state, @@ -289,7 +289,7 @@ def test_voluntary_exit(spec, helpers, state): return pre_state, [initiate_exit_block, exit_block], post_state -def test_transfer(spec, helpers, state): +def test_transfer(state): # overwrite default 0 to test spec.apply_constants_preset({"MAX_TRANSFERS": 1}) @@ -341,7 +341,7 @@ def test_transfer(spec, helpers, state): return pre_state, [block], post_state -def test_balance_driven_status_transitions(spec, helpers, state): +def test_balance_driven_status_transitions(state): current_epoch = spec.get_current_epoch(state) validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] @@ -363,7 +363,7 @@ def test_balance_driven_status_transitions(spec, helpers, state): return state, [block], post_state -def test_historical_batch(spec, helpers, state): +def test_historical_batch(state): state.slot += spec.SLOTS_PER_HISTORICAL_ROOT - (state.slot % spec.SLOTS_PER_HISTORICAL_ROOT) - 1 post_state = deepcopy(state) @@ -379,7 +379,7 @@ def test_historical_batch(spec, helpers, state): return state, [block], post_state -def test_eth1_data_votes(spec, helpers, state): +def test_eth1_data_votes(state): post_state = deepcopy(state) expected_votes = 0 diff --git a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py index e3cf30ef3c..08f7cf9ac7 100644 --- a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py @@ -4,7 +4,7 @@ #mark entire file as 'randao_key_reveals' pytestmark = pytest.mark.randao_key_reveals -def terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal, valid=True): +def terun_early_derived_secret_reveal_processing(state, randao_key_reveal, valid=True): """ Run ``process_randao_key_reveal`` returning the pre and post state. If ``valid == False``, run expecting ``AssertionError`` @@ -34,63 +34,63 @@ def terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_ke return state, post_state -def test_success(spec, helpers, state): +def test_success(state): randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state) - pre_state, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal) + pre_state, post_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal) return pre_state, randao_key_reveal, post_state -def test_reveal_from_current_epoch(spec, helpers, state): +def test_reveal_from_current_epoch(state): randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state)) - pre_state, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal, False) + pre_state, post_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal, False) return pre_state, randao_key_reveal, post_state @pytest.mark.skip(reason="Not currently possible as we are testing at epoch 0") -def test_reveal_from_past_epoch(spec, helpers, state): +def test_reveal_from_past_epoch(state): randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) - 1) - pre_state, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal, False) + pre_state, post_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal, False) return pre_state, randao_key_reveal, post_state -def test_reveal_with_custody_padding(spec, helpers, state): +def test_reveal_with_custody_padding(state): randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING) - pre_state, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal, True) + pre_state, post_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal, True) return pre_state, randao_key_reveal, post_state -def test_reveal_with_custody_padding_minus_one(spec, helpers, state): +def test_reveal_with_custody_padding_minus_one(state): randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING - 1) - pre_state, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal, True) + pre_state, post_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal, True) return pre_state, randao_key_reveal, post_state -def test_double_reveal(spec, helpers, state): +def test_double_reveal(state): randao_key_reveal1 = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) + spec.RANDAO_PENALTY_EPOCHS + 1) - pre_state, intermediate_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal1) + pre_state, intermediate_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal1) randao_key_reveal2 = helpers.get_valid_early_derived_secret_reveal(intermediate_state, spec.get_current_epoch(pre_state) + spec.RANDAO_PENALTY_EPOCHS + 1) - _, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, intermediate_state, randao_key_reveal2, False) + _, post_state = terun_early_derived_secret_reveal_processing(intermediate_state, randao_key_reveal2, False) return pre_state, [randao_key_reveal1, randao_key_reveal2], post_state -def test_revealer_is_slashed(spec, helpers, state): +def test_revealer_is_slashed(state): randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state)) state.validator_registry[randao_key_reveal.revealed_index].slashed = True - pre_state, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal, False) + pre_state, post_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal, False) return pre_state, randao_key_reveal, post_state -def test_far_future_epoch(spec, helpers, state): +def test_far_future_epoch(state): randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) + spec.EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS) - pre_state, post_state = terun_early_derived_secret_reveal_processing(spec, helpers, state, randao_key_reveal, False) + pre_state, post_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal, False) return pre_state, randao_key_reveal, post_state diff --git a/test_libs/pyspec/tests/phase1/conftest.py b/test_libs/pyspec/tests/phase1/conftest.py index 4af3451a5a..62dd2be385 100644 --- a/test_libs/pyspec/tests/phase1/conftest.py +++ b/test_libs/pyspec/tests/phase1/conftest.py @@ -1,9 +1,9 @@ import pytest -from eth2spec.phase1 import spec as _spec +from eth2spec.phase1 import spec from preset_loader import loader -from tests.phase1 import helpers as _helpers +from tests.phase1 import helpers as helpers from tests.phase0.conftest import ( pytest_addoption, @@ -13,23 +13,17 @@ # This is redfined so that the constants are re-applied @pytest.fixture(autouse=True) def config(request): + request.function.__globals__['spec'] = spec + request.function.__globals__['helpers'] = helpers config_name = request.config.getoption("--config") presets = loader.load_presets('../../configs/', config_name) - _spec.apply_constants_preset(presets) + spec.apply_constants_preset(presets) @pytest.fixture def num_validators(config): - return _spec.SLOTS_PER_EPOCH * 8 + return spec.SLOTS_PER_EPOCH * 8 #This is redefined so that the BeaconState is the new SSZ Object @pytest.fixture def state(num_validators, deposit_data_leaves): - return _helpers.create_genesis_state(num_validators, deposit_data_leaves) - -@pytest.fixture -def spec(): - return _spec - -@pytest.fixture -def helpers(): - return _helpers + return helpers.create_genesis_state(num_validators, deposit_data_leaves) From 65b0701f8c68ef9416a5ebbb889c04c0de52fd73 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 20 May 2019 18:14:37 +0200 Subject: [PATCH 154/308] Helpers depend on spec version and p1 tests work --- Makefile | 2 +- test_libs/pyspec/tests/phase0/conftest.py | 5 +- .../test_process_crosslinks.py | 4 +- test_libs/pyspec/tests/phase0/helpers.py | 148 +++++++----------- .../pyspec/tests/phase0/test_finality.py | 2 +- test_libs/pyspec/tests/phase1/conftest.py | 18 ++- test_libs/pyspec/tests/phase1/helpers.py | 31 ++-- 7 files changed, 83 insertions(+), 127 deletions(-) diff --git a/Makefile b/Makefile index a262ac5e1a..f8682734dc 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ PY_SPEC_PHASE_1_DEPS = $(SPEC_DIR)/core/1_*.md PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) $(PY_SPEC_PHASE_1_TARGETS) -.PHONY: clean all test citest gen_yaml_tests pyspec phase0 install_test +.PHONY: clean all test citest gen_yaml_tests pyspec phase0 phase1 install_test all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) diff --git a/test_libs/pyspec/tests/phase0/conftest.py b/test_libs/pyspec/tests/phase0/conftest.py index 4db8f3919d..0549a1422e 100644 --- a/test_libs/pyspec/tests/phase0/conftest.py +++ b/test_libs/pyspec/tests/phase0/conftest.py @@ -13,11 +13,12 @@ def pytest_addoption(parser): @pytest.fixture(autouse=True) def config(request): - request.function.__globals__['spec'] = spec - request.function.__globals__['helpers'] = helpers config_name = request.config.getoption("--config") presets = loader.load_presets('../../configs/', config_name) spec.apply_constants_preset(presets) + helpers.spec = spec + request.function.__globals__['spec'] = spec + request.function.__globals__['helpers'] = helpers @pytest.fixture def num_validators(config): diff --git a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py index a3ec38989a..19c668e749 100644 --- a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py @@ -67,7 +67,7 @@ def test_single_crosslink_update_from_previous_epoch(state): assert post_state.previous_crosslinks[shard] != post_state.current_crosslinks[shard] assert pre_state.current_crosslinks[shard] != post_state.current_crosslinks[shard] # ensure rewarded - for index in helpers.get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.crosslink.shard): + for index in spec.get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.crosslink.shard): assert crosslink_deltas[0][index] > 0 assert crosslink_deltas[1][index] == 0 @@ -108,7 +108,7 @@ def test_double_late_crosslink(state): # ensure that the current crosslinks were not updated by the second attestation assert post_state.previous_crosslinks[shard] == post_state.current_crosslinks[shard] # ensure no reward, only penalties for the failed crosslink - for index in helpers.get_crosslink_committee(state, attestation_2.data.target_epoch, attestation_2.data.crosslink.shard): + for index in spec.get_crosslink_committee(state, attestation_2.data.target_epoch, attestation_2.data.crosslink.shard): assert crosslink_deltas[0][index] == 0 assert crosslink_deltas[1][index] > 0 diff --git a/test_libs/pyspec/tests/phase0/helpers.py b/test_libs/pyspec/tests/phase0/helpers.py index e8b4f789ee..82851c1cca 100644 --- a/test_libs/pyspec/tests/phase0/helpers.py +++ b/test_libs/pyspec/tests/phase0/helpers.py @@ -2,58 +2,18 @@ from py_ecc import bls -import eth2spec.phase0.spec as spec - from eth2spec.utils.minimal_ssz import signing_root -from eth2spec.phase0.spec import ( - # constants - ZERO_HASH, - MAX_EPOCHS_PER_CROSSLINK, - # SSZ - Attestation, - AttestationData, - AttestationDataAndCustodyBit, - AttesterSlashing, - BeaconBlock, - BeaconBlockHeader, - Crosslink, - Deposit, - DepositData, - Eth1Data, - ProposerSlashing, - Transfer, - VoluntaryExit, - # functions - convert_to_indexed, - get_active_validator_indices, - get_attesting_indices, - get_block_root, - get_block_root_at_slot, - get_crosslink_committee, - get_current_epoch, - get_domain, - get_epoch_start_slot, - get_genesis_beacon_state, - get_previous_epoch, - get_shard_delta, - hash_tree_root, - slot_to_epoch, - state_transition, - verify_merkle_branch, - hash, -) + from eth2spec.utils.merkle_minimal import ( calc_merkle_tree_from_leaves, get_merkle_proof, get_merkle_root, ) - privkeys = [i + 1 for i in range(1024)] pubkeys = [bls.privtopub(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} - def advance_slot(state) -> None: state.slot += 1 @@ -83,10 +43,10 @@ def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=N deposit_data_list = [] for i in range(num_validators): pubkey = pubkeys[i] - deposit_data = DepositData( + deposit_data = spec.DepositData( pubkey=pubkey, # insecurely use pubkey as withdrawal key as well - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], + withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(pubkey)[1:], amount=spec.MAX_EFFECTIVE_BALANCE, signature=signature, ) @@ -95,12 +55,12 @@ def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=N tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) root = get_merkle_root((tuple(deposit_data_leaves))) proof = list(get_merkle_proof(tree, item_index=i)) - assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, i, root) + assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, i, root) deposit_data_list.append(deposit_data) genesis_validator_deposits = [] for i in range(num_validators): - genesis_validator_deposits.append(Deposit( + genesis_validator_deposits.append(spec.Deposit( proof=list(get_merkle_proof(tree, item_index=i)), index=i, data=deposit_data_list[i] @@ -113,10 +73,10 @@ def create_genesis_state(num_validators, deposit_data_leaves=None): num_validators, deposit_data_leaves, ) - return get_genesis_beacon_state( + return spec.get_genesis_beacon_state( initial_deposits, genesis_time=0, - genesis_eth1_data=Eth1Data( + genesis_eth1_data=spec.Eth1Data( deposit_root=deposit_root, deposit_count=len(initial_deposits), block_hash=spec.ZERO_HASH, @@ -125,7 +85,7 @@ def create_genesis_state(num_validators, deposit_data_leaves=None): def build_empty_block_for_next_slot(state): - empty_block = BeaconBlock() + empty_block = spec.BeaconBlock() empty_block.slot = state.slot + 1 empty_block.body.eth1_data.deposit_count = state.deposit_index previous_block_header = deepcopy(state.latest_block_header) @@ -136,16 +96,16 @@ def build_empty_block_for_next_slot(state): def build_deposit_data(state, pubkey, privkey, amount): - deposit_data = DepositData( + deposit_data = spec.DepositData( pubkey=pubkey, # insecurely use pubkey as withdrawal key as well - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + hash(pubkey)[1:], + withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(pubkey)[1:], amount=amount, ) signature = bls.sign( message_hash=signing_root(deposit_data), privkey=privkey, - domain=get_domain( + domain=spec.get_domain( state, spec.DOMAIN_DEPOSIT, ) @@ -160,15 +120,15 @@ def build_attestation_data(state, slot, shard): if slot == state.slot: block_root = build_empty_block_for_next_slot(state).parent_root else: - block_root = get_block_root_at_slot(state, slot) + block_root = spec.get_block_root_at_slot(state, slot) - current_epoch_start_slot = get_epoch_start_slot(get_current_epoch(state)) + current_epoch_start_slot = spec.get_epoch_start_slot(spec.get_current_epoch(state)) if slot < current_epoch_start_slot: - epoch_boundary_root = get_block_root(state, get_previous_epoch(state)) + epoch_boundary_root = spec.get_block_root(state, spec.get_previous_epoch(state)) elif slot == current_epoch_start_slot: epoch_boundary_root = block_root else: - epoch_boundary_root = get_block_root(state, get_current_epoch(state)) + epoch_boundary_root = spec.get_block_root(state, spec.get_current_epoch(state)) if slot < current_epoch_start_slot: justified_epoch = state.previous_justified_epoch @@ -177,31 +137,31 @@ def build_attestation_data(state, slot, shard): justified_epoch = state.current_justified_epoch justified_block_root = state.current_justified_root - crosslinks = state.current_crosslinks if slot_to_epoch(slot) == get_current_epoch(state) else state.previous_crosslinks - return AttestationData( + crosslinks = state.current_crosslinks if spec.slot_to_epoch(slot) == spec.get_current_epoch(state) else state.previous_crosslinks + return spec.AttestationData( beacon_block_root=block_root, source_epoch=justified_epoch, source_root=justified_block_root, - target_epoch=slot_to_epoch(slot), + target_epoch=spec.slot_to_epoch(slot), target_root=epoch_boundary_root, - crosslink=Crosslink( + crosslink=spec.Crosslink( shard=shard, - epoch=min(slot_to_epoch(slot), crosslinks[shard].epoch + MAX_EPOCHS_PER_CROSSLINK), + epoch=min(spec.slot_to_epoch(slot), crosslinks[shard].epoch + spec.MAX_EPOCHS_PER_CROSSLINK), data_root=spec.ZERO_HASH, - parent_root=hash_tree_root(crosslinks[shard]), + parent_root=spec.hash_tree_root(crosslinks[shard]), ), ) def build_voluntary_exit(state, epoch, validator_index, privkey): - voluntary_exit = VoluntaryExit( + voluntary_exit = spec.VoluntaryExit( epoch=epoch, validator_index=validator_index, ) voluntary_exit.signature = bls.sign( message_hash=signing_root(voluntary_exit), privkey=privkey, - domain=get_domain( + domain=spec.get_domain( state=state, domain_type=spec.DOMAIN_VOLUNTARY_EXIT, message_epoch=epoch, @@ -224,9 +184,9 @@ def build_deposit(state, tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) root = get_merkle_root((tuple(deposit_data_leaves))) proof = list(get_merkle_proof(tree, item_index=index)) - assert verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) + assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) - deposit = Deposit( + deposit = spec.Deposit( proof=list(proof), index=index, data=deposit_data, @@ -236,22 +196,22 @@ def build_deposit(state, def get_valid_proposer_slashing(state): - current_epoch = get_current_epoch(state) - validator_index = get_active_validator_indices(state, current_epoch)[-1] + current_epoch = spec.get_current_epoch(state) + validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] slot = state.slot - header_1 = BeaconBlockHeader( + header_1 = spec.BeaconBlockHeader( slot=slot, - parent_root=ZERO_HASH, - state_root=ZERO_HASH, - body_root=ZERO_HASH, + parent_root=spec.ZERO_HASH, + state_root=spec.ZERO_HASH, + body_root=spec.ZERO_HASH, ) header_2 = deepcopy(header_1) header_2.parent_root = b'\x02' * 32 header_2.slot = slot + 1 - domain = get_domain( + domain = spec.get_domain( state=state, domain_type=spec.DOMAIN_BEACON_PROPOSER, ) @@ -266,7 +226,7 @@ def get_valid_proposer_slashing(state): domain=domain, ) - return ProposerSlashing( + return spec.ProposerSlashing( proposer_index=validator_index, header_1=header_1, header_2=header_2, @@ -278,9 +238,9 @@ def get_valid_attester_slashing(state): attestation_2 = deepcopy(attestation_1) attestation_2.data.target_root = b'\x01' * 32 - return AttesterSlashing( - attestation_1=convert_to_indexed(state, attestation_1), - attestation_2=convert_to_indexed(state, attestation_2), + return spec.AttesterSlashing( + attestation_1=spec.convert_to_indexed(state, attestation_1), + attestation_2=spec.convert_to_indexed(state, attestation_2), ) @@ -288,26 +248,26 @@ def get_valid_attestation(state, slot=None): if slot is None: slot = state.slot - if slot_to_epoch(slot) == get_current_epoch(state): + if spec.slot_to_epoch(slot) == spec.get_current_epoch(state): shard = (state.latest_start_shard + slot) % spec.SLOTS_PER_EPOCH else: - previous_shard_delta = get_shard_delta(state, get_previous_epoch(state)) + previous_shard_delta = spec.get_shard_delta(state, spec.get_previous_epoch(state)) shard = (state.latest_start_shard - previous_shard_delta + slot) % spec.SHARD_COUNT attestation_data = build_attestation_data(state, slot, shard) - crosslink_committee = get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) + crosslink_committee = spec.get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) committee_size = len(crosslink_committee) bitfield_length = (committee_size + 7) // 8 aggregation_bitfield = b'\xC0' + b'\x00' * (bitfield_length - 1) custody_bitfield = b'\x00' * bitfield_length - attestation = Attestation( + attestation = spec.Attestation( aggregation_bitfield=aggregation_bitfield, data=attestation_data, custody_bitfield=custody_bitfield, ) - participants = get_attesting_indices( + participants = spec.get_attesting_indices( state, attestation.data, attestation.aggregation_bitfield, @@ -332,10 +292,10 @@ def get_valid_attestation(state, slot=None): def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=None): if slot is None: slot = state.slot - current_epoch = get_current_epoch(state) + current_epoch = spec.get_current_epoch(state) if sender_index is None: - sender_index = get_active_validator_indices(state, current_epoch)[-1] - recipient_index = get_active_validator_indices(state, current_epoch)[0] + sender_index = spec.get_active_validator_indices(state, current_epoch)[-1] + recipient_index = spec.get_active_validator_indices(state, current_epoch)[0] transfer_pubkey = pubkeys[-1] transfer_privkey = privkeys[-1] @@ -344,22 +304,22 @@ def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=Non if amount is None: amount = get_balance(state, sender_index) - fee - transfer = Transfer( + transfer = spec.Transfer( sender=sender_index, recipient=recipient_index, amount=amount, fee=fee, slot=slot, pubkey=transfer_pubkey, - signature=ZERO_HASH, + signature=spec.ZERO_HASH, ) transfer.signature = bls.sign( message_hash=signing_root(transfer), privkey=transfer_privkey, - domain=get_domain( + domain=spec.get_domain( state=state, domain_type=spec.DOMAIN_TRANSFER, - message_epoch=get_current_epoch(state), + message_epoch=spec.get_current_epoch(state), ) ) @@ -372,7 +332,7 @@ def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=Non def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0): - message_hash = AttestationDataAndCustodyBit( + message_hash = spec.AttestationDataAndCustodyBit( data=attestation_data, custody_bit=custody_bit, ).hash_tree_root() @@ -380,7 +340,7 @@ def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0) return bls.sign( message_hash=message_hash, privkey=privkey, - domain=get_domain( + domain=spec.get_domain( state=state, domain_type=spec.DOMAIN_ATTESTATION, message_epoch=attestation_data.target_epoch, @@ -389,7 +349,7 @@ def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0) def fill_aggregate_attestation(state, attestation): - crosslink_committee = get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.crosslink.shard) + crosslink_committee = spec.get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.crosslink.shard) for i in range(len(crosslink_committee)): attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i) @@ -398,7 +358,7 @@ def add_attestation_to_state(state, attestation, slot): block = build_empty_block_for_next_slot(state) block.slot = slot block.body.attestations.append(attestation) - state_transition(state, block) + spec.state_transition(state, block) def next_slot(state): @@ -407,7 +367,7 @@ def next_slot(state): Return the empty block that triggered the transition. """ block = build_empty_block_for_next_slot(state) - state_transition(state, block) + spec.state_transition(state, block) return block @@ -418,7 +378,7 @@ def next_epoch(state): """ block = build_empty_block_for_next_slot(state) block.slot += spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH) - state_transition(state, block) + spec.state_transition(state, block) return block diff --git a/test_libs/pyspec/tests/phase0/test_finality.py b/test_libs/pyspec/tests/phase0/test_finality.py index be8ecaa1db..9e2fc361bf 100644 --- a/test_libs/pyspec/tests/phase0/test_finality.py +++ b/test_libs/pyspec/tests/phase0/test_finality.py @@ -42,7 +42,7 @@ def next_epoch_with_attestations(state, block = helpers.build_empty_block_for_next_slot(post_state) if fill_cur_epoch: slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 - if slot_to_attest >= helpers.get_epoch_start_slot(helpers.get_current_epoch(post_state)): + if slot_to_attest >= spec.get_epoch_start_slot(spec.get_current_epoch(post_state)): cur_attestation = helpers.get_valid_attestation(post_state, slot_to_attest) helpers.fill_aggregate_attestation(post_state, cur_attestation) block.body.attestations.append(cur_attestation) diff --git a/test_libs/pyspec/tests/phase1/conftest.py b/test_libs/pyspec/tests/phase1/conftest.py index 62dd2be385..53976aeebb 100644 --- a/test_libs/pyspec/tests/phase1/conftest.py +++ b/test_libs/pyspec/tests/phase1/conftest.py @@ -3,27 +3,35 @@ from eth2spec.phase1 import spec from preset_loader import loader +from tests.phase0 import helpers as phase1_helpers from tests.phase1 import helpers as helpers from tests.phase0.conftest import ( - pytest_addoption, deposit_data_leaves, ) -# This is redfined so that the constants are re-applied +def pytest_addoption(parser): + parser.addoption( + "--config", action="store", default="minimal", help="config: make the pyspec use the specified configuration" + ) + + @pytest.fixture(autouse=True) def config(request): - request.function.__globals__['spec'] = spec - request.function.__globals__['helpers'] = helpers config_name = request.config.getoption("--config") presets = loader.load_presets('../../configs/', config_name) spec.apply_constants_preset(presets) + helpers.spec = spec + phase1_helpers.spec = spec + request.function.__globals__['spec'] = spec + request.function.__globals__['helpers'] = helpers + @pytest.fixture def num_validators(config): return spec.SLOTS_PER_EPOCH * 8 -#This is redefined so that the BeaconState is the new SSZ Object + @pytest.fixture def state(num_validators, deposit_data_leaves): return helpers.create_genesis_state(num_validators, deposit_data_leaves) diff --git a/test_libs/pyspec/tests/phase1/helpers.py b/test_libs/pyspec/tests/phase1/helpers.py index b562666177..5962e45d13 100644 --- a/test_libs/pyspec/tests/phase1/helpers.py +++ b/test_libs/pyspec/tests/phase1/helpers.py @@ -1,48 +1,35 @@ from py_ecc import bls -import eth2spec.phase1.spec as spec -from eth2spec.phase1.spec import ( - # constants - CUSTODY_PERIOD_TO_RANDAO_PADDING, - # SSZ - EarlyDerivedSecretReveal, - # functions - get_active_validator_indices, - get_current_epoch, - get_domain, - hash_tree_root, -) - from tests.phase0.helpers import * def get_valid_early_derived_secret_reveal(state, epoch=None): - current_epoch = get_current_epoch(state) - revealed_index = get_active_validator_indices(state, current_epoch)[-1] - masker_index = get_active_validator_indices(state, current_epoch)[0] + current_epoch = spec.get_current_epoch(state) + revealed_index = spec.get_active_validator_indices(state, current_epoch)[-1] + masker_index = spec.get_active_validator_indices(state, current_epoch)[0] if epoch is None: - epoch = current_epoch + CUSTODY_PERIOD_TO_RANDAO_PADDING + epoch = current_epoch + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING reveal = bls.sign( - message_hash=hash_tree_root(epoch), + message_hash=spec.hash_tree_root(epoch), privkey=privkeys[revealed_index], - domain=get_domain( + domain=spec.get_domain( state=state, domain_type=spec.DOMAIN_RANDAO, message_epoch=epoch, ), ) mask = bls.sign( - message_hash=hash_tree_root(epoch), + message_hash=spec.hash_tree_root(epoch), privkey=privkeys[masker_index], - domain=get_domain( + domain=spec.get_domain( state=state, domain_type=spec.DOMAIN_RANDAO, message_epoch=epoch, ), ) - return EarlyDerivedSecretReveal( + return spec.EarlyDerivedSecretReveal( revealed_index=revealed_index, epoch=epoch, reveal=reveal, From 90a3f56e0feedd491e0c835ab33114e4d8a6d660 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 21 May 2019 09:13:57 +0100 Subject: [PATCH 155/308] Edit BLS spec warning Fix #898. --- specs/bls_signature.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/specs/bls_signature.md b/specs/bls_signature.md index d119c4499c..3fe1bcc0eb 100644 --- a/specs/bls_signature.md +++ b/specs/bls_signature.md @@ -1,6 +1,8 @@ # BLS signature verification -**Warning: This document is pending academic review and should not yet be considered secure.** +**Notice**: This document is a placeholder to facilitate the emergence of cross-client testnets. Substantive changes are postponed until [BLS standardisation](https://github.com/pairingwg/bls_standard) is finalized. + +**Warning**: The constructions in this document should not be considered secure. In particular, the `hash_to_G2` function is known to be unsecure. ## Table of contents From 24c4d21d5ef00060cb57cc5cc1b532adae34625d Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 21 May 2019 12:41:24 +0200 Subject: [PATCH 156/308] Minor edits + PEP8 --- Makefile | 4 +- scripts/build_spec.py | 1 - specs/core/0_beacon-chain.md | 6 +- specs/core/1_custody-game.md | 25 +++++--- specs/core/1_shard-data-chains.md | 2 +- .../eth2spec/phase1/state_transition.py | 4 +- .../pyspec/eth2spec/utils/minimal_ssz.py | 2 - .../block_processing/test_process_transfer.py | 8 ++- test_libs/pyspec/tests/phase0/conftest.py | 6 +- .../test_process_crosslinks.py | 5 +- test_libs/pyspec/tests/phase0/helpers.py | 21 +++++-- test_libs/pyspec/tests/phase0/test_sanity.py | 1 - ...est_process_early_derived_secret_reveal.py | 60 ++++++++++++------- test_libs/pyspec/tests/phase1/conftest.py | 3 +- test_libs/pyspec/tests/phase1/helpers.py | 1 + 15 files changed, 97 insertions(+), 52 deletions(-) diff --git a/Makefile b/Makefile index 041916fabc..443266bb42 100644 --- a/Makefile +++ b/Makefile @@ -52,11 +52,11 @@ citest: $(PY_SPEC_ALL_TARGETS) python -m pytest --junitxml=test-reports/eth2spec/test_results_phase0.xml tests/phase1 install_lint: - cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.5.0 + cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.7.0 lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ - flake8 --max-line-length=120 ./eth2spec; + flake8 --max-line-length=120 --per-file-ignores='./tests/*:F821,F403,F401,F405' ./eth2spec ./tests; # "make pyspec" to create the pyspec for all phases. pyspec: $(PY_SPEC_ALL_TARGETS) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 9e1d0436a5..a1f3794daa 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -50,7 +50,6 @@ ) from eth2spec.utils.hash_function import hash -import math ''' NEW_TYPES = { 'Slot': 'int', diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f69795926e..281c3d9ee7 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -962,7 +962,7 @@ def get_total_balance(state: BeaconState, indices: List[ValidatorIndex]) -> Gwei ```python def get_domain(state: BeaconState, domain_type: int, - message_epoch: int=None) -> int: + message_epoch: int = None) -> int: """ Return the signature domain (fork version concatenated with domain type) of a message. """ @@ -1150,7 +1150,7 @@ def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: ```python def slash_validator(state: BeaconState, slashed_index: ValidatorIndex, - whistleblower_index: ValidatorIndex=None) -> None: + whistleblower_index: ValidatorIndex = None) -> None: """ Slash the validator with index ``slashed_index``. """ @@ -1223,7 +1223,7 @@ Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled excpetion (e.g. a failed `assert` or an out-of-range list access) are considered invalid. ```python -def state_transition(state: BeaconState, block: BeaconBlock, validate_state_root: bool=False) -> BeaconState: +def state_transition(state: BeaconState, block: BeaconBlock, validate_state_root: bool = False) -> BeaconState: # Process slots (including those with no blocks) since block process_slots(state, block.slot) # Process block diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 9831c52ece..ee9867007b 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -33,6 +33,7 @@ - [Helpers](#helpers) - [`type_of`](#type_of) - [`empty`](#empty) + - [`ceillog2`](#ceillog2) - [`get_crosslink_chunk_count`](#get_crosslink_chunk_count) - [`get_custody_chunk_bit`](#get_custody_chunk_bit) - [`get_chunk_bits_root`](#get_chunk_bits_root) @@ -126,7 +127,7 @@ This document details the beacon chain additions and changes in Phase 1 of Ether 'attestation': Attestation, 'challenger_index': ValidatorIndex, 'responder_key': BLSSignature, - 'chunk_bits': "bytes", + 'chunk_bits': 'bytes', 'signature': BLSSignature, } ``` @@ -257,6 +258,13 @@ The `type_of` function accepts an SSZ object as a single input and returns the c The `empty` function accepts an SSZ type as input and returns an object of that type with all fields initialized to default values. +### `ceillog2` + +```python +def ceillog2(x): + return x.bit_length() +``` + ### `get_crosslink_chunk_count` ```python @@ -298,7 +306,7 @@ def get_randao_epoch_for_custody_period(period: int, validator_index: ValidatorI ```python def get_validators_custody_reveal_period(state: BeaconState, validator_index: ValidatorIndex, - epoch: Epoch=None) -> int: + epoch: Epoch = None) -> int: ''' This function returns the reveal period for a given validator. If no epoch is supplied, the current epoch is assumed. @@ -479,7 +487,7 @@ def process_chunk_challenge(state: BeaconState, record.chunk_index != challenge.chunk_index ) # Verify depth - depth = math.ceil(math.log2(get_custody_chunk_count(challenge.attestation.data.crosslink))) + depth = ceillog2(get_custody_chunk_count(challenge.attestation.data.crosslink)) assert challenge.chunk_index < 2**depth # Add new chunk challenge record new_record = CustodyChunkChallengeRecord( @@ -637,7 +645,7 @@ def process_bit_challenge_response(state: BeaconState, assert verify_merkle_branch( leaf=hash_tree_root(response.chunk), branch=response.data_branch, - depth=math.ceil(math.log2(challenge.chunk_count)), + depth=ceillog2(challenge.chunk_count), index=response.chunk_index, root=challenge.data_root, ) @@ -645,7 +653,7 @@ def process_bit_challenge_response(state: BeaconState, assert verify_merkle_branch( leaf=response.chunk_bits_leaf, branch=response.chunk_bits_branch, - depth=math.ceil(math.log2(challenge.chunk_count)) >> 8, + depth=ceillog2(challenge.chunk_count) >> 8, index=response.chunk_index // 256, root=challenge.chunk_bits_merkle_root ) @@ -668,10 +676,9 @@ Run `process_reveal_deadlines(state)` immediately after `process_registry_update ```python def process_reveal_deadlines(state: BeaconState) -> None: for index, validator in enumerate(state.validator_registry): - if (validator.next_custody_reveal_period + - (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) < - get_validators_custody_reveal_period(state, index)): - slash_validator(state, index) + if (validator.next_custody_reveal_period + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) + < get_validators_custody_reveal_period(state, index)): + slash_validator(state, index) ``` Run `process_challenge_deadlines(state)` immediately after `process_reveal_deadlines(state)`: diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 8619354e4c..7cb4bd78bc 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -236,7 +236,7 @@ def verify_shard_attestation_signature(state: BeaconState, ### `compute_crosslink_data_root` ```python -def compute_crosslink_data_root(blocks: List[ShardBlock]) -> 'bytes32': +def compute_crosslink_data_root(blocks: List[ShardBlock]) -> Bytes32: def is_power_of_two(value: int) -> bool: return (value > 0) and (value & (value - 1) == 0) diff --git a/test_libs/pyspec/eth2spec/phase1/state_transition.py b/test_libs/pyspec/eth2spec/phase1/state_transition.py index 87b2e1c12f..a2a78d2738 100644 --- a/test_libs/pyspec/eth2spec/phase1/state_transition.py +++ b/test_libs/pyspec/eth2spec/phase1/state_transition.py @@ -32,7 +32,7 @@ def process_operations(state: BeaconState, block: BeaconBlock) -> None: def process_block(state: BeaconState, block: BeaconBlock, - verify_state_root: bool=False) -> None: + verify_state_root: bool = False) -> None: spec.process_block_header(state, block) spec.process_randao(state, block) spec.process_eth1_data(state, block) @@ -65,6 +65,6 @@ def state_transition_to(state: BeaconState, up_to: Slot) -> BeaconState: def state_transition(state: BeaconState, block: BeaconBlock, - verify_state_root: bool=False) -> BeaconState: + verify_state_root: bool = False) -> BeaconState: state_transition_to(state, block.slot) process_block(state, block, verify_state_root) diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py index 63516780e7..f46c0acef0 100644 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py @@ -6,8 +6,6 @@ BYTES_PER_LENGTH_OFFSET = 4 ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK -cached_typedefs = {} - def SSZType(fields): class SSZObject(): diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py index 724c696478..2d37798e7d 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py @@ -18,7 +18,6 @@ def run_transfer_processing(state, transfer, valid=True): spec.process_transfer(post_state, transfer) return state, None - spec.process_transfer(post_state, transfer) proposer_index = spec.get_beacon_proposer_index(state) @@ -106,7 +105,12 @@ def test_insufficient_balance(state): def test_no_dust(state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] balance = state.balances[sender_index] - transfer = helpers.get_valid_transfer(state, sender_index=sender_index, amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, fee=0) + transfer = helpers.get_valid_transfer( + state, + sender_index=sender_index, + amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, + fee=0 + ) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH diff --git a/test_libs/pyspec/tests/phase0/conftest.py b/test_libs/pyspec/tests/phase0/conftest.py index 0549a1422e..67dc34fa43 100644 --- a/test_libs/pyspec/tests/phase0/conftest.py +++ b/test_libs/pyspec/tests/phase0/conftest.py @@ -4,13 +4,14 @@ from preset_loader import loader from tests.phase0 import helpers -from tests.phase0 import test_finality + def pytest_addoption(parser): parser.addoption( "--config", action="store", default="minimal", help="config: make the pyspec use the specified configuration" ) + @pytest.fixture(autouse=True) def config(request): config_name = request.config.getoption("--config") @@ -20,14 +21,17 @@ def config(request): request.function.__globals__['spec'] = spec request.function.__globals__['helpers'] = helpers + @pytest.fixture def num_validators(config): return spec.SLOTS_PER_EPOCH * 8 + @pytest.fixture def deposit_data_leaves(): return list() + @pytest.fixture def state(num_validators, deposit_data_leaves): return helpers.create_genesis_state(num_validators, deposit_data_leaves) diff --git a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py index 19c668e749..e4913734cd 100644 --- a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py @@ -108,7 +108,10 @@ def test_double_late_crosslink(state): # ensure that the current crosslinks were not updated by the second attestation assert post_state.previous_crosslinks[shard] == post_state.current_crosslinks[shard] # ensure no reward, only penalties for the failed crosslink - for index in spec.get_crosslink_committee(state, attestation_2.data.target_epoch, attestation_2.data.crosslink.shard): + for index in spec.get_crosslink_committee( + state, + attestation_2.data.target_epoch, + attestation_2.data.crosslink.shard): assert crosslink_deltas[0][index] == 0 assert crosslink_deltas[1][index] > 0 diff --git a/test_libs/pyspec/tests/phase0/helpers.py b/test_libs/pyspec/tests/phase0/helpers.py index c73277f961..d41ddd59d1 100644 --- a/test_libs/pyspec/tests/phase0/helpers.py +++ b/test_libs/pyspec/tests/phase0/helpers.py @@ -14,6 +14,7 @@ pubkeys = [bls.privtopub(privkey) for privkey in privkeys] pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} + def advance_slot(state) -> None: state.slot += 1 @@ -137,7 +138,10 @@ def build_attestation_data(state, slot, shard): justified_epoch = state.current_justified_epoch justified_block_root = state.current_justified_root - crosslinks = state.current_crosslinks if spec.slot_to_epoch(slot) == spec.get_current_epoch(state) else state.previous_crosslinks + crosslinks = ( + state.current_crosslinks if spec.slot_to_epoch(slot) == spec.get_current_epoch(state) + else state.previous_crosslinks + ) parent_crosslink = crosslinks[shard] return spec.AttestationData( beacon_block_root=block_root, @@ -258,7 +262,11 @@ def get_valid_attestation(state, slot=None): attestation_data = build_attestation_data(state, slot, shard) - crosslink_committee = spec.get_crosslink_committee(state, attestation_data.target_epoch, attestation_data.crosslink.shard) + crosslink_committee = spec.get_crosslink_committee( + state, + attestation_data.target_epoch, + attestation_data.crosslink.shard + ) committee_size = len(crosslink_committee) bitfield_length = (committee_size + 7) // 8 @@ -351,7 +359,11 @@ def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0) def fill_aggregate_attestation(state, attestation): - crosslink_committee = spec.get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.crosslink.shard) + crosslink_committee = spec.get_crosslink_committee( + state, + attestation.data.target_epoch, + attestation.data.crosslink.shard + ) for i in range(len(crosslink_committee)): attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i) @@ -390,6 +402,3 @@ def get_state_root(state, slot) -> bytes: """ assert slot < state.slot <= slot + spec.SLOTS_PER_HISTORICAL_ROOT return state.latest_state_roots[slot % spec.SLOTS_PER_HISTORICAL_ROOT] - -# Stub to be overwritten by config -import_spec = None \ No newline at end of file diff --git a/test_libs/pyspec/tests/phase0/test_sanity.py b/test_libs/pyspec/tests/phase0/test_sanity.py index 821d973866..47e4917ab0 100644 --- a/test_libs/pyspec/tests/phase0/test_sanity.py +++ b/test_libs/pyspec/tests/phase0/test_sanity.py @@ -227,7 +227,6 @@ def test_attestation(state): assert len(test_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1 - # # Epoch transition should move to previous_epoch_attestations # diff --git a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py index 08f7cf9ac7..75c44697c9 100644 --- a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py @@ -1,10 +1,11 @@ from copy import deepcopy import pytest -#mark entire file as 'randao_key_reveals' +# mark entire file as 'randao_key_reveals' pytestmark = pytest.mark.randao_key_reveals -def terun_early_derived_secret_reveal_processing(state, randao_key_reveal, valid=True): + +def run_early_derived_secret_reveal_processing(state, randao_key_reveal, valid=True): """ Run ``process_randao_key_reveal`` returning the pre and post state. If ``valid == False``, run expecting ``AssertionError`` @@ -37,7 +38,7 @@ def terun_early_derived_secret_reveal_processing(state, randao_key_reveal, valid def test_success(state): randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state) - pre_state, post_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal) + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal) return pre_state, randao_key_reveal, post_state @@ -45,52 +46,71 @@ def test_success(state): def test_reveal_from_current_epoch(state): randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state)) - pre_state, post_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) return pre_state, randao_key_reveal, post_state @pytest.mark.skip(reason="Not currently possible as we are testing at epoch 0") def test_reveal_from_past_epoch(state): - randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) - 1) + randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) - 1) + + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) - pre_state, post_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + return pre_state, randao_key_reveal, post_state - return pre_state, randao_key_reveal, post_state def test_reveal_with_custody_padding(state): - randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING) - pre_state, post_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal, True) + randao_key_reveal = helpers.get_valid_early_derived_secret_reveal( + state, + spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING + ) + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) return pre_state, randao_key_reveal, post_state + def test_reveal_with_custody_padding_minus_one(state): - randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING - 1) - pre_state, post_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal, True) + randao_key_reveal = helpers.get_valid_early_derived_secret_reveal( + state, + spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING - 1 + ) + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) return pre_state, randao_key_reveal, post_state + def test_double_reveal(state): - - randao_key_reveal1 = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) + spec.RANDAO_PENALTY_EPOCHS + 1) - pre_state, intermediate_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal1) - - randao_key_reveal2 = helpers.get_valid_early_derived_secret_reveal(intermediate_state, spec.get_current_epoch(pre_state) + spec.RANDAO_PENALTY_EPOCHS + 1) - _, post_state = terun_early_derived_secret_reveal_processing(intermediate_state, randao_key_reveal2, False) + randao_key_reveal1 = helpers.get_valid_early_derived_secret_reveal( + state, + spec.get_current_epoch(state) + spec.RANDAO_PENALTY_EPOCHS + 1 + ) + pre_state, intermediate_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal1) + + randao_key_reveal2 = helpers.get_valid_early_derived_secret_reveal( + intermediate_state, + spec.get_current_epoch(pre_state) + spec.RANDAO_PENALTY_EPOCHS + 1 + ) + _, post_state = run_early_derived_secret_reveal_processing(intermediate_state, randao_key_reveal2, False) return pre_state, [randao_key_reveal1, randao_key_reveal2], post_state + def test_revealer_is_slashed(state): randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state)) state.validator_registry[randao_key_reveal.revealed_index].slashed = True - pre_state, post_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) return pre_state, randao_key_reveal, post_state + def test_far_future_epoch(state): - randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) + spec.EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS) + randao_key_reveal = helpers.get_valid_early_derived_secret_reveal( + state, + spec.get_current_epoch(state) + spec.EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS + ) - pre_state, post_state = terun_early_derived_secret_reveal_processing(state, randao_key_reveal, False) + pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) return pre_state, randao_key_reveal, post_state diff --git a/test_libs/pyspec/tests/phase1/conftest.py b/test_libs/pyspec/tests/phase1/conftest.py index 53976aeebb..0e634cbf43 100644 --- a/test_libs/pyspec/tests/phase1/conftest.py +++ b/test_libs/pyspec/tests/phase1/conftest.py @@ -10,6 +10,7 @@ deposit_data_leaves, ) + def pytest_addoption(parser): parser.addoption( "--config", action="store", default="minimal", help="config: make the pyspec use the specified configuration" @@ -33,5 +34,5 @@ def num_validators(config): @pytest.fixture -def state(num_validators, deposit_data_leaves): +def state(num_validators, deposit_data_leaves): # noqa: F811 return helpers.create_genesis_state(num_validators, deposit_data_leaves) diff --git a/test_libs/pyspec/tests/phase1/helpers.py b/test_libs/pyspec/tests/phase1/helpers.py index 5962e45d13..d7adbd8027 100644 --- a/test_libs/pyspec/tests/phase1/helpers.py +++ b/test_libs/pyspec/tests/phase1/helpers.py @@ -2,6 +2,7 @@ from tests.phase0.helpers import * + def get_valid_early_derived_secret_reveal(state, epoch=None): current_epoch = spec.get_current_epoch(state) revealed_index = spec.get_active_validator_indices(state, current_epoch)[-1] From 847fcf52cc5228cb32998feffb05511b2312d7b8 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 21 May 2019 11:30:38 -0600 Subject: [PATCH 157/308] utilize bls_domain directly for deposits --- specs/core/0_beacon-chain.md | 20 ++++++++++++++++---- specs/validator/0_beacon-chain-validator.md | 2 +- test_libs/pyspec/tests/helpers.py | 5 +++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f1986989b6..79038eea80 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -174,6 +174,7 @@ These configurations are updated for releases, but may be out of sync during `de | Name | Value | | - | - | | `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | +| `DEPOSIT_FORK_VERSION` | `b'\x00' * 4` | ### Gwei values @@ -629,6 +630,16 @@ The `hash` function is SHA256. `def signing_root(object: SSZContainer) -> Bytes32` is a function defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers) to compute signing messages. +### `bls_domain` + +```python +def bls_domain(domain_type: int, fork_version: bytes) -> int: + """ + Return the bls domain given by the ``domain_type`` and 4 byte ``fork_version``.. + """ + return bytes_to_int(int_to_bytes(domain_type, length=4) + fork_version) +``` + ### `slot_to_epoch` ```python @@ -968,8 +979,7 @@ def get_domain(state: BeaconState, """ epoch = get_current_epoch(state) if message_epoch is None else message_epoch fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version - # fork version is on the big-endian side: when signing using only the type (e.g. deposits), the type can be passed directly. - return bytes_to_int(int_to_bytes(domain_type, length=4) + fork_version) + return bls_domain(domain_type, fork_version) ``` ### `get_bitfield_bit` @@ -1766,8 +1776,10 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: validator_pubkeys = [v.pubkey for v in state.validator_registry] if pubkey not in validator_pubkeys: # Verify the deposit signature (proof of possession) - # Note: deposits are valid regardless of fork version, hence the type is passed directly as domain. - if not bls_verify(pubkey, signing_root(deposit.data), deposit.data.signature, DOMAIN_DEPOSIT): + # Note: deposits are valid across forks, hence the deposit domain is retrieved directly from `bls_domain` + if not bls_verify( + pubkey, signing_root(deposit.data), deposit.data.signature, bls_domain(DOMAIN_DEPOSIT, DEPOSIT_FORK_VERSION) + ): return # Add validator and balance entries diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index c2fe927c02..49bb4fc3ac 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -98,7 +98,7 @@ To submit a deposit: * Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](../core/0_beacon-chain.md#depositdata) SSZ object. * Let `amount` be the amount in Gwei to be deposited by the validator where `MIN_DEPOSIT_AMOUNT <= amount <= MAX_EFFECTIVE_BALANCE`. * Set `deposit_data.amount = amount`. -* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=DOMAIN_DEPOSIT`. (Deposits are valid regardless of fork version, hence the type is passed directly as domain.) +* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=bls_domain(DOMAIN_DEPOSIT, DEPOSIT_FORK_VERSION)`. (Deposits are valid regardless of fork version, hence the static fork version being directly passed into `bls_domain`). * Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])` along with a deposit of `amount` Gwei. *Note*: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validator_registry` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 7af210f85a..eebbd2daf8 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -24,6 +24,7 @@ VoluntaryExit, # functions convert_to_indexed, + bls_domain, get_active_validator_indices, get_attesting_indices, get_block_root, @@ -144,9 +145,9 @@ def build_deposit_data(state, pubkey, privkey, amount): signature = bls.sign( message_hash=signing_root(deposit_data), privkey=privkey, - domain=get_domain( - state, + domain=bls_domain( spec.DOMAIN_DEPOSIT, + spec.DEPOSIT_FORK_VERSION, ) ) deposit_data.signature = signature From b075a7a0ab3e85b40716c3bf23cbaae76753fbb3 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 21 May 2019 11:33:52 -0600 Subject: [PATCH 158/308] add bls_domain to toc --- specs/core/0_beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 79038eea80..9f965c1d7e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -51,6 +51,7 @@ - [`hash`](#hash) - [`hash_tree_root`](#hash_tree_root) - [`signing_root`](#signing_root) + - [`bls_domain`](#bls_domain) - [`slot_to_epoch`](#slot_to_epoch) - [`get_previous_epoch`](#get_previous_epoch) - [`get_current_epoch`](#get_current_epoch) From 4ce441c4a45ee8273aaa94e5b6c65b1772ee6942 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 21 May 2019 12:54:38 -0600 Subject: [PATCH 159/308] minor fix to how eth1 data is counted in validator guide --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index d05f25ef29..d82bf6e560 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -233,7 +233,7 @@ epoch_signature = bls_sign( * Let `deposit_root` and `deposit_count` be the deposit root and deposit count of the Eth 1.0 deposit contract in the post-state of the block referenced by `block_hash` * Let `best_vote_data = Eth1Data(block_hash=block_hash, deposit_root=deposit_root, deposit_count=deposit_count)`. * If `D` is nonempty: - * Let `best_vote_data` be the `eth1_data` of the member of `D` that has the highest `vote.vote_count`, breaking ties by favoring block hashes with higher associated block height. + * Let `best_vote_data` be the `eth1_data` member of `D` that has the highest vote count (`D.count(eth1_data)`), breaking ties by favoring block hashes with higher associated block height. * Set `block.eth1_data = best_vote_data`. ##### Signature From 6b5f4b44eae5d3b1ba7b403eed8954aff5c96097 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 22 May 2019 01:35:24 +0200 Subject: [PATCH 160/308] avoid zero constant for deposits fork-version, just default to it --- specs/core/0_beacon-chain.md | 7 +++---- specs/validator/0_beacon-chain-validator.md | 2 +- test_libs/pyspec/tests/helpers.py | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9f965c1d7e..c14228dc11 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -175,7 +175,6 @@ These configurations are updated for releases, but may be out of sync during `de | Name | Value | | - | - | | `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | -| `DEPOSIT_FORK_VERSION` | `b'\x00' * 4` | ### Gwei values @@ -634,9 +633,9 @@ The `hash` function is SHA256. ### `bls_domain` ```python -def bls_domain(domain_type: int, fork_version: bytes) -> int: +def bls_domain(domain_type: int, fork_version=b'\x00\x00\x00\x00') -> int: """ - Return the bls domain given by the ``domain_type`` and 4 byte ``fork_version``.. + Return the bls domain given by the ``domain_type`` and optional 4 byte ``fork_version`` (defaults to zero). """ return bytes_to_int(int_to_bytes(domain_type, length=4) + fork_version) ``` @@ -1779,7 +1778,7 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: # Verify the deposit signature (proof of possession) # Note: deposits are valid across forks, hence the deposit domain is retrieved directly from `bls_domain` if not bls_verify( - pubkey, signing_root(deposit.data), deposit.data.signature, bls_domain(DOMAIN_DEPOSIT, DEPOSIT_FORK_VERSION) + pubkey, signing_root(deposit.data), deposit.data.signature, bls_domain(DOMAIN_DEPOSIT) ): return diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 49bb4fc3ac..f8272d4464 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -98,7 +98,7 @@ To submit a deposit: * Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](../core/0_beacon-chain.md#depositdata) SSZ object. * Let `amount` be the amount in Gwei to be deposited by the validator where `MIN_DEPOSIT_AMOUNT <= amount <= MAX_EFFECTIVE_BALANCE`. * Set `deposit_data.amount = amount`. -* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=bls_domain(DOMAIN_DEPOSIT, DEPOSIT_FORK_VERSION)`. (Deposits are valid regardless of fork version, hence the static fork version being directly passed into `bls_domain`). +* Let `signature` be the result of `bls_sign` of the `signing_root(deposit_data)` with `domain=bls_domain(DOMAIN_DEPOSIT)`. (Deposits are valid regardless of fork version, `bls_domain` will default to zeroes there). * Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96])` along with a deposit of `amount` Gwei. *Note*: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validator_registry` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index eebbd2daf8..06ca8a1d50 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -147,7 +147,6 @@ def build_deposit_data(state, pubkey, privkey, amount): privkey=privkey, domain=bls_domain( spec.DOMAIN_DEPOSIT, - spec.DEPOSIT_FORK_VERSION, ) ) deposit_data.signature = signature From 2c79e584c2b797efd4a19b1ed85c0f16dbc28d1a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 21 May 2019 22:42:47 -0600 Subject: [PATCH 161/308] minor lint --- test_libs/pyspec/tests/helpers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index 06ca8a1d50..a4849bfbbc 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -145,9 +145,7 @@ def build_deposit_data(state, pubkey, privkey, amount): signature = bls.sign( message_hash=signing_root(deposit_data), privkey=privkey, - domain=bls_domain( - spec.DOMAIN_DEPOSIT, - ) + domain=bls_domain(spec.DOMAIN_DEPOSIT), ) deposit_data.signature = signature return deposit_data From 9058647b67dbce3631d13a61c93fc5fc01e8e9aa Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Wed, 22 May 2019 10:28:03 +0200 Subject: [PATCH 162/308] flake8v3.7->flake8v3.5 --- Makefile | 5 +++-- specs/core/0_beacon-chain.md | 6 +++--- specs/core/1_custody-game.md | 2 +- test_libs/pyspec/eth2spec/phase1/state_transition.py | 4 ++-- .../phase0/block_processing/test_process_transfer.py | 4 ++-- test_libs/pyspec/tests/phase0/helpers.py | 6 +++--- .../test_process_early_derived_secret_reveal.py | 10 +++++----- test_libs/pyspec/tests/phase1/conftest.py | 11 ++++++----- 8 files changed, 25 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index 443266bb42..d700d19008 100644 --- a/Makefile +++ b/Makefile @@ -52,11 +52,12 @@ citest: $(PY_SPEC_ALL_TARGETS) python -m pytest --junitxml=test-reports/eth2spec/test_results_phase0.xml tests/phase1 install_lint: - cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.7.0 + cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.5.0 lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ - flake8 --max-line-length=120 --per-file-ignores='./tests/*:F821,F403,F401,F405' ./eth2spec ./tests; + flake8 --max-line-length=120 ./eth2spec; \ + flake8 --max-line-length=120 --ignore=F401,F403,F405,F821 ./tests; # "make pyspec" to create the pyspec for all phases. pyspec: $(PY_SPEC_ALL_TARGETS) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 281c3d9ee7..f69795926e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -962,7 +962,7 @@ def get_total_balance(state: BeaconState, indices: List[ValidatorIndex]) -> Gwei ```python def get_domain(state: BeaconState, domain_type: int, - message_epoch: int = None) -> int: + message_epoch: int=None) -> int: """ Return the signature domain (fork version concatenated with domain type) of a message. """ @@ -1150,7 +1150,7 @@ def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: ```python def slash_validator(state: BeaconState, slashed_index: ValidatorIndex, - whistleblower_index: ValidatorIndex = None) -> None: + whistleblower_index: ValidatorIndex=None) -> None: """ Slash the validator with index ``slashed_index``. """ @@ -1223,7 +1223,7 @@ Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. The post-state corresponding to a pre-state `state` and a block `block` is defined as `state_transition(state, block)`. State transitions that trigger an unhandled excpetion (e.g. a failed `assert` or an out-of-range list access) are considered invalid. ```python -def state_transition(state: BeaconState, block: BeaconBlock, validate_state_root: bool = False) -> BeaconState: +def state_transition(state: BeaconState, block: BeaconBlock, validate_state_root: bool=False) -> BeaconState: # Process slots (including those with no blocks) since block process_slots(state, block.slot) # Process block diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index ee9867007b..b96ea613a0 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -306,7 +306,7 @@ def get_randao_epoch_for_custody_period(period: int, validator_index: ValidatorI ```python def get_validators_custody_reveal_period(state: BeaconState, validator_index: ValidatorIndex, - epoch: Epoch = None) -> int: + epoch: Epoch=None) -> int: ''' This function returns the reveal period for a given validator. If no epoch is supplied, the current epoch is assumed. diff --git a/test_libs/pyspec/eth2spec/phase1/state_transition.py b/test_libs/pyspec/eth2spec/phase1/state_transition.py index a2a78d2738..87b2e1c12f 100644 --- a/test_libs/pyspec/eth2spec/phase1/state_transition.py +++ b/test_libs/pyspec/eth2spec/phase1/state_transition.py @@ -32,7 +32,7 @@ def process_operations(state: BeaconState, block: BeaconBlock) -> None: def process_block(state: BeaconState, block: BeaconBlock, - verify_state_root: bool = False) -> None: + verify_state_root: bool=False) -> None: spec.process_block_header(state, block) spec.process_randao(state, block) spec.process_eth1_data(state, block) @@ -65,6 +65,6 @@ def state_transition_to(state: BeaconState, up_to: Slot) -> BeaconState: def state_transition(state: BeaconState, block: BeaconBlock, - verify_state_root: bool = False) -> BeaconState: + verify_state_root: bool=False) -> BeaconState: state_transition_to(state, block.slot) process_block(state, block, verify_state_root) diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py index 2d37798e7d..6c57c54f00 100644 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py @@ -79,7 +79,7 @@ def test_active_but_transfer_past_effective_balance(state): def test_incorrect_slot(state): - transfer = helpers.get_valid_transfer(state, slot=state.slot+1) + transfer = helpers.get_valid_transfer(state, slot=state.slot + 1) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH @@ -109,7 +109,7 @@ def test_no_dust(state): state, sender_index=sender_index, amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, - fee=0 + fee=0, ) # un-activate so validator can transfer diff --git a/test_libs/pyspec/tests/phase0/helpers.py b/test_libs/pyspec/tests/phase0/helpers.py index d41ddd59d1..4d19739210 100644 --- a/test_libs/pyspec/tests/phase0/helpers.py +++ b/test_libs/pyspec/tests/phase0/helpers.py @@ -32,7 +32,7 @@ def set_bitfield_bit(bitfield, i): return ( bitfield[:byte_index] + bytes([bitfield[byte_index] | (1 << bit_index)]) + - bitfield[byte_index+1:] + bitfield[byte_index + 1:] ) @@ -265,7 +265,7 @@ def get_valid_attestation(state, slot=None): crosslink_committee = spec.get_crosslink_committee( state, attestation_data.target_epoch, - attestation_data.crosslink.shard + attestation_data.crosslink.shard, ) committee_size = len(crosslink_committee) @@ -362,7 +362,7 @@ def fill_aggregate_attestation(state, attestation): crosslink_committee = spec.get_crosslink_committee( state, attestation.data.target_epoch, - attestation.data.crosslink.shard + attestation.data.crosslink.shard, ) for i in range(len(crosslink_committee)): attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i) diff --git a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py index 75c44697c9..ad436ff29e 100644 --- a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py @@ -63,7 +63,7 @@ def test_reveal_from_past_epoch(state): def test_reveal_with_custody_padding(state): randao_key_reveal = helpers.get_valid_early_derived_secret_reveal( state, - spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING + spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING, ) pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) @@ -73,7 +73,7 @@ def test_reveal_with_custody_padding(state): def test_reveal_with_custody_padding_minus_one(state): randao_key_reveal = helpers.get_valid_early_derived_secret_reveal( state, - spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING - 1 + spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING - 1, ) pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) @@ -83,13 +83,13 @@ def test_reveal_with_custody_padding_minus_one(state): def test_double_reveal(state): randao_key_reveal1 = helpers.get_valid_early_derived_secret_reveal( state, - spec.get_current_epoch(state) + spec.RANDAO_PENALTY_EPOCHS + 1 + spec.get_current_epoch(state) + spec.RANDAO_PENALTY_EPOCHS + 1, ) pre_state, intermediate_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal1) randao_key_reveal2 = helpers.get_valid_early_derived_secret_reveal( intermediate_state, - spec.get_current_epoch(pre_state) + spec.RANDAO_PENALTY_EPOCHS + 1 + spec.get_current_epoch(pre_state) + spec.RANDAO_PENALTY_EPOCHS + 1, ) _, post_state = run_early_derived_secret_reveal_processing(intermediate_state, randao_key_reveal2, False) @@ -108,7 +108,7 @@ def test_revealer_is_slashed(state): def test_far_future_epoch(state): randao_key_reveal = helpers.get_valid_early_derived_secret_reveal( state, - spec.get_current_epoch(state) + spec.EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS + spec.get_current_epoch(state) + spec.EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS, ) pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) diff --git a/test_libs/pyspec/tests/phase1/conftest.py b/test_libs/pyspec/tests/phase1/conftest.py index 0e634cbf43..4226241d77 100644 --- a/test_libs/pyspec/tests/phase1/conftest.py +++ b/test_libs/pyspec/tests/phase1/conftest.py @@ -6,10 +6,6 @@ from tests.phase0 import helpers as phase1_helpers from tests.phase1 import helpers as helpers -from tests.phase0.conftest import ( - deposit_data_leaves, -) - def pytest_addoption(parser): parser.addoption( @@ -34,5 +30,10 @@ def num_validators(config): @pytest.fixture -def state(num_validators, deposit_data_leaves): # noqa: F811 +def deposit_data_leaves(): + return list() + + +@pytest.fixture +def state(num_validators, deposit_data_leaves): return helpers.create_genesis_state(num_validators, deposit_data_leaves) From 271f615873e3165e0ea345125b10ff56cf07f53e Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Wed, 22 May 2019 12:40:57 +0200 Subject: [PATCH 163/308] Dramatically speed up tests --- configs/constant_presets/minimal.yaml | 2 ++ scripts/build_spec.py | 2 +- .../{test_phase0.py => test_phase0_epoch_processing.py} | 0 3 files changed, 3 insertions(+), 1 deletion(-) rename test_libs/pyspec/tests/phase1/epoch_processing/{test_phase0.py => test_phase0_epoch_processing.py} (100%) diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index caae4623ba..73448c3c60 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -74,6 +74,8 @@ PERSISTENT_COMMITTEE_PERIOD: 2048 MAX_EPOCHS_PER_CROSSLINK: 64 # 2**2 (= 4) epochs 25.6 minutes MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 +# [customized] 2**12 (= 4,096) epochs 18 days +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 # State list lengths diff --git a/scripts/build_spec.py b/scripts/build_spec.py index a1f3794daa..7619efbe7b 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -114,7 +114,7 @@ def objects_to_spec(functions, constants, ssz_objects, imports): constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]),constants)) ssz_objects_instantiation_spec = '\n'.join(map(lambda x: '%s = SSZType(%s)' % (x, ssz_objects[x][:-1]), ssz_objects)) ssz_objects_reinitialization_spec = '\n'.join( - map(lambda x: ' global_vars[%s] = SSZType(%s })' % (x, re.sub('( ){4}', ' '*8, ssz_objects[x][:-2])), ssz_objects)) + map(lambda x: ' global_vars[\'%s\'] = SSZType(%s })' % (x, re.sub('( ){4}', ' '*8, ssz_objects[x][:-2])), ssz_objects)) ssz_objects_reinitialization_spec = ( 'def init_SSZ_types():\n global_vars = globals()\n' + ssz_objects_reinitialization_spec diff --git a/test_libs/pyspec/tests/phase1/epoch_processing/test_phase0.py b/test_libs/pyspec/tests/phase1/epoch_processing/test_phase0_epoch_processing.py similarity index 100% rename from test_libs/pyspec/tests/phase1/epoch_processing/test_phase0.py rename to test_libs/pyspec/tests/phase1/epoch_processing/test_phase0_epoch_processing.py From c13421a9a7d13585b0558e1c4a8a0221d97d3c2d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 22 May 2019 16:52:44 -0400 Subject: [PATCH 164/308] type hinting for fork version Co-Authored-By: Hsiao-Wei Wang --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c14228dc11..60f774e9a8 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -633,7 +633,7 @@ The `hash` function is SHA256. ### `bls_domain` ```python -def bls_domain(domain_type: int, fork_version=b'\x00\x00\x00\x00') -> int: +def bls_domain(domain_type: int, fork_version: bytes=b'\x00\x00\x00\x00') -> int: """ Return the bls domain given by the ``domain_type`` and optional 4 byte ``fork_version`` (defaults to zero). """ From ae6d30fd62e4c4f20086388dcbf8f100265932b1 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 23 May 2019 11:48:04 +1000 Subject: [PATCH 165/308] Update with discv5 --- specs/networking/libp2p-standardization.md | 31 +++++----------------- 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index d9e8565866..b6a46db1bf 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -147,29 +147,12 @@ an `attestation`. The [RPC Interface](./rpc-interface.md) is specified in this repository. -## Identify - -**Note: This protocol is a placeholder and will be updated once the discv5 -discovery protocol is added to this document** - -#### Protocol Id: `/eth/serenity/id/1.0.0` - -The Identify protocol (defined in go - [identify-go](https://github.com/ipfs/go-ipfs/blob/master/core/commands/id.go) and rust [rust-identify](https://github.com/libp2p/rust-libp2p/blob/master/protocols/identify/src/lib.rs)) -allows a node A to query another node B which information B knows about A. This also includes the addresses B is listening on. - -This protocol allows nodes to discover addresses of other nodes to be added to -peer discovery. It further allows nodes to determine the capabilities of all it's connected -peers. - -### Configuration Parameters - -The protocol has two configurable parameters, which can be used to identify the -type of connecting node. Suggested format: -``` - version: `/eth/serenity/1.0.0` - user_agent: -``` - ## Discovery -**To be updated to incorporate discv5** +Discovery Version 5 +([discv5])(https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md) +will be used for discovery. This protocol uses a UDP transport and specifies +its own encryption, ip-discovery and topic advertisement. Therefore, it has no +need to establish establish streams through `multistream-select`, rather, act +as a standalone implementation that feeds discovered peers/topics (ENR-records) as +`multiaddrs` into the libp2p service. From 7b30c55cd4a774af76d53013c1c5e0b8f63a1604 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 23 May 2019 13:10:34 -0400 Subject: [PATCH 166/308] minor copy edits to vc api --- specs/validator/0_beacon-node-validator-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-node-validator-api.md b/specs/validator/0_beacon-node-validator-api.md index ec0300e9a9..afecc395c4 100644 --- a/specs/validator/0_beacon-node-validator-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -15,7 +15,7 @@ The validator client is a conceptually separate entity which utilizes private ke Since it is recommended to separate these concerns in the client implementations, we must clearly define the communication between them. -The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects. For example, the validator client from Lighthouse, could communicate with a running instance of the beacon node from Prysm. +The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects and to encourage innovation in validator clients independet of beacon node development. For example, the validator client from Lighthouse could communicate with a running instance of the beacon node from Prysm, or a staking pool might create a decentrally managed validator client plugging into the same API. This specification is derived from a proposal and discussion on Issues [#1011](https://github.com/ethereum/eth2.0-specs/issues/1011) and [#1012](https://github.com/ethereum/eth2.0-specs/issues/1012) From 650c4244bd8266fd35bc683f60e5eb54ae7b6243 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 24 May 2019 01:34:39 +0800 Subject: [PATCH 167/308] Things are different after rebasing #1035 --- specs/core/1_custody-game.md | 38 ++++++++++++------------------------ 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index f91fe81de0..82bb352ddb 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -38,6 +38,16 @@ - [`get_chunk_bits_root`](#get_chunk_bits_root) - [`get_randao_epoch_for_custody_period`](#get_randao_epoch_for_custody_period) - [`get_validators_custody_reveal_period`](#get_validators_custody_reveal_period) + - [`replace_empty_or_append`](#replace_empty_or_append) + - [Per-block processing](#per-block-processing) + - [Operations](#operations) + - [Custody key reveals](#custody-key-reveals) + - [Early derived secret reveals](#early-derived-secret-reveals) + - [Chunk challenges](#chunk-challenges) + - [Bit challenges](#bit-challenges) + - [Custody responses](#custody-responses) + - [Per-epoch processing](#per-epoch-processing) + - [Handling of custody-related deadlines](#handling-of-custody-related-deadlines) @@ -289,7 +299,7 @@ def get_randao_epoch_for_custody_period(period: int, validator_index: ValidatorI ### `get_validators_custody_reveal_period` - ```python +```python def get_validators_custody_reveal_period(state: BeaconState, validator_index: ValidatorIndex, epoch: Epoch=None) -> int: @@ -370,7 +380,7 @@ def process_custody_key_reveal(state: BeaconState, increase_balance(state, proposer_index, base_reward(state, index) // MINOR_REWARD_QUOTIENT) ``` -##### Early derived secret reveals +#### Early derived secret reveals Verify that `len(block.body.early_derived_secret_reveals) <= MAX_EARLY_DERIVED_SECRET_REVEALS`. @@ -689,28 +699,6 @@ Append this to `process_final_updates(state)`: ) for index, validator in enumerate(state.validator_registry): if index not in validator_indices_in_records: - if validator.exit_epoch != FAR_FUTURE_EPOCH and validator.withdrawable_epoch == FAR_FUTURE_EPOCH: + if validator.exit_epoch != FAR_FUTURE_EPOCH: validator.withdrawable_epoch = validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY ``` - -In `process_penalties_and_exits`, change the definition of `eligible` to the following (note that it is not a pure function because `state` is declared in the surrounding scope): - -```python -def eligible(state: BeaconState, index: ValidatorIndex) -> bool: - validator = state.validator_registry[index] - # Cannot exit if there are still open chunk challenges - if len([record for record in state.custody_chunk_challenge_records if record.responder_index == index]) > 0: - return False - # Cannot exit if there are still open bit challenges - if len([record for record in state.custody_bit_challenge_records if record.responder_index == index]) > 0: - return False - # Cannot exit if you have not revealed all of your custody keys - elif validator.next_custody_reveal_period <= get_validators_custody_reveal_period(state, index, validator.exit_epoch): - return False - # Cannot exit if you already have - elif validator.withdrawable_epoch < FAR_FUTURE_EPOCH: - return False - # Return minimum time - else: - return current_epoch >= validator.exit_epoch + MIN_VALIDATOR_WITHDRAWAL_EPOCHS -``` From 0f7abfa4ce85316be09ee7657c425b0d6fcfb155 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 24 May 2019 02:54:41 +0800 Subject: [PATCH 168/308] Update specs/core/1_custody-game.md Co-Authored-By: Carl Beekhuizen --- specs/core/1_custody-game.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 82bb352ddb..568879b179 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -699,6 +699,6 @@ Append this to `process_final_updates(state)`: ) for index, validator in enumerate(state.validator_registry): if index not in validator_indices_in_records: - if validator.exit_epoch != FAR_FUTURE_EPOCH: + if validator.exit_epoch != FAR_FUTURE_EPOCH and validator.withdrawable_epoch == FAR_FUTURE_EPOCH: validator.withdrawable_epoch = validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY ``` From 32f97a1c8025206c3c033a159f7e82dff3a5030c Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 23 May 2019 15:12:29 -0700 Subject: [PATCH 169/308] update version used in test generators to get SHA-256 hash --- test_generators/bls/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/bls/requirements.txt b/test_generators/bls/requirements.txt index 8a933d41ca..5eebde29f9 100644 --- a/test_generators/bls/requirements.txt +++ b/test_generators/bls/requirements.txt @@ -1,3 +1,3 @@ -py-ecc==1.6.0 +py-ecc==1.7.0 eth-utils==1.4.1 ../../test_libs/gen_helpers From 2cd188358bf982439fa29b87cdb7e29492ab9bbc Mon Sep 17 00:00:00 2001 From: Age Manning Date: Fri, 24 May 2019 21:24:35 +1000 Subject: [PATCH 170/308] Fix typo in libp2p-standardization --- specs/networking/libp2p-standardization.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/networking/libp2p-standardization.md b/specs/networking/libp2p-standardization.md index b6a46db1bf..d1ba07e652 100644 --- a/specs/networking/libp2p-standardization.md +++ b/specs/networking/libp2p-standardization.md @@ -150,9 +150,9 @@ The [RPC Interface](./rpc-interface.md) is specified in this repository. ## Discovery Discovery Version 5 -([discv5])(https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md) +([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) will be used for discovery. This protocol uses a UDP transport and specifies its own encryption, ip-discovery and topic advertisement. Therefore, it has no -need to establish establish streams through `multistream-select`, rather, act +need to establish streams through `multistream-select`, rather, act as a standalone implementation that feeds discovered peers/topics (ENR-records) as `multiaddrs` into the libp2p service. From ac9564c2b73cf913387fdc604c87b242759aebce Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 24 May 2019 16:51:21 +0200 Subject: [PATCH 171/308] Adds Insert-into-label functionality --- scripts/build_spec.py | 24 ++++++++++++++++-------- scripts/function_puller.py | 30 ++++++++++++++++++++---------- specs/core/0_beacon-chain.md | 3 +++ specs/core/1_custody-game.md | 11 ++++++++++- 4 files changed, 49 insertions(+), 19 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 7619efbe7b..70051ef072 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -1,4 +1,3 @@ -import sys import re import function_puller from argparse import ArgumentParser @@ -108,7 +107,7 @@ def apply_constants_preset(preset: Dict[str, Any]): ''' -def objects_to_spec(functions, constants, ssz_objects, imports): +def objects_to_spec(functions, constants, ssz_objects, imports, inserts={}): new_type_definitions = '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in NEW_TYPES.items()]) functions_spec = '\n\n'.join(functions.values()) constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]),constants)) @@ -119,7 +118,7 @@ def objects_to_spec(functions, constants, ssz_objects, imports): 'def init_SSZ_types():\n global_vars = globals()\n' + ssz_objects_reinitialization_spec ) - return ( + spec = ( imports + '\n' + new_type_definitions + '\n\n' + constants_spec @@ -129,11 +128,15 @@ def objects_to_spec(functions, constants, ssz_objects, imports): + '\n\n' + ssz_objects_reinitialization_spec + '\n' ) + # Handle @inserts + for key, value in inserts.items(): + spec = re.sub('[ ]*# %s\\n' % key, value, spec) + return spec + def combine_functions(old_funcitons, new_functions): for key, value in new_functions.items(): old_funcitons[key] = value - # TODO: Add insert functionality return old_funcitons @@ -167,8 +170,12 @@ def combine_ssz_objects(old_objects, new_objects): return old_objects +# inserts are handeled the same way as functions +combine_inserts = combine_functions + + def build_phase0_spec(sourcefile, outfile=None): - functions, constants, ssz_objects = function_puller.get_spec(sourcefile) + functions, constants, ssz_objects, _ = function_puller.get_spec(sourcefile) spec = objects_to_spec(functions, constants, ssz_objects, PHASE0_IMPORTS) if outfile is not None: with open(outfile, 'w') as out: @@ -178,12 +185,13 @@ def build_phase0_spec(sourcefile, outfile=None): def build_phase1_spec(phase0_sourcefile, phase1_sourcefile, outfile=None): - phase0_functions, phase0_constants, phase0_ssz_objects = function_puller.get_spec(phase0_sourcefile) - phase1_functions, phase1_constants, phase1_ssz_objects = function_puller.get_spec(phase1_sourcefile) + phase0_functions, phase0_constants, phase0_ssz_objects, phase0_inserts = function_puller.get_spec(phase0_sourcefile) + phase1_functions, phase1_constants, phase1_ssz_objects, phase1_inserts = function_puller.get_spec(phase1_sourcefile) functions = combine_functions(phase0_functions, phase1_functions) constants = combine_constants(phase0_constants, phase1_constants) ssz_objects = combine_ssz_objects(phase0_ssz_objects, phase1_ssz_objects) - spec = objects_to_spec(functions, constants, ssz_objects, PHASE1_IMPORTS) + inserts = combine_inserts(phase0_inserts, phase1_inserts) + spec = objects_to_spec(functions, constants, ssz_objects, PHASE1_IMPORTS, inserts) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) diff --git a/scripts/function_puller.py b/scripts/function_puller.py index 29ecccfbf5..72cb0365c4 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -1,19 +1,22 @@ -import sys import re -from typing import List +from typing import Dict, Tuple FUNCTION_REGEX = r'^def [\w_]*' +BEGIN_INSERT_REGEX = r'# begin insert ' +END_INSERT_REGEX = r'# end insert' -def get_spec(file_name: str): - code_lines = [] - pulling_from = None # line number of start of latest object - current_name = None # most recent section title +def get_spec(file_name: str) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str], Dict[str, str]]: + pulling_from = None # line number of start of latest object + current_name = None # most recent section title + insert_name = None # stores the label of the current insert object functions = {} constants = {} ssz_objects = {} + inserts = {} function_matcher = re.compile(FUNCTION_REGEX) + inserts_matcher = re.compile(BEGIN_INSERT_REGEX) for linenum, line in enumerate(open(file_name).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': @@ -23,12 +26,19 @@ def get_spec(file_name: str): pulling_from = linenum + 1 elif line[:3] == '```': pulling_from = None + elif inserts_matcher.match(line) is not None: + insert_name = re.search(r'@[\w]*', line).group(0) + elif insert_name is not None: + if re.match(END_INSERT_REGEX, line) is not None: + insert_name = None + else: + inserts[insert_name] = inserts.get(insert_name, '') + line + '\n' else: # Handle function definitions if pulling_from is not None: - match = function_matcher.match(line) - if match is not None: - current_name = match.group(0) + func_match = function_matcher.match(line) + if func_match is not None: + current_name = func_match.group(0) if function_matcher.match(current_name) is None: ssz_objects[current_name] = ssz_objects.get(current_name, '') + line + '\n' else: @@ -49,4 +59,4 @@ def get_spec(file_name: str): eligible = False if eligible: constants[row[0]] = row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890') - return functions, constants, ssz_objects + return functions, constants, ssz_objects, inserts diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 60f774e9a8..be3e3d7d07 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1280,8 +1280,11 @@ def process_epoch(state: BeaconState) -> None: process_crosslinks(state) process_rewards_and_penalties(state) process_registry_updates(state) + # @process_reveal_deadlines + # @process_challenge_deadlines process_slashings(state) process_final_updates(state) + # @after_process_final_updates ``` #### Helper functions diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index b96ea613a0..e71a92cd99 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -674,6 +674,9 @@ def process_bit_challenge_response(state: BeaconState, Run `process_reveal_deadlines(state)` immediately after `process_registry_updates(state)`: ```python +# begin insert @process_reveal_deadlines + process_reveal_deadlines(state) +# end insert @process_reveal_deadlines def process_reveal_deadlines(state: BeaconState) -> None: for index, validator in enumerate(state.validator_registry): if (validator.next_custody_reveal_period + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) @@ -684,6 +687,9 @@ def process_reveal_deadlines(state: BeaconState) -> None: Run `process_challenge_deadlines(state)` immediately after `process_reveal_deadlines(state)`: ```python +# begin insert @process_challenge_deadlines + process_challenge_deadlines(state) +# end insert @process_challenge_deadlines def process_challenge_deadlines(state: BeaconState) -> None: for challenge in state.custody_chunk_challenge_records: if get_current_epoch(state) > challenge.inclusion_epoch + CUSTODY_RESPONSE_DEADLINE: @@ -701,12 +707,15 @@ def process_challenge_deadlines(state: BeaconState) -> None: Append this to `process_final_updates(state)`: ```python +# begin insert @after_process_final_updates + after_process_final_updates(state) +# end insert @after_process_final_updates def after_process_final_updates(state: BeaconState) -> None: current_epoch = get_current_epoch(state) # Clean up exposed RANDAO key reveals state.exposed_derived_secrets[current_epoch % EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS] = [] # Reset withdrawable epochs if challenge records are empty - records = state.custody_chunk_challenge_records + state.bit_challenge_records + records = state.custody_chunk_challenge_records + state.custody_bit_challenge_records validator_indices_in_records = set( [record.challenger_index for record in records] + [record.responder_index for record in records] ) From 28b76bcd4c84a93a51d87207a5a825496809c4c6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 24 May 2019 11:59:22 -0600 Subject: [PATCH 172/308] a number of minor fixes in phsae 1 --- specs/core/1_shard-data-chains.md | 48 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index e1092b4044..1d1186247b 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -215,7 +215,7 @@ def get_shard_header(block: ShardBlock) -> ShardBlockHeader: def verify_shard_attestation_signature(state: BeaconState, attestation: ShardAttestation) -> None: data = attestation.data - persistent_committee = get_persistent_committee(state, data.crosslink.shard, data.slot) + persistent_committee = get_persistent_committee(state, data.shard, data.slot) assert verify_bitfield(attestation.aggregation_bitfield, len(persistent_committee)) pubkeys = [] for i, index in enumerate(persistent_committee): @@ -225,7 +225,7 @@ def verify_shard_attestation_signature(state: BeaconState, pubkeys.append(validator.pubkey) assert bls_verify( pubkey=bls_aggregate_pubkeys(pubkeys), - message_hash=data.crosslink.shard_block_root, + message_hash=data.shard_block_root, signature=attestation.aggregate_signature, domain=get_domain(state, slot_to_epoch(data.slot), DOMAIN_SHARD_ATTESTER) ) @@ -280,22 +280,22 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], return True # Check slot number - assert block.slot >= PHASE_1_GENESIS_SLOT + assert candidate.slot >= PHASE_1_GENESIS_SLOT assert unix_time >= beacon_state.genesis_time + (block.slot - GENESIS_SLOT) * SECONDS_PER_SLOT # Check shard number - assert block.shard <= SHARD_COUNT + assert candidate.shard <= SHARD_COUNT # Check beacon block - beacon_block = beacon_blocks[block.slot] - assert block.beacon_block_root == signing_root(beacon_block) - assert beacon_block.slot <= block.slot: + beacon_block = beacon_blocks[candidate.slot] + assert candidate.beacon_block_root == signing_root(beacon_block) + assert beacon_block.slot <= candidate.slot: # Check state root - assert block.state_root == ZERO_HASH # [to be removed in phase 2] + assert candidate.state_root == ZERO_HASH # [to be removed in phase 2] # Check parent block - if block.slot == PHASE_1_GENESIS_SLOT: + if candidate.slot == PHASE_1_GENESIS_SLOT: assert candidate.parent_root == ZERO_HASH else: parent_block = next( @@ -303,26 +303,26 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], signing_root(block) == candidate.parent_root , None) assert parent_block != None - assert parent_block.shard == block.shard - assert parent_block.slot < block.slot + assert parent_block.shard == candidate.shard + assert parent_block.slot < candidate.slot assert signing_root(beacon_blocks[parent_block.slot]) == parent_block.beacon_chain_root # Check attestations - assert len(block.attestations) <= MAX_SHARD_ATTESTIONS - for _, attestation in enumerate(block.attestations): - assert max(GENESIS_SHARD_SLOT, block.slot - SLOTS_PER_EPOCH) <= attestation.data.slot - assert attestation.data.slot <= block.slot - MIN_ATTESTATION_INCLUSION_DELAY - assert attestation.data.crosslink.shard == block.shard + assert len(candidate.attestations) <= MAX_SHARD_ATTESTIONS + for _, attestation in enumerate(candidate.attestations): + assert max(GENESIS_SHARD_SLOT, candidate.slot - SLOTS_PER_EPOCH) <= attestation.data.slot + assert attestation.data.slot <= candidate.slot - MIN_ATTESTATION_INCLUSION_DELAY + assert attestation.data.crosslink.shard == candidate.shard verify_shard_attestation_signature(beacon_state, attestation) # Check signature - proposer_index = get_shard_proposer_index(beacon_state, block.shard, block.slot) + proposer_index = get_shard_proposer_index(beacon_state, candidate.shard, candidate.slot) assert proposer_index is not None assert bls_verify( pubkey=validators[proposer_index].pubkey, message_hash=signing_root(block), - signature=block.signature, - domain=get_domain(beacon_state, slot_to_epoch(block.slot), DOMAIN_SHARD_PROPOSER) + signature=candidate.signature, + domain=get_domain(beacon_state, slot_to_epoch(candidate.slot), DOMAIN_SHARD_PROPOSER) ) return True @@ -339,18 +339,18 @@ Let: ```python def is_valid_shard_attestation(valid_shard_blocks: List[ShardBlock], beacon_state: BeaconState, - candidate: Attestation) -> bool: + candidate: ShardAttestation) -> bool: # Check shard block shard_block = next( block for block in valid_shard_blocks if - signing_root(block) == candidate.attestation.data.crosslink.shard_block_root + signing_root(block) == candidate.data.shard_block_root , None) assert shard_block != None - assert shard_block.slot == attestation.data.slot - assert shard_block.shard == attestation.data.crosslink.shard + assert shard_block.slot == candidate.data.slot + assert shard_block.shard == candidate.data.shard # Check signature - verify_shard_attestation_signature(beacon_state, attestation) + verify_shard_attestation_signature(beacon_state, candidate) return True ``` From d805fb508446c8b9a6a0299cf5ac585b2597362d Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 25 May 2019 00:35:17 +0300 Subject: [PATCH 173/308] Simplify deposits --- specs/core/0_beacon-chain.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 60f774e9a8..48a686735d 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -471,8 +471,6 @@ The types are defined topologically to aid in facilitating an executable version { # Branch in the deposit tree 'proof': ['bytes32', DEPOSIT_CONTRACT_TREE_DEPTH], - # Index in the deposit tree - 'index': 'uint64', # Data 'data': DepositData, } @@ -1763,12 +1761,11 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: leaf=hash_tree_root(deposit.data), proof=deposit.proof, depth=DEPOSIT_CONTRACT_TREE_DEPTH, - index=deposit.index, + index=state.deposit_index, root=state.latest_eth1_data.deposit_root, ) # Deposits must be processed in order - assert deposit.index == state.deposit_index state.deposit_index += 1 pubkey = deposit.data.pubkey From edf0b9d05fd77d3c1a1035f4e97252cfc18d9d8e Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Sat, 25 May 2019 07:59:35 +1000 Subject: [PATCH 174/308] Addressed some of @djrtwo's suggestions. - Rewording of specification goal paragraph - Clarify get duties description regarding chain reorgs. - Add epoch parameter to get duties, and new error 406 - Block publishing action is clarified around validation, with a new status code 202 - The validator pubkey and PoC bit are passed to produce attestation - Attestation publishing action is clarified around validation, new status code 202 - Rewording of genesis_time, 'block' -> 'slot' - Update Crosslink to latest spec - Added missing signature field to IndexedAttestation --- .../validator/0_beacon-node-validator-api.md | 2 +- specs/validator/beacon_node_oapi.yaml | 69 +++++++++++++------ 2 files changed, 50 insertions(+), 21 deletions(-) diff --git a/specs/validator/0_beacon-node-validator-api.md b/specs/validator/0_beacon-node-validator-api.md index afecc395c4..c53ce37aca 100644 --- a/specs/validator/0_beacon-node-validator-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -15,7 +15,7 @@ The validator client is a conceptually separate entity which utilizes private ke Since it is recommended to separate these concerns in the client implementations, we must clearly define the communication between them. -The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects and to encourage innovation in validator clients independet of beacon node development. For example, the validator client from Lighthouse could communicate with a running instance of the beacon node from Prysm, or a staking pool might create a decentrally managed validator client plugging into the same API. +The goal of this specification is to promote interoperability between beacon nodes and validator clients derived from different projects and to encourage innovation in validator client implementations, independently from beacon node development. For example, the validator client from Lighthouse could communicate with a running instance of the beacon node from Prysm, or a staking pool might create a decentrally managed validator client which utilises the same API. This specification is derived from a proposal and discussion on Issues [#1011](https://github.com/ethereum/eth2.0-specs/issues/1011) and [#1012](https://github.com/ethereum/eth2.0-specs/issues/1012) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 900b8eefa8..c98fbd0482 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -92,7 +92,7 @@ paths: tags: - MinimalSet summary: "Get validator duties for the requested validators." - description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators. This API call should be polled at every slot, to ensure that any chain reorganisations are catered for, and to ensure that the currently connected beacon node is properly synchronized." + description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators, for a particular epoch. Duties should only need to be checked once per epoch, however a chain reorganisation (of > MIN_SEED_LOOKAHEAD epochs) could occur, resulting in a change of duties. For full safety, this API call should be polled at every slot to ensure that chain reorganisations are recognised, and to ensure that the beacon node is properly synchronized." parameters: - name: validator_pubkeys in: query @@ -103,6 +103,11 @@ paths: items: $ref: '#/components/schemas/pubkey' minItems: 1 + - name: epoch + in: query + required: false + schema: + type: integer responses: 200: description: Success response @@ -114,6 +119,8 @@ paths: $ref: '#/components/schemas/ValidatorDuty' 400: $ref: '#/components/responses/InvalidRequest' + 406: + description: "Duties cannot be provided for the requested epoch." 500: $ref: '#/components/responses/InternalError' 503: @@ -157,7 +164,7 @@ paths: tags: - MinimalSet summary: "Publish a signed block." - description: "Instructs the beacon node to publish a newly signed beacon block to the beacon network, to be included in the beacon chain." + description: "Instructs the beacon node to broadcast a newly signed beacon block to the beacon network, to be included in the beacon chain. The beacon node is not required to validate the signed `BeaconBlock`, and a successful response (20X) only indicates that the broadcast has been successful. The beacon node is expected to integrate the new block into its state, and therefore validate the block internally, however blocks which fail the validation are still broadcast but a different status code is returned (202)" parameters: - name: beacon_block in: query @@ -167,7 +174,9 @@ paths: $ref: '#/components/schemas/BeaconBlock' responses: 200: - $ref: '#/components/responses/Success' + description: "The block was validated successfully and has been broadcast. It has also been integrated into the beacon node's database." + 202: + description: "The block failed validation, but was successfully broadcast anyway. It was not integrated into the beacon node's database." 400: $ref: '#/components/responses/InvalidRequest' 500: @@ -182,6 +191,18 @@ paths: summary: "Produce an attestation, without signature." description: "Requests that the beacon node produce an IndexedAttestation, with a blank signature field, which the validator will then sign." parameters: + - name: validator_pubkey + in: query + required: true + description: "Uniquely identifying which validator this attestation is to be produced for." + schema: + $ref: '#/components/schemas/pubkey' + - name: poc_bit + in: query + required: true + description: "The proof-of-custody bit that is reported by this " + schema: + # Still need to establish a schema for this. - name: slot in: query required: true @@ -210,8 +231,8 @@ paths: post: tags: - MinimalSet - summary: "Published a signed attestation." - description: "Instructs the beacon node to publish a newly signed IndexedAttestation object, to be incorporated into the beacon chain." + summary: "Publish a signed attestation." + description: "Instructs the beacon node to broadcast a newly signed IndexedAttestation object to the intended shard subnet. The beacon node is not required to validate the signed IndexedAttestation, and a successful response (20X) only indicates that the broadcast has been successful. The beacon node is expected to integrate the new attestation into its state, and therefore validate the attestation internally, however attestations which fail the validation are still broadcast but a different status code is returned (202)" parameters: - name: attestation in: query @@ -221,7 +242,9 @@ paths: $ref: '#/components/schemas/IndexedAttestation' responses: 200: - $ref: '#/components/responses/Success' + description: "The attestation was validated successfully and has been broadcast. It has also been integrated into the beacon node's database." + 202: + description: "The attestation failed validation, but was successfully broadcast anyway. It was not integrated into the beacon node's database." 400: $ref: '#/components/responses/InvalidRequest' 500: @@ -244,17 +267,13 @@ components: genesis_time: type: integer format: uint64 - description: "The genesis_time configured for the beacon node, which is the time the Eth1.0 validator deposit smart contract has enough ETH staked (i.e. Eth2.0 begins)." + description: "The genesis_time configured for the beacon node, which is the unix time at which the Eth2.0 chain began." example: 1557716289 ValidatorDuty: type: object properties: validator_pubkey: $ref: '#/components/schemas/pubkey' - committee_index: - type: integer - format: uint64 - description: "The index of the validator in the committee." attestation_slot: type: integer format: uint64 @@ -272,18 +291,18 @@ components: type: object nullable: true properties: - starting_block: + starting_slot: type: integer format: uint64 - description: "The block at which syncing started (will only be reset after the sync reached its head)" - current_block: + description: "The slot at which syncing started (will only be reset after the sync reached its head)" + current_slot: type: integer format: uint64 - description: "The current highest block sync'd by the beacon node." - highest_block: + description: "The most recent slot sync'd by the beacon node." + highest_slot: type: integer format: uint64 - description: "The estimated highest block, or current target block number." + description: "Globally, the estimated most recent slot number, or current target slot number." BeaconBlock: description: "The [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblock) object from the Eth2.0 spec." @@ -543,6 +562,12 @@ components: items: type: integer format: uint64 + signature: + type: string + format: bytes + pattern: "^0x[a-fA-F0-9]{192}$" + example: "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505" + description: "The BLS signature of the `IndexedAttestation`, created by the validator of the attestation." data: $ref: '#/components/schemas/AttestationData' AttestationData: @@ -575,16 +600,20 @@ components: crosslink: title: CrossLink type: object - description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec." + description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec, contains data from epochs [`start_epoch`, `end_epoch`)." properties: shard: type: integer format: uint64 description: "The shard number." - epoch: + start_epoch: + type: integer + format: uint64 + description: "The first epoch which the crosslinking data references." + end_epoch: type: integer format: uint64 - description: "The epoch number." + description: "The 'end' epoch referred to by the crosslinking data; no data in this Crosslink should refer to the `end_epoch` since it is not included in the crosslinking data interval." parent_root: type: string format: byte From ed4416ba348cae92b1326c357b1c6e6f946e095e Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 25 May 2019 00:05:03 +0200 Subject: [PATCH 175/308] update SSZ implementation --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 94 ++++-- .../pyspec/eth2spec/utils/ssz/ssz_switch.py | 105 ------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 282 ++++++++++++++---- 3 files changed, 290 insertions(+), 191 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 74bc1bf99b..2b0328bfac 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,5 +1,27 @@ from eth2spec.utils.merkle_minimal import merkleize_chunks -from .ssz_switch import * +from .ssz_typing import * + + +# SSZ Defaults +# ----------------------------- + + +def get_zero_value(typ): + if is_uint(typ): + return 0 + if issubclass(typ, bool): + return False + if issubclass(typ, list): + return [] + if issubclass(typ, Vector): + return typ() + if issubclass(typ, BytesN): + return typ() + if issubclass(typ, bytes): + return b'' + if issubclass(typ, SSZContainer): + return typ(**{f: get_zero_value(t) for f, t in typ.get_fields().items()}), + # SSZ Helpers # ----------------------------- @@ -14,23 +36,10 @@ def chunkify(byte_string): return [byte_string[i:i + 32] for i in range(0, len(byte_string), 32)] -BYTES_PER_LENGTH_OFFSET = 4 - - -# SSZ Implementation +# SSZ Serialization # ----------------------------- -get_zero_value = ssz_type_switch({ - ssz_bool: lambda: False, - ssz_uint: lambda: 0, - ssz_list: lambda byte_form: b'' if byte_form else [], - ssz_vector: lambda length, elem_typ, byte_form: - (b'\x00' * length if length > 0 else b'') if byte_form else - [get_zero_value(elem_typ) for _ in range(length)], - ssz_container: lambda typ, field_names, field_types: - typ(**{f_name: get_zero_value(f_typ) for f_name, f_typ in zip(field_names, field_types)}), -}) - +BYTES_PER_LENGTH_OFFSET = 4 serialize = ssz_switch({ ssz_bool: lambda value: b'\x01' if value else b'\x00', @@ -40,13 +49,6 @@ def chunkify(byte_string): ssz_container: lambda value, get_field_values, field_types: encode_series(get_field_values(value), field_types), }) -ssz_basic_type = (ssz_bool, ssz_uint) - -is_basic_type = ssz_type_switch({ - ssz_basic_type: lambda: True, - ssz_default: lambda: False, -}) - is_fixed_size = ssz_type_switch({ ssz_basic_type: lambda: True, ssz_vector: lambda elem_typ: is_fixed_size(elem_typ), @@ -55,6 +57,27 @@ def chunkify(byte_string): }) +# SSZ Hash-tree-root +# ----------------------------- + +def serialize_basic(value, typ): + if is_uint(typ): + return value.to_bytes(typ.byte_len, 'little') + if issubclass(typ, bool): + if value: + return b'\x01' + else: + return b'\x00' + + +def pack(values, subtype): + return b''.join([serialize_basic(value, subtype) for value in values]) + + +def is_basic_type(typ): + return is_uint(typ) or issubclass(typ, bool) + + def hash_tree_root_list(value, elem_typ): if is_basic_type(elem_typ): return merkleize_chunks(chunkify(pack(value, elem_typ))) @@ -77,10 +100,9 @@ def hash_tree_root_container(fields): ssz_container: lambda value, get_field_values, field_types: hash_tree_root_container(zip(get_field_values(value), field_types)), }) -signing_root = ssz_switch({ - ssz_container: lambda value, get_field_values, field_types: hash_tree_root_container(zip(get_field_values(value), field_types)[:-1]), - ssz_default: lambda value, typ: hash_tree_root(value, typ), -}) +# todo: signing root +def signing_root(value, typ): + pass def encode_series(values, types): @@ -115,3 +137,21 @@ def encode_series(values, types): # Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts return b''.join(fixed_parts + variable_parts) + + +# Implementation notes: +# - SSZContainer,Vector/BytesN.hash_tree_root/serialize functions are for ease, implementation here +# - uint types have a 'byte_len' attribute +# - uint types are not classes. They use NewType(), for performance. +# This forces us to check type equivalence by exact reference. +# There's no class. The type data comes from an annotation/argument from the context of the value. +# - Vector is not valid to create instances with. Give it a elem-type and length: Vector[FooBar, 123] +# - *The class of* a Vector instance has a `elem_type` (type, may not be a class, see uint) and `length` (int) +# - BytesN is not valid to create instances with. Give it a length: BytesN[123] +# - *The class of* a BytesN instance has a `length` (int) +# Where possible, it is preferable to create helpers that just act on the type, and don't unnecessarily use a value +# E.g. is_basic_type(). This way, we can use them in type-only contexts and have no duplicate logic. +# For every class-instance, you can get the type with my_object.__class__ +# For uints, and other NewType related, you have to rely on type information. It cannot be retrieved from the value. +# Note: we may just want to box integers instead. And then we can do bounds checking too. But it is SLOW and MEMORY INTENSIVE. +# diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py deleted file mode 100644 index 3da1e7cb1f..0000000000 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_switch.py +++ /dev/null @@ -1,105 +0,0 @@ -from typing import Dict, Any - -from .ssz_typing import * - -# SSZ Switch statement runner factory -# ----------------------------- - - -def ssz_switch(sw: Dict[Any, Any], arg_names=None): - """ - Creates an SSZ switch statement: a function, that when executed, checks every switch-statement - """ - if arg_names is None: - arg_names = ["value", "typ"] - - # Runner, the function that executes the switch when called. - # Accepts a arguments based on the arg_names declared in the ssz_switch. - def run_switch(*args): - # value may be None - value = None - try: - value = args[arg_names.index("value")] - except ValueError: - pass # no value argument - - # typ may be None when value is not None - typ = None - try: - typ = args[arg_names.index("typ")] - except ValueError: - # no typ argument expected - pass - except IndexError: - # typ argument expected, but not passed. Try to get it from the class info - typ = value.__class__ - if hasattr(typ, '__forward_arg__'): - typ = typ.__forward_arg__ - - # Now, go over all switch cases - for matchers, worker in sw.items(): - if not isinstance(matchers, tuple): - matchers = (matchers,) - # for each matcher of the case key - for m in matchers: - data = m(typ) - # if we have data, the matcher matched, and we can return the result - if data is not None: - # Supply value and type by default, and any data presented by the matcher. - kwargs = {"value": value, "typ": typ, **data} - # Filter out unwanted arguments - filtered_kwargs = {k: kwargs[k] for k in worker.__code__.co_varnames} - # run the switch case and return result - return worker(**filtered_kwargs) - raise Exception("cannot find matcher for type: %s (value: %s)" % (typ, value)) - return run_switch - - -def ssz_type_switch(sw: Dict[Any, Any]): - return ssz_switch(sw, ["typ"]) - - -# SSZ Switch matchers -# ----------------------------- - -def ssz_bool(typ): - if typ == bool: - return {} - - -def ssz_uint(typ): - # Note: only the type reference exists, - # but it really resolves to 'int' during run-time for zero computational/memory overhead. - # Hence, we check equality to the type references (which are really just 'NewType' instances), - # and don't use any sub-classing like we normally would. - if typ == uint8 or typ == uint16 or typ == uint32 or typ == uint64\ - or typ == uint128 or typ == uint256 or typ == byte: - return {"byte_len": typ.byte_len} - - -def ssz_list(typ): - if hasattr(typ, '__bases__') and List in typ.__bases__: - return {"elem_typ": read_list_elem_typ(typ), "byte_form": False} - if typ == bytes: - return {"elem_typ": uint8, "byte_form": True} - - -def ssz_vector(typ): - if hasattr(typ, '__bases__'): - if Vector in typ.__bases__: - return {"elem_typ": read_vec_elem_typ(typ), "length": read_vec_len(typ), "byte_form": False} - if BytesN in typ.__bases__: - return {"elem_typ": uint8, "length": read_bytesN_len(typ), "byte_form": True} - - -def ssz_container(typ): - if hasattr(typ, '__bases__') and SSZContainer in typ.__bases__: - def get_field_values(value): - return [getattr(value, field) for field in typ.__annotations__.keys()] - field_names = list(typ.__annotations__.keys()) - field_types = list(typ.__annotations__.values()) - return {"get_field_values": get_field_values, "field_names": field_names, "field_types": field_types} - - -def ssz_default(typ): - return {} diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 6a8a22586e..dc23427b34 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,72 +1,18 @@ -from typing import Generic, List, TypeVar, Type, Iterable, NewType +from typing import List, Iterable, TypeVar, Type, NewType +from typing import Union +from inspect import isclass -# SSZ base length, to limit length generic type param of vector/bytesN -SSZLenAny = type('SSZLenAny', (), {}) - - -def SSZLen(length: int): - """ - SSZ length factory. Creates a type corresponding to a given length. To be used as parameter in type generics. - """ - assert length >= 0 - typ = type('SSZLen_%d' % length, (SSZLenAny,), {}) - typ.length = length - return typ - - -# SSZ element type T = TypeVar('T') -# SSZ vector/bytesN length -L = TypeVar('L', bound=SSZLenAny) - -# SSZ vector +# SSZ list # ----------------------------- -class Vector(Generic[T, L]): - def __init__(self, *args: Iterable[T]): - self.items = list(args) - - def __getitem__(self, key): - return self.items[key] - - def __setitem__(self, key, value): - self.items[key] = value - - def __iter__(self): - return iter(self.items) - - def __len__(self): - return len(self.items) - - -def read_vec_elem_typ(vec_typ: Type[Vector[T,L]]) -> T: - assert vec_typ.__args__ is not None - return vec_typ.__args__[0] - -def read_vec_len(vec_typ: Type[Vector[T,L]]) -> int: - assert vec_typ.__args__ is not None - return vec_typ.__args__[1].length - - -# SSZ list -# ----------------------------- def read_list_elem_typ(list_typ: Type[List[T]]) -> T: assert list_typ.__args__ is not None return list_typ.__args__[0] -# SSZ bytesN -# ----------------------------- -class BytesN(Generic[L]): - pass - - -def read_bytesN_len(bytesN_typ: Type[BytesN[L]]) -> int: - assert bytesN_typ.__args__ is not None - return bytesN_typ.__args__[0].length - # SSZ integer types, with 0 computational overhead (NewType) # ----------------------------- @@ -94,8 +40,9 @@ def read_bytesN_len(bytesN_typ: Type[BytesN[L]]) -> int: class SSZContainer(object): def __init__(self, **kwargs): + cls = self.__class__ from .ssz_impl import get_zero_value - for f, t in self.__annotations__.items(): + for f, t in cls.get_fields().items(): if f not in kwargs: setattr(self, f, get_zero_value(t)) else: @@ -113,3 +60,220 @@ def signing_root(self): from .ssz_impl import signing_root return signing_root(self, self.__class__) + def get_field_values(self): + cls = self.__class__ + return [getattr(self, field) for field in cls.get_field_names()] + + @classmethod + def get_fields(cls): + return dict(cls.__annotations__) + + @classmethod + def get_field_names(cls): + return list(cls.__annotations__.keys()) + + @classmethod + def get_field_types(cls): + # values of annotations are the types corresponding to the fields, not instance values. + return list(cls.__annotations__.values()) + + +def is_uint(typ): + # Note: only the type reference exists, + # but it really resolves to 'int' during run-time for zero computational/memory overhead. + # Hence, we check equality to the type references (which are really just 'NewType' instances), + # and don't use any sub-classing like we normally would. + return typ == uint8 or typ == uint16 or typ == uint32 or typ == uint64 \ + or typ == uint128 or typ == uint256 or typ == byte + +# SSZ vector +# ----------------------------- + + +def _is_vector_instance_of(a, b): + if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): + # Vector (b) is not an instance of Vector[X, Y] (a) + return False + if not hasattr(a, 'elem_type') or not hasattr(a, 'length'): + # Vector[X, Y] (b) is an instance of Vector (a) + return True + + # Vector[X, Y] (a) is an instance of Vector[X, Y] (b) + return a.elem_type == b.elem_type and a.length == b.length + + +def _is_equal_vector_type(a, b): + if not hasattr(a, 'elem_type') or not hasattr(a, 'length'): + if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): + # Vector == Vector + return True + # Vector != Vector[X, Y] + return False + if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): + # Vector[X, Y] != Vector + return False + # Vector[X, Y] == Vector[X, Y] + return a.elem_type == b.elem_type and a.length == b.length + + +class VectorMeta(type): + def __new__(cls, class_name, parents, attrs): + out = type.__new__(cls, class_name, parents, attrs) + if 'elem_type' in attrs and 'length' in attrs: + setattr(out, 'elem_type', attrs['elem_type']) + setattr(out, 'length', attrs['length']) + return out + + def __getitem__(self, params): + return self.__class__(self.__name__, (Vector,), {'elem_type': params[0], 'length': params[1]}) + + def __subclasscheck__(self, sub): + return _is_vector_instance_of(self, sub) + + def __instancecheck__(self, other): + return _is_vector_instance_of(self, other.__class__) + + def __eq__(self, other): + return _is_equal_vector_type(self, other) + + def __ne__(self, other): + return not _is_equal_vector_type(self, other) + + +class Vector(metaclass=VectorMeta): + + def __init__(self, *args: Iterable[T]): + + cls = self.__class__ + if not hasattr(cls, 'elem_type'): + raise TypeError("Type Vector without elem_type data cannot be instantiated") + if not hasattr(cls, 'length'): + raise TypeError("Type Vector without length data cannot be instantiated") + + if len(args) != cls.length: + if len(args) == 0: + from .ssz_impl import get_zero_value + args = [get_zero_value(cls.elem_type) for _ in range(cls.length)] + else: + raise TypeError("Typed vector with length %d cannot hold %d items" % (cls.length, len(args))) + + self.items = list(args) + + # cannot check non-class objects + if isclass(cls.elem_type): + for i, item in enumerate(self.items): + if not isinstance(item, cls.elem_type): + raise TypeError("Typed vector cannot hold differently typed value" + " at index %d. Got type: %s, expected type: %s" % (i, type(item), cls.elem_type)) + + def serialize(self): + from .ssz_impl import serialize + return serialize(self, self.__class__) + + def hash_tree_root(self): + from .ssz_impl import hash_tree_root + return hash_tree_root(self, self.__class__) + + def __getitem__(self, key): + return self.items[key] + + def __setitem__(self, key, value): + self.items[key] = value + + def __iter__(self): + return iter(self.items) + + def __len__(self): + return len(self.items) + + +def _is_bytes_n_instance_of(a, b): + if not hasattr(b, 'length'): + # BytesN (b) is not an instance of BytesN[X] (a) + return False + if not hasattr(a, 'length'): + # BytesN[X] (b) is an instance of BytesN (a) + return True + + # BytesN[X] (a) is an instance of BytesN[X] (b) + return a.length == b.length + + +def _is_equal_bytes_n_type(a, b): + if not hasattr(a, 'length'): + if not hasattr(b, 'length'): + # BytesN == BytesN + return True + # BytesN != BytesN[X] + return False + if not hasattr(b, 'length'): + # BytesN[X] != BytesN + return False + # BytesN[X] == BytesN[X] + return a.length == b.length + + +class BytesNMeta(type): + def __new__(cls, class_name, parents, attrs): + out = type.__new__(cls, class_name, parents, attrs) + if 'length' in attrs: + setattr(out, 'length', attrs['length']) + return out + + def __getitem__(self, n): + return self.__class__(self.__name__, (BytesN,), {'length': n}) + + def __subclasscheck__(self, sub): + return _is_bytes_n_instance_of(self, sub) + + def __instancecheck__(self, other): + return _is_bytes_n_instance_of(self, other.__class__) + + def __eq__(self, other): + return _is_equal_bytes_n_type(self, other) + + def __ne__(self, other): + return not _is_equal_bytes_n_type(self, other) + + +def parse_bytes(val): + if val is None: + return None + if isinstance(val, str): + # TODO: import from eth-utils instead, and do: hexstr_if_str(to_bytes, val) + return None + if isinstance(val, bytes): + return val + if isinstance(val, int): + return bytes([val]) + return None + + +class BytesN(bytes, metaclass=BytesNMeta): + def __new__(cls, *args): + if not hasattr(cls, 'length'): + return + bytesval = None + if len(args) == 1: + val: Union[bytes, int, str] = args[0] + bytesval = parse_bytes(val) + elif len(args) > 1: + # TODO: each int is 1 byte, check size, create bytesval + bytesval = bytes(args) + + if bytesval is None: + if cls.length == 0: + bytesval = b'' + else: + bytesval = b'\x00' * cls.length + if len(bytesval) != cls.length: + raise TypeError("bytesN[%d] cannot be initialized with value of %d bytes" % (cls.length, len(bytesval))) + return super().__new__(cls, bytesval) + + def serialize(self): + from .ssz_impl import serialize + return serialize(self, self.__class__) + + def hash_tree_root(self): + from .ssz_impl import hash_tree_root + return hash_tree_root(self, self.__class__) From 81cb4a23b31ef11ae2b17d7b4f19d92608466cfe Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 25 May 2019 00:10:06 +0200 Subject: [PATCH 176/308] update some common usage of SSZ types, as a start --- scripts/phase0/build_spec.py | 6 ++-- scripts/phase0/function_puller.py | 18 ------------ specs/core/0_beacon-chain.md | 48 +++++++++++++++---------------- 3 files changed, 27 insertions(+), 45 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index b226194d65..b188dada25 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -33,9 +33,9 @@ def build_phase0_spec(sourcefile, outfile): Shard = NewType('Shard', int) # uint64 ValidatorIndex = NewType('ValidatorIndex', int) # uint64 Gwei = NewType('Gwei', int) # uint64 -Bytes32 = NewType('Bytes32', bytes) # bytes32 -BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 -BLSSignature = NewType('BLSSignature', bytes) # bytes96 +Bytes32 = BytesN[32] +BLSPubkey = NewType('BLSPubkey', BytesN[48]) +BLSSignature = NewType('BLSSignature', BytesN[96]) Store = None """) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index f94687344e..5c1cf859bb 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -2,22 +2,6 @@ from typing import List -def translate_ssz_type_line(line: str) -> str: - if ':' not in line: - return line - start = line[:line.index(':')] - field_type = line[line.index(':')+2:] - if field_type.startswith('['): - if ',' in line: - # TODO: translate [Foobar, SOME_THING] to Vector[Foobar, SSZLen(SOME_THING)] cleanly. - # just brute it here - field_type = 'Vector[%s, SSLen(%s)]' % (field_type[1:field_type.index(',')], field_type[field_type.index(',')+2:len(field_type)-1]) - else: - field_type = 'List[%s]' % field_type[1:len(field_type)-1] - line = start + ': ' + field_type - return line - - def get_spec(file_name: str) -> List[str]: code_lines = [] pulling_from = None @@ -51,8 +35,6 @@ def get_spec(file_name: str) -> List[str]: if line[:3] == 'def': code_lines.append('') code_lines.append('') - if current_typedef is not None: - line = translate_ssz_type_line(line) code_lines.append(line) elif pulling_from is None and len(line) > 0 and line[0] == '|': row = line[1:].split('|') diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index ff5a082efa..35d273a2e0 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -340,8 +340,8 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Validator indices - custody_bit_0_indices: [uint64] - custody_bit_1_indices: [uint64] + custody_bit_0_indices: List[uint64] + custody_bit_1_indices: List[uint64] # Attestation data data: AttestationData # Aggregate signature @@ -418,9 +418,9 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Block roots - block_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] + block_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] # State roots - state_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] } ``` @@ -470,7 +470,7 @@ The types are defined topologically to aid in facilitating an executable version ```python { # Branch in the deposit tree - proof: [bytes32, DEPOSIT_CONTRACT_TREE_DEPTH] + proof: Vector[bytes32, DEPOSIT_CONTRACT_TREE_DEPTH] # Index in the deposit tree index: uint64 # Data @@ -521,12 +521,12 @@ The types are defined topologically to aid in facilitating an executable version randao_reveal: bytes96 eth1_data: Eth1Data graffiti: bytes32 - proposer_slashings: [ProposerSlashing] - attester_slashings: [AttesterSlashing] - attestations: [Attestation] - deposits: [Deposit] - voluntary_exits: [VoluntaryExit] - transfers: [Transfer] + proposer_slashings: List[ProposerSlashing] + attester_slashings: List[AttesterSlashing] + attestations: List[Attestation] + deposits: List[Deposit] + voluntary_exits: List[VoluntaryExit] + transfers: List[Transfer] } ``` @@ -555,16 +555,16 @@ The types are defined topologically to aid in facilitating an executable version fork: Fork # For versioning hard forks # Validator registry - validator_registry: [Validator] - balances: [uint64] + validator_registry: List[Validator] + balances: List[uint64] # Randomness and committees - latest_randao_mixes: [bytes32, LATEST_RANDAO_MIXES_LENGTH] + latest_randao_mixes: Vector[bytes32, LATEST_RANDAO_MIXES_LENGTH] latest_start_shard: uint64 # Finality - previous_epoch_attestations: [PendingAttestation] - current_epoch_attestations: [PendingAttestation] + previous_epoch_attestations: List[PendingAttestation] + current_epoch_attestations: List[PendingAttestation] previous_justified_epoch: uint64 current_justified_epoch: uint64 previous_justified_root: bytes32 @@ -574,20 +574,20 @@ The types are defined topologically to aid in facilitating an executable version finalized_root: bytes32 # Recent state - current_crosslinks: [Crosslink, SHARD_COUNT] - previous_crosslinks: [Crosslink, SHARD_COUNT] - latest_block_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] - latest_state_roots: [bytes32, SLOTS_PER_HISTORICAL_ROOT] - latest_active_index_roots: [bytes32, LATEST_ACTIVE_INDEX_ROOTS_LENGTH] + current_crosslinks: Vector[Crosslink, SHARD_COUNT] + previous_crosslinks: Vector[Crosslink, SHARD_COUNT] + latest_block_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] + latest_state_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] + latest_active_index_roots: Vector[bytes32, LATEST_ACTIVE_INDEX_ROOTS_LENGTH] # Balances slashed at every withdrawal period - latest_slashed_balances: [uint64, LATEST_SLASHED_EXIT_LENGTH] + latest_slashed_balances: Vector[uint64, LATEST_SLASHED_EXIT_LENGTH] # `latest_block_header.state_root == ZERO_HASH` temporarily latest_block_header: BeaconBlockHeader - historical_roots: [bytes32] + historical_roots: List[bytes32] # Ethereum 1.0 chain data latest_eth1_data: Eth1Data - eth1_data_votes: [Eth1Data] + eth1_data_votes: List[Eth1Data] deposit_index: uint64 } ``` From c4b9c6372c8b632fc931296d6e4b68c34e1dd2c5 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 25 May 2019 15:54:14 +0200 Subject: [PATCH 177/308] Merge in state_transition.py --- .../eth2spec/phase1/state_transition.py | 70 ------------------- 1 file changed, 70 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/phase1/state_transition.py diff --git a/test_libs/pyspec/eth2spec/phase1/state_transition.py b/test_libs/pyspec/eth2spec/phase1/state_transition.py deleted file mode 100644 index 87b2e1c12f..0000000000 --- a/test_libs/pyspec/eth2spec/phase1/state_transition.py +++ /dev/null @@ -1,70 +0,0 @@ -from . import spec - -from .spec import ( - BeaconState, - BeaconBlock, - Slot, -) - -from eth2spec.phase0.state_transition import ( - process_operation_type, - process_operations as process_operations_phase0, -) - - -def process_operations(state: BeaconState, block: BeaconBlock) -> None: - process_operations_phase0(state, block) - - process_operation_type( - state, - block.body.custody_key_reveals, - spec.MAX_CUSTODY_KEY_REVEALS, - spec.process_custody_key_reveal, - ) - - process_operation_type( - state, - block.body.early_derived_secret_reveals, - spec.MAX_EARLY_DERIVED_SECRET_REVEALS, - spec.process_early_derived_secret_reveal, - ) - - -def process_block(state: BeaconState, - block: BeaconBlock, - verify_state_root: bool=False) -> None: - spec.process_block_header(state, block) - spec.process_randao(state, block) - spec.process_eth1_data(state, block) - - process_operations(state, block) - if verify_state_root: - spec.verify_block_state_root(state, block) - - -def process_epoch_transition(state: BeaconState) -> None: - spec.process_justification_and_finalization(state) - spec.process_crosslinks(state) - # TODO: Eligible - spec.process_rewards_and_penalties(state) - spec.process_registry_updates(state) - spec.process_reveal_deadlines(state) - spec.process_challenge_deadlines(state) - spec.process_slashings(state) - spec.process_final_updates(state) - spec.after_process_final_updates(state) - - -def state_transition_to(state: BeaconState, up_to: Slot) -> BeaconState: - while state.slot < up_to: - spec.cache_state(state) - if (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0: - process_epoch_transition(state) - spec.advance_slot(state) - - -def state_transition(state: BeaconState, - block: BeaconBlock, - verify_state_root: bool=False) -> BeaconState: - state_transition_to(state, block.slot) - process_block(state, block, verify_state_root) From 5b6a98b1075f0f0c24505d065e83817eca76edf0 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Sat, 25 May 2019 14:06:42 -0400 Subject: [PATCH 178/308] Some updates --- test_libs/pyspec/eth2spec/utils/__init__.py | 4 + .../pyspec/eth2spec/utils/ssz/__init__.py | 1 + .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 153 ++++++++---------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 79 +++++++-- 4 files changed, 138 insertions(+), 99 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/__init__.py b/test_libs/pyspec/eth2spec/utils/__init__.py index e69de29bb2..9f8b364287 100644 --- a/test_libs/pyspec/eth2spec/utils/__init__.py +++ b/test_libs/pyspec/eth2spec/utils/__init__.py @@ -0,0 +1,4 @@ +from .merkle_minimal import * +from .hash_function import * +from .bls_stub import * +from .ssz import * diff --git a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py b/test_libs/pyspec/eth2spec/utils/ssz/__init__.py index e69de29bb2..752d77c437 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/__init__.py @@ -0,0 +1 @@ +from .ssz_impl import * diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 2b0328bfac..8b6c20a179 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,64 +1,13 @@ -from eth2spec.utils.merkle_minimal import merkleize_chunks +from ..merkle_minimal import merkleize_chunks, hash from .ssz_typing import * - -# SSZ Defaults -# ----------------------------- - - -def get_zero_value(typ): - if is_uint(typ): - return 0 - if issubclass(typ, bool): - return False - if issubclass(typ, list): - return [] - if issubclass(typ, Vector): - return typ() - if issubclass(typ, BytesN): - return typ() - if issubclass(typ, bytes): - return b'' - if issubclass(typ, SSZContainer): - return typ(**{f: get_zero_value(t) for f, t in typ.get_fields().items()}), - - -# SSZ Helpers -# ----------------------------- - - -def pack(values, subtype): - return b''.join([serialize(value, subtype) for value in values]) - - -def chunkify(byte_string): - byte_string += b'\x00' * (-len(byte_string) % 32) - return [byte_string[i:i + 32] for i in range(0, len(byte_string), 32)] - - # SSZ Serialization # ----------------------------- BYTES_PER_LENGTH_OFFSET = 4 -serialize = ssz_switch({ - ssz_bool: lambda value: b'\x01' if value else b'\x00', - ssz_uint: lambda value, byte_len: value.to_bytes(byte_len, 'little'), - ssz_list: lambda value, elem_typ: encode_series(value, [elem_typ] * len(value)), - ssz_vector: lambda value, elem_typ, length: encode_series(value, [elem_typ] * length), - ssz_container: lambda value, get_field_values, field_types: encode_series(get_field_values(value), field_types), -}) - -is_fixed_size = ssz_type_switch({ - ssz_basic_type: lambda: True, - ssz_vector: lambda elem_typ: is_fixed_size(elem_typ), - ssz_container: lambda field_types: all(is_fixed_size(f_typ) for f_typ in field_types), - ssz_list: lambda: False, -}) - - -# SSZ Hash-tree-root -# ----------------------------- +def is_basic_type(typ): + return is_uint(typ) or typ == bool def serialize_basic(value, typ): if is_uint(typ): @@ -69,41 +18,29 @@ def serialize_basic(value, typ): else: return b'\x00' - -def pack(values, subtype): - return b''.join([serialize_basic(value, subtype) for value in values]) - - -def is_basic_type(typ): - return is_uint(typ) or issubclass(typ, bool) - - -def hash_tree_root_list(value, elem_typ): - if is_basic_type(elem_typ): - return merkleize_chunks(chunkify(pack(value, elem_typ))) +def is_fixed_size(typ): + if is_basic_type(typ): + return True + elif is_list_type(typ): + return False + elif is_vector_type(typ): + return is_fixed_size(read_vector_elem_typ(typ)) + elif is_container_typ(typ): + return all([is_fixed_size(t) for t in typ.get_field_types()]) else: - return merkleize_chunks([hash_tree_root(element, elem_typ) for element in value]) - - -def mix_in_length(root, length): - return hash(root + length.to_bytes(32, 'little')) - - -def hash_tree_root_container(fields): - return merkleize_chunks([hash_tree_root(field, subtype) for field, subtype in fields]) - - -hash_tree_root = ssz_switch({ - ssz_basic_type: lambda value, typ: merkleize_chunks(chunkify(pack([value], typ))), - ssz_list: lambda value, elem_typ: mix_in_length(hash_tree_root_list(value, elem_typ), len(value)), - ssz_vector: lambda value, elem_typ: hash_tree_root_list(value, elem_typ), - ssz_container: lambda value, get_field_values, field_types: hash_tree_root_container(zip(get_field_values(value), field_types)), -}) - -# todo: signing root -def signing_root(value, typ): - pass - + raise Exception("Type not supported: {}".format(typ)) + +def serialize(obj, typ=None): + if typ is None: + typ = infer_type(obj) + if is_basic_type(typ): + return serialize_basic(obj, typ) + elif is_list_type(typ) or is_vector_type(typ): + return encode_series(list(obj), [read_elem_typ(typ)]*len(obj)) + elif is_container_typ(typ): + return encode_series(obj.get_field_values(), typ.get_field_types()) + else: + raise Exception("Type not supported: {}".format(typ)) def encode_series(values, types): # bytes and bytesN are already in the right format. @@ -138,6 +75,46 @@ def encode_series(values, types): # Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts return b''.join(fixed_parts + variable_parts) +# SSZ Hash-tree-root +# ----------------------------- + +def pack(values, subtype): + if isinstance(values, bytes): + return values + return b''.join([serialize_basic(value, subtype) for value in values]) + +def chunkify(bytez): + bytez += b'\x00' * (-len(bytez) % 32) + return [bytez[i:i + 32] for i in range(0, len(bytez), 32)] + +def mix_in_length(root, length): + return hash(root + length.to_bytes(32, 'little')) + +def hash_tree_root(obj, typ=None): + if typ is None: + typ = infer_type(obj) + if is_basic_type(typ): + return merkleize_chunks(chunkify(serialize_basic(obj, typ))) + elif is_list_type(typ) or is_vector_type(typ): + subtype = read_elem_typ(typ) + if is_basic_type(subtype): + leaves = chunkify(pack(obj, subtype)) + else: + leaves = [hash_tree_root(elem, subtype) for elem in obj] + leaf_root = merkleize_chunks(leaves) + return mix_in_length(leaf_root, len(obj)) if is_list_type(typ) else leaf_root + elif is_container_typ(typ): + leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in zip(obj.get_field_values(), typ.get_field_types())] + return merkleize_chunks(chunkify(b''.join(leaves))) + else: + raise Exception("Type not supported: obj {} type {}".format(obj, typ)) + +def signing_root(value, typ): + if typ is None: + typ = infer_type(obj) + assert is_container_typ(typ) + leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in zip(obj.get_field_values(), typ.get_field_types())[:-1]] + return merkleize_chunks(chunkify(b''.join(leaves))) # Implementation notes: # - SSZContainer,Vector/BytesN.hash_tree_root/serialize functions are for ease, implementation here diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index dc23427b34..78c82be45a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -3,14 +3,7 @@ from inspect import isclass T = TypeVar('T') - -# SSZ list -# ----------------------------- - - -def read_list_elem_typ(list_typ: Type[List[T]]) -> T: - assert list_typ.__args__ is not None - return list_typ.__args__[0] +L = TypeVar('L') @@ -41,7 +34,6 @@ class SSZContainer(object): def __init__(self, **kwargs): cls = self.__class__ - from .ssz_impl import get_zero_value for f, t in cls.get_fields().items(): if f not in kwargs: setattr(self, f, get_zero_value(t)) @@ -125,7 +117,11 @@ def __new__(cls, class_name, parents, attrs): return out def __getitem__(self, params): - return self.__class__(self.__name__, (Vector,), {'elem_type': params[0], 'length': params[1]}) + if not isinstance(params, tuple) or len(params) != 2: + raise Exception("Vector must be instantiated with two args: elem type and length") + o = self.__class__(self.__name__, (Vector,), {'elem_type': params[0], 'length': params[1]}) + o._name = 'Vector' + return o def __subclasscheck__(self, sub): return _is_vector_instance_of(self, sub) @@ -152,7 +148,6 @@ def __init__(self, *args: Iterable[T]): if len(args) != cls.length: if len(args) == 0: - from .ssz_impl import get_zero_value args = [get_zero_value(cls.elem_type) for _ in range(cls.length)] else: raise TypeError("Typed vector with length %d cannot hold %d items" % (cls.length, len(args))) @@ -218,6 +213,8 @@ def __new__(cls, class_name, parents, attrs): out = type.__new__(cls, class_name, parents, attrs) if 'length' in attrs: setattr(out, 'length', attrs['length']) + out._name = 'Vector' + out.elem_type = byte return out def __getitem__(self, n): @@ -277,3 +274,63 @@ def serialize(self): def hash_tree_root(self): from .ssz_impl import hash_tree_root return hash_tree_root(self, self.__class__) + +# SSZ Defaults +# ----------------------------- + +def get_zero_value(typ): + if is_uint(typ): + return 0 + if issubclass(typ, bool): + return False + if issubclass(typ, list): + return [] + if issubclass(typ, Vector): + return typ() + if issubclass(typ, BytesN): + return typ() + if issubclass(typ, bytes): + return b'' + if issubclass(typ, SSZContainer): + return typ(**{f: get_zero_value(t) for f, t in typ.get_fields().items()}), + +# Type helpers +# ----------------------------- + +def infer_type(obj): + if is_uint(obj.__class__): + return obj.__class__ + elif isinstance(obj, int): + return uint64 + elif isinstance(obj, list): + return List[infer_type(obj[0])] + elif isinstance(obj, (Vector, SSZContainer, bool, BytesN, bytes)): + return obj.__class__ + else: + raise Exception("Unknown type for {}".format(obj)) + +def is_list_type(typ): + return (hasattr(typ, '_name') and typ._name == 'List') or typ == bytes + +def is_vector_type(typ): + return hasattr(typ, '_name') and typ._name == 'Vector' + +def is_container_typ(typ): + return hasattr(typ, 'get_fields') + +def read_list_elem_typ(list_typ: Type[List[T]]) -> T: + assert list_typ.__args__ is not None + return list_typ.__args__[0] + +def read_vector_elem_typ(vector_typ: Type[Vector[T, L]]) -> T: + return vector_typ.elem_type + +def read_elem_typ(typ): + if typ == bytes: + return byte + elif is_list_type(typ): + return read_list_elem_typ(typ) + elif is_vector_type(typ): + return read_vector_elem_typ(typ) + else: + raise Exception("Unexpected type: {}".format(typ)) From 3b4d9b7a859777a3e228e4348e2ca7075f87d78a Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 25 May 2019 16:14:52 -0400 Subject: [PATCH 179/308] Class-ified the type definitions --- specs/core/0_beacon-chain.md | 60 ++++++++++++------------------------ 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 35d273a2e0..8442819c44 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -268,20 +268,19 @@ The types are defined topologically to aid in facilitating an executable version #### `Fork` ```python -{ +class Fork(SSZContainer): # Previous fork version previous_version: bytes4 # Current fork version current_version: bytes4 # Fork epoch number epoch: uint64 -} ``` #### `Crosslink` ```python -{ +class Crosslink(SSZContainer): # Shard number shard: uint64 # Epoch number @@ -290,26 +289,24 @@ The types are defined topologically to aid in facilitating an executable version parent_root: bytes32 # Root of the crosslinked shard data since the previous crosslink data_root: bytes32 -} ``` #### `Eth1Data` ```python -{ +class Eth1Data(SSZContainer): # Root of the deposit tree deposit_root: bytes32 # Total number of deposits deposit_count: uint64 # Block hash block_hash: bytes32 -} ``` #### `AttestationData` ```python -{ +class AttestationData(SSZContainer): # LMD GHOST vote beacon_block_root: bytes32 @@ -321,24 +318,22 @@ The types are defined topologically to aid in facilitating an executable version # Crosslink vote crosslink: Crosslink -} ``` #### `AttestationDataAndCustodyBit` ```python -{ +class AttestationDataAndCustodyBit(SSZContainer): # Attestation data data: AttestationData # Custody bit custody_bit: bool -} ``` #### `IndexedAttestation` ```python -{ +class IndexedAttestation(SSZContainer): # Validator indices custody_bit_0_indices: List[uint64] custody_bit_1_indices: List[uint64] @@ -346,13 +341,12 @@ The types are defined topologically to aid in facilitating an executable version data: AttestationData # Aggregate signature signature: bytes96 -} ``` #### `DepositData` ```python -{ +class DepositData(SSZContainer): # BLS pubkey pubkey: bytes48 # Withdrawal credentials @@ -361,24 +355,22 @@ The types are defined topologically to aid in facilitating an executable version amount: uint64 # Container self-signature signature: bytes96 -} ``` #### `BeaconBlockHeader` ```python -{ +class BeaconBlockHeader(SSZContainer): slot: uint64 parent_root: bytes32 state_root: bytes32 body_root: bytes32 signature: bytes96 -} ``` #### `Validator` ```python -{ +class Validator(SSZContainer): # BLS public key pubkey: bytes48 # Withdrawal credentials @@ -395,13 +387,12 @@ The types are defined topologically to aid in facilitating an executable version slashed: bool # Effective balance effective_balance: uint64 -} ``` #### `PendingAttestation` ```python -{ +class PendingAttestation(SSZContainer): # Attester aggregation bitfield aggregation_bitfield: bytes # Attestation data @@ -410,18 +401,16 @@ The types are defined topologically to aid in facilitating an executable version inclusion_delay: uint64 # Proposer index proposer_index: uint64 -} ``` #### `HistoricalBatch` ```python -{ +class HistoricalBatch(SSZContainer): # Block roots block_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] # State roots state_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] -} ``` ### Beacon operations @@ -429,31 +418,29 @@ The types are defined topologically to aid in facilitating an executable version #### `ProposerSlashing` ```python -{ +class ProposerSlashing(SSZContainer): # Proposer index proposer_index: uint64 # First block header header_1: BeaconBlockHeader # Second block header header_2: BeaconBlockHeader -} ``` #### `AttesterSlashing` ```python -{ +class AttesterSlashing(SSZContainer): # First attestation attestation_1: IndexedAttestation # Second attestation attestation_2: IndexedAttestation -} ``` #### `Attestation` ```python -{ +class Attestation(SSZContainer): # Attester aggregation bitfield aggregation_bitfield: bytes # Attestation data @@ -462,39 +449,36 @@ The types are defined topologically to aid in facilitating an executable version custody_bitfield: bytes # BLS aggregate signature signature: bytes96 -} ``` #### `Deposit` ```python -{ +class Deposit(SSZContainer): # Branch in the deposit tree proof: Vector[bytes32, DEPOSIT_CONTRACT_TREE_DEPTH] # Index in the deposit tree index: uint64 # Data data: DepositData -} ``` #### `VoluntaryExit` ```python -{ +class VoluntaryExit(SSZContainer): # Minimum epoch for processing exit epoch: uint64 # Index of the exiting validator validator_index: uint64 # Validator signature signature: bytes96 -} ``` #### `Transfer` ```python -{ +class Transfer(SSZContainer): # Sender index sender: uint64 # Recipient index @@ -509,7 +493,6 @@ The types are defined topologically to aid in facilitating an executable version pubkey: bytes48 # Sender signature signature: bytes96 -} ``` ### Beacon blocks @@ -517,7 +500,7 @@ The types are defined topologically to aid in facilitating an executable version #### `BeaconBlockBody` ```python -{ +class BeaconBlockBody(SSZContainer): randao_reveal: bytes96 eth1_data: Eth1Data graffiti: bytes32 @@ -527,20 +510,18 @@ The types are defined topologically to aid in facilitating an executable version deposits: List[Deposit] voluntary_exits: List[VoluntaryExit] transfers: List[Transfer] -} ``` #### `BeaconBlock` ```python -{ +class BeaconBlock(SSZContainer): # Header slot: uint64 parent_root: bytes32 state_root: bytes32 body: BeaconBlockBody signature: bytes96 -} ``` ### Beacon state @@ -548,7 +529,7 @@ The types are defined topologically to aid in facilitating an executable version #### `BeaconState` ```python -{ +class BeaconState(SSZContainer): # Misc slot: uint64 genesis_time: uint64 @@ -589,7 +570,6 @@ The types are defined topologically to aid in facilitating an executable version latest_eth1_data: Eth1Data eth1_data_votes: List[Eth1Data] deposit_index: uint64 -} ``` ## Custom types From a7dbbafc92e8b824e6f3f1b75c49e4c2aafb61ff Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 26 May 2019 14:14:48 +0200 Subject: [PATCH 180/308] Integrates shard-data-chains --- Makefile | 2 +- scripts/build_spec.py | 37 ++++++++++++++++++++----------- specs/core/1_shard-data-chains.md | 10 ++++----- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index d700d19008..0f74fc6d3f 100644 --- a/Makefile +++ b/Makefile @@ -72,7 +72,7 @@ $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $@ CURRENT_DIR = ${CURDIR} diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 70051ef072..a1d390a95a 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -107,7 +107,7 @@ def apply_constants_preset(preset: Dict[str, Any]): ''' -def objects_to_spec(functions, constants, ssz_objects, imports, inserts={}): +def objects_to_spec(functions, constants, ssz_objects, inserts, imports): new_type_definitions = '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in NEW_TYPES.items()]) functions_spec = '\n\n'.join(functions.values()) constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]),constants)) @@ -174,9 +174,19 @@ def combine_ssz_objects(old_objects, new_objects): combine_inserts = combine_functions +def combine_spec_objects(spec0, spec1): + functions0, constants0, ssz_objects0, inserts0 = spec0 + functions1, constants1, ssz_objects1, inserts1 = spec1 + functions = combine_functions(functions0, functions1) + constants = combine_constants(constants0, constants1) + ssz_objects = combine_ssz_objects(ssz_objects0, ssz_objects1) + inserts = combine_inserts(inserts0, inserts1) + return functions, constants, ssz_objects, inserts + + def build_phase0_spec(sourcefile, outfile=None): - functions, constants, ssz_objects, _ = function_puller.get_spec(sourcefile) - spec = objects_to_spec(functions, constants, ssz_objects, PHASE0_IMPORTS) + functions, constants, ssz_objects, inserts = function_puller.get_spec(sourcefile) + spec = objects_to_spec(functions, constants, ssz_objects, inserts, PHASE0_IMPORTS) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) @@ -184,14 +194,14 @@ def build_phase0_spec(sourcefile, outfile=None): return spec -def build_phase1_spec(phase0_sourcefile, phase1_sourcefile, outfile=None): - phase0_functions, phase0_constants, phase0_ssz_objects, phase0_inserts = function_puller.get_spec(phase0_sourcefile) - phase1_functions, phase1_constants, phase1_ssz_objects, phase1_inserts = function_puller.get_spec(phase1_sourcefile) - functions = combine_functions(phase0_functions, phase1_functions) - constants = combine_constants(phase0_constants, phase1_constants) - ssz_objects = combine_ssz_objects(phase0_ssz_objects, phase1_ssz_objects) - inserts = combine_inserts(phase0_inserts, phase1_inserts) - spec = objects_to_spec(functions, constants, ssz_objects, PHASE1_IMPORTS, inserts) +def build_phase1_spec(phase0_sourcefile, phase1_custody_sourcefile, phase1_shard_sourcefile, outfile=None): + phase0_spec = function_puller.get_spec(phase0_sourcefile) + phase1_custody = function_puller.get_spec(phase1_custody_sourcefile) + phase1_shard_data = function_puller.get_spec(phase1_shard_sourcefile) + spec_objects = phase0_spec + for value in [phase1_custody, phase1_shard_data]: + spec_objects = combine_spec_objects(spec_objects, value) + spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) @@ -219,9 +229,10 @@ def build_phase1_spec(phase0_sourcefile, phase1_sourcefile, outfile=None): if args.phase == 0: build_phase0_spec(*args.files) elif args.phase == 1: - if len(args.files) == 3: + print(args.files) + if len(args.files) == 4: build_phase1_spec(*args.files) else: - print(" Phase 1 requires an output as well as 2 input files (phase0.md and phase1.md)") + print(" Phase 1 requires an output as well as 3 input files (phase0.md and phase1.md, phase1.md)") else: print("Invalid phase: {0}".format(args.phase)) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index aed570b806..746d5c7331 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -47,7 +47,7 @@ This document describes the shard data layer and the shard fork choice rule in P | `BYTES_PER_SHARD_BLOCK_BODY` | `2**14` (= 16,384) | | `MAX_SHARD_ATTESTIONS` | `2**4` (= 16) | | `PHASE_1_GENESIS_EPOCH` | **TBD** | -| `PHASE_1_GENESIS_SLOT` | get_epoch_start_slot(PHASE_1_GENESIS_EPOCH) | +| `PHASE_1_GENESIS_SLOT` | **TBD** | ### Time parameters @@ -291,7 +291,7 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], # Check beacon block beacon_block = beacon_blocks[candidate.slot] assert candidate.beacon_block_root == signing_root(beacon_block) - assert beacon_block.slot <= candidate.slot: + assert beacon_block.slot <= candidate.slot # Check state root assert candidate.state_root == ZERO_HASH # [to be removed in phase 2] @@ -344,9 +344,9 @@ def is_valid_shard_attestation(valid_shard_blocks: List[ShardBlock], candidate: ShardAttestation) -> bool: # Check shard block shard_block = next( - block for block in valid_shard_blocks if - signing_root(block) == candidate.data.shard_block_root - , None) + (block for block in valid_shard_blocks if signing_root(block) == candidate.data.shard_block_root), + None, + ) assert shard_block != None assert shard_block.slot == candidate.data.slot assert shard_block.shard == candidate.data.shard From cdfb886c22473f8c145b97e726f6182d68495f57 Mon Sep 17 00:00:00 2001 From: Justin Date: Sun, 26 May 2019 18:42:37 +0300 Subject: [PATCH 181/308] Avoid divisions by zero Possible fix to avoid four cases of divisions by zero: * `return state.validator_registry[index].effective_balance // adjusted_quotient // BASE_REWARDS_PER_EPOCH` * `rewards[index] += get_base_reward(state, index) * attesting_balance // total_balance` * `validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance` * `rewards[index] += base_reward * attesting_balance // committee_balance` See also #1107. --- specs/core/0_beacon-chain.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 60f774e9a8..104f3eb921 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -963,9 +963,9 @@ def bytes_to_int(data: bytes) -> int: ```python def get_total_balance(state: BeaconState, indices: List[ValidatorIndex]) -> Gwei: """ - Return the combined effective balance of an array of ``validators``. + Return the combined effective balance of the ``indices``. (1 Gwei minimum to avoid divisions by zero.) """ - return sum([state.validator_registry[index].effective_balance for index in indices]) + return max(sum([state.validator_registry[index].effective_balance for index in indices]), 1) ``` ### `get_domain` @@ -1413,10 +1413,9 @@ def process_crosslinks(state: BeaconState) -> None: ```python def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: - adjusted_quotient = integer_squareroot(get_total_active_balance(state)) // BASE_REWARD_QUOTIENT - if adjusted_quotient == 0: - return 0 - return state.validator_registry[index].effective_balance // adjusted_quotient // BASE_REWARDS_PER_EPOCH + total_balance = get_total_active_balance(state) + effective_balance = state.validator_registry[index].effective_balance + return effective_balance * BASE_REWARD_QUOTIENT // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH ``` ```python @@ -1531,10 +1530,9 @@ def process_registry_updates(state: BeaconState) -> None: ```python def process_slashings(state: BeaconState) -> None: current_epoch = get_current_epoch(state) - active_validator_indices = get_active_validator_indices(state, current_epoch) - total_balance = get_total_balance(state, active_validator_indices) + total_balance = get_total_active_balance(state) - # Compute `total_penalties` + # Compute slashed balances in the current epoch total_at_start = state.latest_slashed_balances[(current_epoch + 1) % LATEST_SLASHED_EXIT_LENGTH] total_at_end = state.latest_slashed_balances[current_epoch % LATEST_SLASHED_EXIT_LENGTH] total_penalties = total_at_end - total_at_start From f7f069d6b5d3bac350d440c3137d7f7d031c9eb3 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sun, 26 May 2019 19:41:36 +0200 Subject: [PATCH 182/308] Bug fixes and cleanup of 2_shard-data-chains --- specs/core/1_shard-data-chains.md | 46 ++++++++++++++++++------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 746d5c7331..f26b220d74 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -48,6 +48,7 @@ This document describes the shard data layer and the shard fork choice rule in P | `MAX_SHARD_ATTESTIONS` | `2**4` (= 16) | | `PHASE_1_GENESIS_EPOCH` | **TBD** | | `PHASE_1_GENESIS_SLOT` | **TBD** | +| `GENESIS_SHARD_SLOT` | 0 | ### Time parameters @@ -55,6 +56,7 @@ This document describes the shard data layer and the shard fork choice rule in P | - | - | :-: | :-: | | `CROSSLINK_LOOKBACK` | `2**0` (= 1) | epochs | 6.2 minutes | | `PERSISTENT_COMMITTEE_PERIOD` | `2**11` (= 2,048) | epochs | ~9 days | +| `SECONDS_PER_SLOT` | `2**1 * 3**1` (= 6) | 6 seconds | ### Signature domains @@ -122,7 +124,11 @@ This document describes the shard data layer and the shard fork choice rule in P ### `get_period_committee` ```python -def get_period_committee(state: BeaconState, epoch: Epoch, shard: Shard, index: int, count: int) -> List[ValidatorIndex]: +def get_period_committee(state: BeaconState, + epoch: Epoch, + shard: Shard, + index: int, + count: int) -> List[ValidatorIndex]: """ Return committee for a period. Used to construct persistent committees. """ @@ -139,7 +145,8 @@ def get_period_committee(state: BeaconState, epoch: Epoch, shard: Shard, index: ```python def get_switchover_epoch(state: BeaconState, epoch: Epoch, index: ValidatorIndex): earlier_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2 - return bytes_to_int(hash(generate_seed(state, earlier_start_epoch) + bytes3(index))[0:8]) % PERSISTENT_COMMITTEE_PERIOD + return (bytes_to_int(hash(generate_seed(state, earlier_start_epoch) + bytes3(index))[0:8]) + % PERSISTENT_COMMITTEE_PERIOD) ``` ### `get_persistent_committee` @@ -245,15 +252,15 @@ def compute_crosslink_data_root(blocks: List[ShardBlock]) -> Bytes32: values += [b'\x00' * BYTES_PER_SHARD_BLOCK_BODY] return values - def merkle_root_of_bytes(data: bytes) -> bytes: - return merkle_root([data[i:i + 32] for i in range(0, len(data), 32)]) + def hash_tree_root_of_bytes(data: bytes) -> bytes: + return hash_tree_root([data[i:i + 32] for i in range(0, len(data), 32)]) return hash( - merkle_root(pad_to_power_of_2([ - merkle_root_of_bytes(zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_BODY)) for block in blocks + hash_tree_root(pad_to_power_of_2([ + hash_tree_root_of_bytes(zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_BODY)) for block in blocks ])) + - merkle_root(pad_to_power_of_2([ - merkle_root_of_bytes(block.body) for block in blocks + hash_tree_root(pad_to_power_of_2([ + hash_tree_root_of_bytes(block.body) for block in blocks ])) ) ``` @@ -301,10 +308,10 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], assert candidate.parent_root == ZERO_HASH else: parent_block = next( - (block for block in valid_shard_blocks if - signing_root(block) == candidate.parent_root) - , None) - assert parent_block != None + (block for block in valid_shard_blocks if signing_root(block) == candidate.parent_root), + None + ) + assert parent_block is not None assert parent_block.shard == candidate.shard assert parent_block.slot < candidate.slot assert signing_root(beacon_blocks[parent_block.slot]) == parent_block.beacon_chain_root @@ -321,10 +328,10 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], proposer_index = get_shard_proposer_index(beacon_state, candidate.shard, candidate.slot) assert proposer_index is not None assert bls_verify( - pubkey=validators[proposer_index].pubkey, + pubkey=beacon_state.validator_registry[proposer_index].pubkey, message_hash=signing_root(block), signature=candidate.signature, - domain=get_domain(beacon_state, slot_to_epoch(candidate.slot), DOMAIN_SHARD_PROPOSER) + domain=get_domain(beacon_state, slot_to_epoch(candidate.slot), DOMAIN_SHARD_PROPOSER), ) return True @@ -347,7 +354,7 @@ def is_valid_shard_attestation(valid_shard_blocks: List[ShardBlock], (block for block in valid_shard_blocks if signing_root(block) == candidate.data.shard_block_root), None, ) - assert shard_block != None + assert shard_block is not None assert shard_block.slot == candidate.data.slot assert shard_block.shard == candidate.data.shard @@ -384,13 +391,14 @@ def is_valid_beacon_attestation(shard: Shard, else: previous_attestation = next( (attestation for attestation in valid_attestations if - attestation.data.crosslink.data_root == candidate.data.previous_crosslink.data_root) - , None) - assert previous_attestation != None + attestation.data.crosslink.data_root == candidate.data.previous_crosslink.data_root), + None, + ) + assert previous_attestation is not None assert candidate.data.previous_attestation.epoch < slot_to_epoch(candidate.data.slot) # Check crosslink data root - start_epoch = state.latest_crosslinks[shard].epoch + start_epoch = beacon_state.latest_crosslinks[shard].epoch end_epoch = min(slot_to_epoch(candidate.data.slot) - CROSSLINK_LOOKBACK, start_epoch + MAX_EPOCHS_PER_CROSSLINK) blocks = [] for slot in range(start_epoch * SLOTS_PER_EPOCH, end_epoch * SLOTS_PER_EPOCH): From af798a3065f2fe293a1823ebe843ada31b11863a Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 27 May 2019 11:21:23 +1000 Subject: [PATCH 183/308] Minor updates. - Fixed spelling (and made American English) - Clarified the schema for the new poc_bit field, and description. --- specs/validator/beacon_node_oapi.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index c98fbd0482..6dca1a1074 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -83,7 +83,7 @@ paths: chain_id: type: integer format: uint64 - description: "Sometimes called the network id, this number discerns the active chain for the beacon node. Analagous to Eth1.0 JSON-RPC net_version." + description: "Sometimes called the network id, this number discerns the active chain for the beacon node. Analogous to Eth1.0 JSON-RPC net_version." 500: $ref: '#/components/responses/InternalError' @@ -92,7 +92,7 @@ paths: tags: - MinimalSet summary: "Get validator duties for the requested validators." - description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators, for a particular epoch. Duties should only need to be checked once per epoch, however a chain reorganisation (of > MIN_SEED_LOOKAHEAD epochs) could occur, resulting in a change of duties. For full safety, this API call should be polled at every slot to ensure that chain reorganisations are recognised, and to ensure that the beacon node is properly synchronized." + description: "Requests the beacon node to provide a set of _duties_, which are actions that should be performed by validators, for a particular epoch. Duties should only need to be checked once per epoch, however a chain reorganization (of > MIN_SEED_LOOKAHEAD epochs) could occur, resulting in a change of duties. For full safety, this API call should be polled at every slot to ensure that chain reorganizations are recognized, and to ensure that the beacon node is properly synchronized." parameters: - name: validator_pubkeys in: query @@ -200,9 +200,12 @@ paths: - name: poc_bit in: query required: true - description: "The proof-of-custody bit that is reported by this " + description: "The proof-of-custody bit that is to be reported by the requesting validator. This bit will be inserted into the appropriate location in the returned `IndexedAttestation`." schema: - # Still need to establish a schema for this. + type: integer + format: uint32 + minimum: 0 + maximum: 1 - name: slot in: query required: true From fa177e03553f444c1ba1ae845f23470f9e9d7b1d Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Mon, 27 May 2019 13:01:36 +1000 Subject: [PATCH 184/308] Bumped API version number to 0.2.0 --- specs/validator/beacon_node_oapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 6dca1a1074..940f82137f 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -2,7 +2,7 @@ openapi: "3.0.2" info: title: "Minimal Beacon Node API for Validator" description: "A minimal API specification for the beacon node, which enables a validator to connect and perform its obligations on the Ethereum 2.0 phase 0 beacon chain." - version: "0.1.2" + version: "0.2.0" license: name: "Apache 2.0" url: "https://www.apache.org/licenses/LICENSE-2.0.html" From 09f7114b63d65ad4fa369f35f365d8e94a623f8c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 May 2019 13:15:10 +0800 Subject: [PATCH 185/308] copied from deposit contract --- deposit_contract/contracts/__init__.py | 0 deposit_contract/contracts/utils.py | 16 ++ .../contracts/validator_registration.json | 1 + .../contracts/validator_registration.v.py | 138 +++++++++++ deposit_contract/requirements-testing.txt | 7 + deposit_contract/tests/__init__.py | 0 deposit_contract/tests/contracts/__init__.py | 0 deposit_contract/tests/contracts/conftest.py | 111 +++++++++ .../tests/contracts/test_compile.py | 18 ++ .../tests/contracts/test_deposit.py | 221 ++++++++++++++++++ deposit_contract/tests/core/conftest.py | 0 deposit_contract/tests/core/test_import.py | 4 + .../tool/compile_deposit_contract.py | 33 +++ 13 files changed, 549 insertions(+) create mode 100644 deposit_contract/contracts/__init__.py create mode 100644 deposit_contract/contracts/utils.py create mode 100644 deposit_contract/contracts/validator_registration.json create mode 100644 deposit_contract/contracts/validator_registration.v.py create mode 100644 deposit_contract/requirements-testing.txt create mode 100644 deposit_contract/tests/__init__.py create mode 100644 deposit_contract/tests/contracts/__init__.py create mode 100644 deposit_contract/tests/contracts/conftest.py create mode 100644 deposit_contract/tests/contracts/test_compile.py create mode 100644 deposit_contract/tests/contracts/test_deposit.py create mode 100644 deposit_contract/tests/core/conftest.py create mode 100644 deposit_contract/tests/core/test_import.py create mode 100644 deposit_contract/tool/compile_deposit_contract.py diff --git a/deposit_contract/contracts/__init__.py b/deposit_contract/contracts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deposit_contract/contracts/utils.py b/deposit_contract/contracts/utils.py new file mode 100644 index 0000000000..2aa42ecd3e --- /dev/null +++ b/deposit_contract/contracts/utils.py @@ -0,0 +1,16 @@ +import json +import os + +DIR = os.path.dirname(__file__) + + +def get_deposit_contract_code(): + file_path = os.path.join(DIR, './validator_registration.v.py') + deposit_contract_code = open(file_path).read() + return deposit_contract_code + + +def get_deposit_contract_json(): + file_path = os.path.join(DIR, './validator_registration.json') + deposit_contract_json = open(file_path).read() + return json.loads(deposit_contract_json) diff --git a/deposit_contract/contracts/validator_registration.json b/deposit_contract/contracts/validator_registration.json new file mode 100644 index 0000000000..253eedf6d0 --- /dev/null +++ b/deposit_contract/contracts/validator_registration.json @@ -0,0 +1 @@ +{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "merkle_tree_index", "indexed": false}], "anonymous": false, "type": "event"}, {"name": "Eth2Genesis", "inputs": [{"type": "bytes32", "name": "deposit_root", "indexed": false}, {"type": "bytes", "name": "deposit_count", "indexed": false}, {"type": "bytes", "name": "time", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "to_little_endian_64", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [{"type": "uint256", "name": "value"}], "constant": true, "payable": false, "type": "function", "gas": 7077}, {"name": "from_little_endian_64", "outputs": [{"type": "uint256", "name": "out"}], "inputs": [{"type": "bytes", "name": "value"}], "constant": true, "payable": false, "type": "function", "gas": 5983}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79251}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 11056}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 456517}, {"name": "chainStarted", "outputs": [{"type": "bool", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 633}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600060c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600060c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600060c052602060c0200155606051600161014051018060405190131561018757600080fd5b809190121561019557600080fd5b602081106101a257600080fd5b600060c052602060c020015460605160016101405101806040519013156101c857600080fd5b80919012156101d657600080fd5b602081106101e357600080fd5b600160c052602060c02001555b81516001018083528114156100aa575b505061160856600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a0526380673289600051141561026b57602060046101403734156100b457600080fd5b67ffffffffffffffff6101405111156100cc57600080fd5b60006101605261014051610180526101a060006008818352015b6101605160086000811215610103578060000360020a820461010a565b8060020a82025b905090506101605260ff61018051166101c052610160516101c0516101605101101561013557600080fd5b6101c051610160510161016052610180517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8600081121561017e578060000360020a8204610185565b8060020a82025b90509050610180525b81516001018083528114156100e6575b505060186008602082066101e001602082840111156101bc57600080fd5b60208061020082610160600060046015f15050818152809050905090508051602001806102a0828460006004600a8704601201f16101f957600080fd5b50506102a05160206001820306601f82010390506103006102a0516008818352015b8261030051111561022b57610247565b6000610300516102c001535b815160010180835281141561021b575b50505060206102805260406102a0510160206001820306601f8201039050610280f3005b639d70e8066000511415610405576020600461014037341561028c57600080fd5b60286004356004016101603760086004356004013511156102ac57600080fd5b60006101c0526101608060200151600082518060209013156102cd57600080fd5b80919012156102db57600080fd5b806020036101000a82049050905090506101e05261020060006008818352015b60ff6101e05116606051606051610200516007038060405190131561031f57600080fd5b809190121561032d57600080fd5b6008028060405190131561034057600080fd5b809190121561034e57600080fd5b6000811215610365578060000360020a820461036c565b8060020a82025b90509050610220526101c051610220516101c05101101561038c57600080fd5b610220516101c051016101c0526101e0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff860008112156103d5578060000360020a82046103dc565b8060020a82025b905090506101e0525b81516001018083528114156102fb575b50506101c05160005260206000f3005b63c5f2892f600051141561055d57341561041e57600080fd5b6000610140526002546101605261018060006020818352015b60016001610160511614156104b8576000610180516020811061045957600080fd5b600160c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af16104aa57600080fd5b60c051905061014052610526565b6000610140516020826101a001015260208101905061018051602081106104de57600080fd5b600060c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161051c57600080fd5b60c0519050610140525b610160600261053457600080fd5b60028151048152505b8151600101808352811415610437575b50506101405160005260206000f3005b63621fd130600051141561063357341561057657600080fd5b60606101c060246380673289610140526002546101605261015c6000305af161059e57600080fd5b6101e0805160200180610260828460006004600a8704601201f16105c157600080fd5b50506102605160206001820306601f82010390506102c0610260516008818352015b826102c05111156105f35761060f565b60006102c05161028001535b81516001018083528114156105e3575b5050506020610240526040610260510160206001820306601f8201039050610240f3005b63c47e300d60005114156113d557606060046101403760506004356004016101a037603060043560040135111561066957600080fd5b604060243560040161022037602060243560040135111561068957600080fd5b60806044356004016102803760606044356004013511156106a957600080fd5b633b9aca0061034052610340516106bf57600080fd5b61034051340461032052633b9aca006103205110156106dd57600080fd5b6060610440602463806732896103c052610320516103e0526103dc6000305af161070657600080fd5b610460805160200180610360828460006004600a8704601201f161072957600080fd5b50506002546104a05260006104c05260026104e05261050060006020818352015b60006104e05161075957600080fd5b6104e0516104a05160016104a05101101561077357600080fd5b60016104a0510106141515610787576107f3565b6104c06060516001825101806040519013156107a257600080fd5b80919012156107b057600080fd5b8152506104e0805115156107c55760006107df565b60028151600283510204146107d957600080fd5b60028151025b8152505b815160010180835281141561074a575b505060006101a06030806020846105e001018260208501600060046016f15050805182019150506000601060208206610560016020828401111561083657600080fd5b60208061058082610520600060046015f15050818152809050905090506010806020846105e001018260208501600060046013f1505080518201915050806105e0526105e09050602060c0825160208401600060025af161089657600080fd5b60c05190506105405260006000604060208206610680016102805182840111156108bf57600080fd5b6060806106a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af16108ff57600080fd5b60c0519050602082610880010152602081019050600060406020602082066107400161028051828401111561093357600080fd5b606080610760826020602088068803016102800160006004601bf150508181528090509050905060208060208461080001018260208501600060046015f15050805182019150506105205160208261080001015260208101905080610800526108009050602060c0825160208401600060025af16109b057600080fd5b60c051905060208261088001015260208101905080610880526108809050602060c0825160208401600060025af16109e757600080fd5b60c051905061066052600060006105405160208261092001015260208101905061022060208060208461092001018260208501600060046015f150508051820191505080610920526109209050602060c0825160208401600060025af1610a4d57600080fd5b60c0519050602082610aa00101526020810190506000610360600880602084610a2001018260208501600060046012f150508051820191505060006018602082066109a00160208284011115610aa257600080fd5b6020806109c082610520600060046015f1505081815280905090509050601880602084610a2001018260208501600060046014f150508051820191505061066051602082610a2001015260208101905080610a2052610a209050602060c0825160208401600060025af1610b1557600080fd5b60c0519050602082610aa001015260208101905080610aa052610aa09050602060c0825160208401600060025af1610b4c57600080fd5b60c051905061090052610b2060006020818352015b6104c051610b20511215610be1576000610b205160208110610b8257600080fd5b600160c052602060c0200154602082610b4001015260208101905061090051602082610b4001015260208101905080610b4052610b409050602060c0825160208401600060025af1610bd357600080fd5b60c051905061090052610be6565b610bf7565b5b8151600101808352811415610b61575b5050610900516104c05160208110610c0e57600080fd5b600160c052602060c02001556002805460018254011015610c2e57600080fd5b60018154018155506020610c40600463c5f2892f610be052610bfc6000305af1610c5757600080fd5b610c4051610bc0526060610ce060246380673289610c60526104a051610c8052610c7c6000305af1610c8857600080fd5b610d00805160200180610d40828460006004600a8704601201f1610cab57600080fd5b505060a0610dc052610dc051610e00526101a0805160200180610dc051610e0001828460006004600a8704601201f1610ce357600080fd5b5050610dc051610e00015160206001820306601f8201039050610dc051610e0001610da081516040818352015b83610da051101515610d2157610d3e565b6000610da0516020850101535b8151600101808352811415610d10575b505050506020610dc051610e00015160206001820306601f8201039050610dc0510101610dc052610dc051610e2052610220805160200180610dc051610e0001828460006004600a8704601201f1610d9557600080fd5b5050610dc051610e00015160206001820306601f8201039050610dc051610e0001610da081516020818352015b83610da051101515610dd357610df0565b6000610da0516020850101535b8151600101808352811415610dc2575b505050506020610dc051610e00015160206001820306601f8201039050610dc0510101610dc052610dc051610e4052610360805160200180610dc051610e0001828460006004600a8704601201f1610e4757600080fd5b5050610dc051610e00015160206001820306601f8201039050610dc051610e0001610da081516020818352015b83610da051101515610e8557610ea2565b6000610da0516020850101535b8151600101808352811415610e74575b505050506020610dc051610e00015160206001820306601f8201039050610dc0510101610dc052610dc051610e6052610280805160200180610dc051610e0001828460006004600a8704601201f1610ef957600080fd5b5050610dc051610e00015160206001820306601f8201039050610dc051610e0001610da081516060818352015b83610da051101515610f3757610f54565b6000610da0516020850101535b8151600101808352811415610f26575b505050506020610dc051610e00015160206001820306601f8201039050610dc0510101610dc052610dc051610e8052610d40805160200180610dc051610e0001828460006004600a8704601201f1610fab57600080fd5b5050610dc051610e00015160206001820306601f8201039050610dc051610e0001610da081516020818352015b83610da051101515610fe957611006565b6000610da0516020850101535b8151600101808352811415610fd8575b505050506020610dc051610e00015160206001820306601f8201039050610dc0510101610dc0527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce610dc051610e00a1640773594000610320511015156113d357600380546001825401101561107b57600080fd5b60018154018155506201000060035414156113d25742610ec05242610ee052620151806110a757600080fd5b62015180610ee05106610ec05110156110bf57600080fd5b42610ee052620151806110d157600080fd5b62015180610ee05106610ec051036202a30042610ec05242610ee052620151806110fa57600080fd5b62015180610ee05106610ec051101561111257600080fd5b42610ee0526201518061112457600080fd5b62015180610ee05106610ec0510301101561113e57600080fd5b6202a30042610ec05242610ee0526201518061115957600080fd5b62015180610ee05106610ec051101561117157600080fd5b42610ee0526201518061118357600080fd5b62015180610ee05106610ec0510301610ea0526060610f8060246380673289610f0052600254610f2052610f1c6000305af16111be57600080fd5b610fa0805160200180610fe0828460006004600a8704601201f16111e157600080fd5b505060606110c06024638067328961104052610ea0516110605261105c6000305af161120c57600080fd5b6110e0805160200180611120828460006004600a8704601201f161122f57600080fd5b5050610bc0516111e05260606111a0526111a05161120052610fe08051602001806111a0516111e001828460006004600a8704601201f161126f57600080fd5b50506111a0516111e0015160206001820306601f82010390506111a0516111e00161118081516020818352015b83611180511015156112ad576112ca565b6000611180516020850101535b815160010180835281141561129c575b5050505060206111a0516111e0015160206001820306601f82010390506111a05101016111a0526111a051611220526111208051602001806111a0516111e001828460006004600a8704601201f161132157600080fd5b50506111a0516111e0015160206001820306601f82010390506111a0516111e00161118081516020818352015b836111805110151561135f5761137c565b6000611180516020850101535b815160010180835281141561134e575b5050505060206111a0516111e0015160206001820306601f82010390506111a05101016111a0527f08b71ef3f1b58f7a23ffb82e27f12f0888c8403f1ceb0ea7ea26b274e2189d4c6111a0516111e0a160016004555b5b005b63845980e860005114156113fb5734156113ee57600080fd5b60045460005260206000f3005b60006000fd5b61020761160803610207600039610207611608036000f3"} \ No newline at end of file diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py new file mode 100644 index 0000000000..751d3a2e83 --- /dev/null +++ b/deposit_contract/contracts/validator_registration.v.py @@ -0,0 +1,138 @@ +MIN_DEPOSIT_AMOUNT: constant(uint256) = 1000000000 # Gwei +FULL_DEPOSIT_AMOUNT: constant(uint256) = 32000000000 # Gwei +CHAIN_START_FULL_DEPOSIT_THRESHOLD: constant(uint256) = 65536 # 2**16 +DEPOSIT_CONTRACT_TREE_DEPTH: constant(uint256) = 32 +SECONDS_PER_DAY: constant(uint256) = 86400 +MAX_64_BIT_VALUE: constant(uint256) = 18446744073709551615 # 2**64 - 1 + +Deposit: event({ + pubkey: bytes[48], + withdrawal_credentials: bytes[32], + amount: bytes[8], + signature: bytes[96], + merkle_tree_index: bytes[8], +}) +Eth2Genesis: event({deposit_root: bytes32, deposit_count: bytes[8], time: bytes[8]}) + +zerohashes: bytes32[DEPOSIT_CONTRACT_TREE_DEPTH] +branch: bytes32[DEPOSIT_CONTRACT_TREE_DEPTH] +deposit_count: uint256 +full_deposit_count: uint256 +chainStarted: public(bool) + + +@public +def __init__(): + for i in range(DEPOSIT_CONTRACT_TREE_DEPTH - 1): + self.zerohashes[i+1] = sha256(concat(self.zerohashes[i], self.zerohashes[i])) + self.branch[i+1] = self.zerohashes[i + 1] + + +@public +@constant +def to_little_endian_64(value: uint256) -> bytes[8]: + assert value <= MAX_64_BIT_VALUE + + # array access for bytes[] not currently supported in vyper so + # reversing bytes using bitwise uint256 manipulations + y: uint256 = 0 + x: uint256 = value + for i in range(8): + y = shift(y, 8) + y = y + bitwise_and(x, 255) + x = shift(x, -8) + + return slice(convert(y, bytes32), start=24, len=8) + + +@public +@constant +def from_little_endian_64(value: bytes[8]) -> uint256: + y: uint256 = 0 + x: uint256 = convert(value, uint256) + for i in range(8): + y = y + shift(bitwise_and(x, 255), 8 * (7-i)) + x = shift(x, -8) + + return y + + +@public +@constant +def get_deposit_root() -> bytes32: + root: bytes32 = 0x0000000000000000000000000000000000000000000000000000000000000000 + size: uint256 = self.deposit_count + for h in range(DEPOSIT_CONTRACT_TREE_DEPTH): + if bitwise_and(size, 1) == 1: + root = sha256(concat(self.branch[h], root)) + else: + root = sha256(concat(root, self.zerohashes[h])) + size /= 2 + return root + +@public +@constant +def get_deposit_count() -> bytes[8]: + return self.to_little_endian_64(self.deposit_count) + +@payable +@public +def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96]): + deposit_amount: uint256 = msg.value / as_wei_value(1, "gwei") + assert deposit_amount >= MIN_DEPOSIT_AMOUNT + amount: bytes[8] = self.to_little_endian_64(deposit_amount) + + index: uint256 = self.deposit_count + + # add deposit to merkle tree + i: int128 = 0 + power_of_two: uint256 = 2 + for _ in range(DEPOSIT_CONTRACT_TREE_DEPTH): + if (index+1) % power_of_two != 0: + break + i += 1 + power_of_two *= 2 + + zero_bytes_32: bytes32 + pubkey_root: bytes32 = sha256(concat(pubkey, slice(zero_bytes_32, start=0, len=16))) + signature_root: bytes32 = sha256(concat( + sha256(slice(signature, start=0, len=64)), + sha256(concat(slice(signature, start=64, len=32), zero_bytes_32)) + )) + value: bytes32 = sha256(concat( + sha256(concat(pubkey_root, withdrawal_credentials)), + sha256(concat( + amount, + slice(zero_bytes_32, start=0, len=24), + signature_root, + )) + )) + for j in range(DEPOSIT_CONTRACT_TREE_DEPTH): + if j < i: + value = sha256(concat(self.branch[j], value)) + else: + break + self.branch[i] = value + + self.deposit_count += 1 + new_deposit_root: bytes32 = self.get_deposit_root() + log.Deposit( + pubkey, + withdrawal_credentials, + amount, + signature, + self.to_little_endian_64(index), + ) + + if deposit_amount >= FULL_DEPOSIT_AMOUNT: + self.full_deposit_count += 1 + if self.full_deposit_count == CHAIN_START_FULL_DEPOSIT_THRESHOLD: + timestamp_day_boundary: uint256 = ( + as_unitless_number(block.timestamp) - + as_unitless_number(block.timestamp) % SECONDS_PER_DAY + + 2 * SECONDS_PER_DAY + ) + log.Eth2Genesis(new_deposit_root, + self.to_little_endian_64(self.deposit_count), + self.to_little_endian_64(timestamp_day_boundary)) + self.chainStarted = True diff --git a/deposit_contract/requirements-testing.txt b/deposit_contract/requirements-testing.txt new file mode 100644 index 0000000000..5900af8f00 --- /dev/null +++ b/deposit_contract/requirements-testing.txt @@ -0,0 +1,7 @@ +pytest>=3.6,<3.7 +tox==3.0.0, +eth-tester[py-evm]==0.1.0b29, +vyper==0.1.0b9, +web3==4.8.3, +pytest==3.6.1, +../../test_libs/pyspec diff --git a/deposit_contract/tests/__init__.py b/deposit_contract/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deposit_contract/tests/contracts/__init__.py b/deposit_contract/tests/contracts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deposit_contract/tests/contracts/conftest.py b/deposit_contract/tests/contracts/conftest.py new file mode 100644 index 0000000000..634ab26fa5 --- /dev/null +++ b/deposit_contract/tests/contracts/conftest.py @@ -0,0 +1,111 @@ +from random import ( + randint, +) +import re + +import pytest + +from deposit_contract.contracts.utils import ( + get_deposit_contract_code, + get_deposit_contract_json, +) +import eth_tester +from eth_tester import ( + EthereumTester, + PyEVMBackend, +) +from vyper import ( + compiler, +) +from web3 import Web3 +from web3.providers.eth_tester import ( + EthereumTesterProvider, +) + +# Constants +MIN_DEPOSIT_AMOUNT = 1000000000 # Gwei +FULL_DEPOSIT_AMOUNT = 32000000000 # Gwei +CHAIN_START_FULL_DEPOSIT_THRESHOLD = 65536 # 2**16 +DEPOSIT_CONTRACT_TREE_DEPTH = 32 +TWO_TO_POWER_OF_TREE_DEPTH = 2**DEPOSIT_CONTRACT_TREE_DEPTH + + +@pytest.fixture +def tester(): + return EthereumTester(PyEVMBackend()) + + +@pytest.fixture +def a0(tester): + return tester.get_accounts()[0] + + +@pytest.fixture +def w3(tester): + web3 = Web3(EthereumTesterProvider(tester)) + return web3 + + +@pytest.fixture +def registration_contract(w3, tester): + contract_bytecode = get_deposit_contract_json()['bytecode'] + contract_abi = get_deposit_contract_json()['abi'] + registration = w3.eth.contract( + abi=contract_abi, + bytecode=contract_bytecode) + tx_hash = registration.constructor().transact() + tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash) + registration_deployed = w3.eth.contract( + address=tx_receipt.contractAddress, + abi=contract_abi + ) + return registration_deployed + + +@pytest.fixture(scope="session") +def chain_start_full_deposit_thresholds(): + return [randint(1, 5), randint(6, 10), randint(11, 15)] + + +@pytest.fixture(params=[0, 1, 2]) +def modified_registration_contract( + request, + w3, + tester, + chain_start_full_deposit_thresholds): + # Set CHAIN_START_FULL_DEPOSIT_THRESHOLD to different threshold t + registration_code = get_deposit_contract_code() + t = str(chain_start_full_deposit_thresholds[request.param]) + modified_registration_code = re.sub( + r'CHAIN_START_FULL_DEPOSIT_THRESHOLD: constant\(uint256\) = [0-9]+', + 'CHAIN_START_FULL_DEPOSIT_THRESHOLD: constant(uint256) = ' + t, + registration_code, + ) + assert modified_registration_code != registration_code + contract_bytecode = compiler.compile_code(modified_registration_code)['bytecode'] + contract_abi = compiler.mk_full_signature(modified_registration_code) + registration = w3.eth.contract( + abi=contract_abi, + bytecode=contract_bytecode) + tx_hash = registration.constructor().transact() + tx_receipt = w3.eth.waitForTransactionReceipt(tx_hash) + registration_deployed = w3.eth.contract( + address=tx_receipt.contractAddress, + abi=contract_abi + ) + setattr( + registration_deployed, + 'chain_start_full_deposit_threshold', + chain_start_full_deposit_thresholds[request.param] + ) + return registration_deployed + + +@pytest.fixture +def assert_tx_failed(tester): + def assert_tx_failed(function_to_test, exception=eth_tester.exceptions.TransactionFailed): + snapshot_id = tester.take_snapshot() + with pytest.raises(exception): + function_to_test() + tester.revert_to_snapshot(snapshot_id) + return assert_tx_failed diff --git a/deposit_contract/tests/contracts/test_compile.py b/deposit_contract/tests/contracts/test_compile.py new file mode 100644 index 0000000000..72593fd5b5 --- /dev/null +++ b/deposit_contract/tests/contracts/test_compile.py @@ -0,0 +1,18 @@ +from deposit_contract.contracts.utils import ( + get_deposit_contract_code, + get_deposit_contract_json, +) +from vyper import ( + compiler, +) + + +def test_compile_deposit_contract(): + compiled_deposit_contract_json = get_deposit_contract_json() + + deposit_contract_code = get_deposit_contract_code() + abi = compiler.mk_full_signature(deposit_contract_code) + bytecode = compiler.compile_code(deposit_contract_code)['bytecode'] + + assert abi == compiled_deposit_contract_json["abi"] + assert bytecode == compiled_deposit_contract_json["bytecode"] diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py new file mode 100644 index 0000000000..600d74c856 --- /dev/null +++ b/deposit_contract/tests/contracts/test_deposit.py @@ -0,0 +1,221 @@ +from hashlib import ( + sha256, +) +from random import ( + randint, +) + +import pytest + +import eth_utils +from tests.contracts.conftest import ( + DEPOSIT_CONTRACT_TREE_DEPTH, + FULL_DEPOSIT_AMOUNT, + MIN_DEPOSIT_AMOUNT, +) +from eth2spec.utils.minimal_ssz import + SSZType, + hash_tree_root, +) + + +DepositData = SSZType({ + # BLS pubkey + 'pubkey': 'bytes48', + # Withdrawal credentials + 'withdrawal_credentials': 'bytes32', + # Amount in Gwei + 'amount': 'uint64', + # Container self-signature + 'signature': 'bytes96', +}) + + +def hash(data): + return sha256(data).digest() + + +def compute_merkle_root(leaf_nodes): + assert len(leaf_nodes) >= 1 + empty_node = b'\x00' * 32 + child_nodes = leaf_nodes[:] + for _ in range(DEPOSIT_CONTRACT_TREE_DEPTH): + parent_nodes = [] + if len(child_nodes) % 2 == 1: + child_nodes.append(empty_node) + for j in range(0, len(child_nodes), 2): + parent_nodes.append(hash(child_nodes[j] + child_nodes[j + 1])) + child_nodes = parent_nodes + empty_node = hash(empty_node + empty_node) + return child_nodes[0] + + +@pytest.fixture +def deposit_input(): + """ + pubkey: bytes[48] + withdrawal_credentials: bytes[32] + signature: bytes[96] + """ + return ( + b'\x11' * 48, + b'\x22' * 32, + b'\x33' * 96, + ) + + +@pytest.mark.parametrize( + 'value,success', + [ + (0, True), + (10, True), + (55555, True), + (2**64 - 1, True), + (2**64, False), + ] +) +def test_to_little_endian_64(registration_contract, value, success, assert_tx_failed): + call = registration_contract.functions.to_little_endian_64(value) + + if success: + little_endian_64 = call.call() + assert little_endian_64 == (value).to_bytes(8, 'little') + else: + assert_tx_failed( + lambda: call.call() + ) + + +def test_from_little_endian_64(registration_contract, assert_tx_failed): + values = [0, 2**64 - 1] + [randint(1, 2**64 - 2) for _ in range(10)] + for value in values: + call = registration_contract.functions.from_little_endian_64((value).to_bytes(8, 'little')) + assert call.call() == value + + +@pytest.mark.parametrize( + 'success,deposit_amount', + [ + (True, FULL_DEPOSIT_AMOUNT), + (True, MIN_DEPOSIT_AMOUNT), + (False, MIN_DEPOSIT_AMOUNT - 1), + (True, FULL_DEPOSIT_AMOUNT + 1) + ] +) +def test_deposit_amount(registration_contract, + w3, + success, + deposit_amount, + assert_tx_failed, + deposit_input): + call = registration_contract.functions.deposit(*deposit_input) + if success: + assert call.transact({"value": deposit_amount * eth_utils.denoms.gwei}) + else: + assert_tx_failed( + lambda: call.transact({"value": deposit_amount * eth_utils.denoms.gwei}) + ) + + +def test_deposit_log(registration_contract, a0, w3, deposit_input): + log_filter = registration_contract.events.Deposit.createFilter( + fromBlock='latest', + ) + + deposit_amount_list = [randint(MIN_DEPOSIT_AMOUNT, FULL_DEPOSIT_AMOUNT * 2) for _ in range(3)] + for i in range(3): + registration_contract.functions.deposit( + *deposit_input, + ).transact({"value": deposit_amount_list[i] * eth_utils.denoms.gwei}) + + logs = log_filter.get_new_entries() + assert len(logs) == 1 + log = logs[0]['args'] + + assert log['pubkey'] == deposit_input[0] + assert log['withdrawal_credentials'] == deposit_input[1] + assert log['amount'] == deposit_amount_list[i].to_bytes(8, 'little') + assert log['signature'] == deposit_input[2] + assert log['merkle_tree_index'] == i.to_bytes(8, 'little') + + +def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input): + log_filter = registration_contract.events.Deposit.createFilter( + fromBlock='latest', + ) + + deposit_amount_list = [randint(MIN_DEPOSIT_AMOUNT, FULL_DEPOSIT_AMOUNT * 2) for _ in range(10)] + leaf_nodes = [] + for i in range(0, 10): + tx_hash = registration_contract.functions.deposit( + *deposit_input, + ).transact({"value": deposit_amount_list[i] * eth_utils.denoms.gwei}) + receipt = w3.eth.getTransactionReceipt(tx_hash) + print("deposit transaction consumes %d gas" % receipt['gasUsed']) + + logs = log_filter.get_new_entries() + assert len(logs) == 1 + log = logs[0]['args'] + + assert log["merkle_tree_index"] == i.to_bytes(8, 'little') + + deposit_data = DepositData( + pubkey=deposit_input[0][:20], + withdrawal_credentials=deposit_input[1], + amount=deposit_amount_list[i], + signature=deposit_input[2], + ) + hash_tree_root_result = hash_tree_root(deposit_data) + leaf_nodes.append(hash_tree_root_result) + root = compute_merkle_root(leaf_nodes) + assert root == registration_contract.functions.get_deposit_root().call() + + +def test_chain_start(modified_registration_contract, w3, assert_tx_failed, deposit_input): + t = getattr(modified_registration_contract, 'chain_start_full_deposit_threshold') + # CHAIN_START_FULL_DEPOSIT_THRESHOLD is set to t + min_deposit_amount = MIN_DEPOSIT_AMOUNT * eth_utils.denoms.gwei # in wei + full_deposit_amount = FULL_DEPOSIT_AMOUNT * eth_utils.denoms.gwei + log_filter = modified_registration_contract.events.Eth2Genesis.createFilter( + fromBlock='latest', + ) + + index_not_full_deposit = randint(0, t - 1) + for i in range(t): + if i == index_not_full_deposit: + # Deposit with value below FULL_DEPOSIT_AMOUNT + modified_registration_contract.functions.deposit( + *deposit_input, + ).transact({"value": min_deposit_amount}) + logs = log_filter.get_new_entries() + # Eth2Genesis event should not be triggered + assert len(logs) == 0 + else: + # Deposit with value FULL_DEPOSIT_AMOUNT + modified_registration_contract.functions.deposit( + *deposit_input, + ).transact({"value": full_deposit_amount}) + logs = log_filter.get_new_entries() + # Eth2Genesis event should not be triggered + assert len(logs) == 0 + + # Make 1 more deposit with value FULL_DEPOSIT_AMOUNT to trigger Eth2Genesis event + modified_registration_contract.functions.deposit( + *deposit_input, + ).transact({"value": full_deposit_amount}) + logs = log_filter.get_new_entries() + assert len(logs) == 1 + timestamp = int(w3.eth.getBlock(w3.eth.blockNumber)['timestamp']) + timestamp_day_boundary = timestamp + (86400 - timestamp % 86400) + 86400 + log = logs[0]['args'] + assert log['deposit_root'] == modified_registration_contract.functions.get_deposit_root().call() + assert int.from_bytes(log['time'], byteorder='little') == timestamp_day_boundary + assert modified_registration_contract.functions.chainStarted().call() is True + + # Make 1 deposit with value FULL_DEPOSIT_AMOUNT and + # check that Eth2Genesis event is not triggered + modified_registration_contract.functions.deposit( + *deposit_input, + ).transact({"value": full_deposit_amount}) + logs = log_filter.get_new_entries() + assert len(logs) == 0 diff --git a/deposit_contract/tests/core/conftest.py b/deposit_contract/tests/core/conftest.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/deposit_contract/tests/core/test_import.py b/deposit_contract/tests/core/test_import.py new file mode 100644 index 0000000000..d25e6ed67e --- /dev/null +++ b/deposit_contract/tests/core/test_import.py @@ -0,0 +1,4 @@ + + +def test_import(): + import deposit_contract # noqa: F401 diff --git a/deposit_contract/tool/compile_deposit_contract.py b/deposit_contract/tool/compile_deposit_contract.py new file mode 100644 index 0000000000..58f974b8d8 --- /dev/null +++ b/deposit_contract/tool/compile_deposit_contract.py @@ -0,0 +1,33 @@ +import argparse +import json +import os + +from vyper import ( + compiler, +) + +DIR = os.path.dirname(__file__) + + +def generate_compiled_json(file_path: str): + deposit_contract_code = open(file_path).read() + abi = compiler.mk_full_signature(deposit_contract_code) + bytecode = compiler.compile_code(deposit_contract_code)['bytecode'] + contract_json = { + 'abi': abi, + 'bytecode': bytecode, + } + # write json + basename = os.path.basename(file_path) + dirname = os.path.dirname(file_path) + contract_name = basename.split('.')[0] + with open(dirname + "/{}.json".format(contract_name), 'w') as f_write: + json.dump(contract_json, f_write) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("path", type=str, help="the path of the contract") + args = parser.parse_args() + path = args.path + generate_compiled_json(path) From a82a6f9a1a72a701aa9669ea6bf9141e49f3c769 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 27 May 2019 17:17:20 +1000 Subject: [PATCH 186/308] Sort attester slashing indices to avoid arbitrary ordering Closes #1125 --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 60f774e9a8..0678de9d27 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1707,7 +1707,7 @@ def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSla slashed_any = False attesting_indices_1 = attestation_1.custody_bit_0_indices + attestation_1.custody_bit_1_indices attesting_indices_2 = attestation_2.custody_bit_0_indices + attestation_2.custody_bit_1_indices - for index in set(attesting_indices_1).intersection(attesting_indices_2): + for index in sorted(set(attesting_indices_1).intersection(attesting_indices_2)): if is_slashable_validator(state.validator_registry[index], get_current_epoch(state)): slash_validator(state, index) slashed_any = True From b05444a92ce8c8747c4367eb68d318fe0df17126 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 May 2019 15:57:55 +0800 Subject: [PATCH 187/308] Update Makefile and CI config 1. Add `install_deposit_contract_test` command 2. Add `test_deposit_contract` command --- .circleci/config.yml | 18 +++++++++++++++++- Makefile | 10 ++++++++++ deposit_contract/requirements-testing.txt | 13 ++++++------- deposit_contract/tests/contracts/conftest.py | 9 +++++---- .../tests/contracts/test_compile.py | 9 +++++---- .../tests/contracts/test_deposit.py | 17 ++++------------- .../{ => tests}/contracts/utils.py | 4 ++-- deposit_contract/tests/core/conftest.py | 0 deposit_contract/tests/core/test_import.py | 4 ---- 9 files changed, 49 insertions(+), 35 deletions(-) rename deposit_contract/{ => tests}/contracts/utils.py (64%) delete mode 100644 deposit_contract/tests/core/conftest.py delete mode 100644 deposit_contract/tests/core/test_import.py diff --git a/.circleci/config.yml b/.circleci/config.yml index f3c5f6a81c..8d44c76ac3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -64,7 +64,7 @@ jobs: reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements - command: make install_test && make install_lint + command: make install_test && make install_lint && make install_deposit_contract_test - save_cached_venv: venv_name: v1-pyspec-03 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' @@ -97,6 +97,19 @@ jobs: - run: name: Run linter command: make lint + deposit_contract: + docker: + - image: circleci/python:3.6 + working_directory: ~/specs-repo + steps: + - restore_cache: + key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} + - restore_cached_venv: + venv_name: v1-pyspec-03 + reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + - run: + name: Run deposit contract test + command: make test_deposit_contract workflows: version: 2.1 test_spec: @@ -111,3 +124,6 @@ workflows: - lint: requires: - test + - deposit_contract: + requires: + - test diff --git a/Makefile b/Makefile index a6b379b719..ff1891a7c7 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ TEST_LIBS_DIR = ./test_libs PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec YAML_TEST_DIR = ./eth2.0-spec-tests/tests GENERATOR_DIR = ./test_generators +DEPOSIT_CONTRACT_DIR = ./deposit_contract CONFIGS_DIR = ./configs # Collect a list of generator names @@ -46,6 +47,15 @@ lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ flake8 --max-line-length=120 ./eth2spec; +install_deposit_contract_test: + cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; \ + cd ../..; cd $(DEPOSIT_CONTRACT_DIR); \ + pip3 install -r requirements-testing.txt + +test_deposit_contract: + cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; \ + cd ../.. && cd $(DEPOSIT_CONTRACT_DIR) && python -m pytest . + # "make pyspec" to create the pyspec for all phases. pyspec: $(PY_SPEC_ALL_TARGETS) diff --git a/deposit_contract/requirements-testing.txt b/deposit_contract/requirements-testing.txt index 5900af8f00..c1f96a0120 100644 --- a/deposit_contract/requirements-testing.txt +++ b/deposit_contract/requirements-testing.txt @@ -1,7 +1,6 @@ -pytest>=3.6,<3.7 -tox==3.0.0, -eth-tester[py-evm]==0.1.0b29, -vyper==0.1.0b9, -web3==4.8.3, -pytest==3.6.1, -../../test_libs/pyspec +tox==3.0.0 +eth-tester[py-evm]==0.1.0b29 +vyper==0.1.0b9 +web3==4.8.3 +pytest==3.6.1 +../test_libs/pyspec diff --git a/deposit_contract/tests/contracts/conftest.py b/deposit_contract/tests/contracts/conftest.py index 634ab26fa5..69ece247d6 100644 --- a/deposit_contract/tests/contracts/conftest.py +++ b/deposit_contract/tests/contracts/conftest.py @@ -5,10 +5,6 @@ import pytest -from deposit_contract.contracts.utils import ( - get_deposit_contract_code, - get_deposit_contract_json, -) import eth_tester from eth_tester import ( EthereumTester, @@ -21,6 +17,11 @@ from web3.providers.eth_tester import ( EthereumTesterProvider, ) +from .utils import ( + get_deposit_contract_code, + get_deposit_contract_json, +) + # Constants MIN_DEPOSIT_AMOUNT = 1000000000 # Gwei diff --git a/deposit_contract/tests/contracts/test_compile.py b/deposit_contract/tests/contracts/test_compile.py index 72593fd5b5..fc732a6db1 100644 --- a/deposit_contract/tests/contracts/test_compile.py +++ b/deposit_contract/tests/contracts/test_compile.py @@ -1,11 +1,12 @@ -from deposit_contract.contracts.utils import ( - get_deposit_contract_code, - get_deposit_contract_json, -) from vyper import ( compiler, ) +from .utils import ( + get_deposit_contract_code, + get_deposit_contract_json, +) + def test_compile_deposit_contract(): compiled_deposit_contract_json = get_deposit_contract_json() diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py index 600d74c856..92371421e2 100644 --- a/deposit_contract/tests/contracts/test_deposit.py +++ b/deposit_contract/tests/contracts/test_deposit.py @@ -13,22 +13,13 @@ FULL_DEPOSIT_AMOUNT, MIN_DEPOSIT_AMOUNT, ) -from eth2spec.utils.minimal_ssz import +from eth2spec.utils.minimal_ssz import ( SSZType, hash_tree_root, ) - - -DepositData = SSZType({ - # BLS pubkey - 'pubkey': 'bytes48', - # Withdrawal credentials - 'withdrawal_credentials': 'bytes32', - # Amount in Gwei - 'amount': 'uint64', - # Container self-signature - 'signature': 'bytes96', -}) +from eth2spec.phase0.spec import ( + DepositData, +) def hash(data): diff --git a/deposit_contract/contracts/utils.py b/deposit_contract/tests/contracts/utils.py similarity index 64% rename from deposit_contract/contracts/utils.py rename to deposit_contract/tests/contracts/utils.py index 2aa42ecd3e..de7c544896 100644 --- a/deposit_contract/contracts/utils.py +++ b/deposit_contract/tests/contracts/utils.py @@ -5,12 +5,12 @@ def get_deposit_contract_code(): - file_path = os.path.join(DIR, './validator_registration.v.py') + file_path = os.path.join(DIR, './../../contracts/validator_registration.v.py') deposit_contract_code = open(file_path).read() return deposit_contract_code def get_deposit_contract_json(): - file_path = os.path.join(DIR, './validator_registration.json') + file_path = os.path.join(DIR, './../../contracts/validator_registration.json') deposit_contract_json = open(file_path).read() return json.loads(deposit_contract_json) diff --git a/deposit_contract/tests/core/conftest.py b/deposit_contract/tests/core/conftest.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/deposit_contract/tests/core/test_import.py b/deposit_contract/tests/core/test_import.py deleted file mode 100644 index d25e6ed67e..0000000000 --- a/deposit_contract/tests/core/test_import.py +++ /dev/null @@ -1,4 +0,0 @@ - - -def test_import(): - import deposit_contract # noqa: F401 From bff945a2989cf81177510b1aa6e0eaaecf6455c3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 May 2019 17:39:50 +0800 Subject: [PATCH 188/308] Add `restore_default_cached_venv` --- .circleci/config.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8d44c76ac3..52c03ca391 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,6 +31,12 @@ commands: - save_cache: key: << parameters.venv_name >>-venv-<< parameters.reqs_checksum >> paths: << parameters.venv_path >> + restore_default_cached_venv: + description: "Restore the cache with default keys" + steps: + - restore_cached_venv: + venv_name: v1-pyspec-04 + reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' jobs: checkout_specs: docker: @@ -59,9 +65,7 @@ jobs: steps: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_cached_venv: - venv_name: v1-pyspec-03 - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + - restore_default_cached_venv - run: name: Install pyspec requirements command: make install_test && make install_lint && make install_deposit_contract_test @@ -76,9 +80,7 @@ jobs: steps: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_cached_venv: - venv_name: v1-pyspec-03 - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + - restore_default_cached_venv - run: name: Run py-tests command: make citest @@ -91,9 +93,7 @@ jobs: steps: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_cached_venv: - venv_name: v1-pyspec-03 - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + - restore_default_cached_venv - run: name: Run linter command: make lint @@ -104,9 +104,7 @@ jobs: steps: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_cached_venv: - venv_name: v1-pyspec-03 - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + - restore_default_cached_venv - run: name: Run deposit contract test command: make test_deposit_contract From 18cb67425af35b544ab84687df01a349485f69e9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 May 2019 18:05:21 +0800 Subject: [PATCH 189/308] Add `deposit_contract/requirements-testing.txt` to checksum --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 52c03ca391..d15783cfe3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,8 +35,8 @@ commands: description: "Restore the cache with default keys" steps: - restore_cached_venv: - venv_name: v1-pyspec-04 - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' + venv_name: v1-pyspec-01 + reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}--{{ checksum "deposit_contract/requirements-testing.txt" }}' jobs: checkout_specs: docker: From 98cabb665abeb6f08c278701c127962c93d88ead Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 May 2019 18:11:15 +0800 Subject: [PATCH 190/308] Add `save_default_cached_venv` --- .circleci/config.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d15783cfe3..23d68113b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 commands: restore_cached_venv: - description: "Restores a cached venv" + description: "Restore a cached venv" parameters: reqs_checksum: type: string @@ -16,7 +16,7 @@ commands: # fallback to using the latest cache if no exact match is found - << parameters.venv_name >>-venv- save_cached_venv: - description: "Saves a venv into a cache" + description: "Save a venv into a cache" parameters: reqs_checksum: type: string @@ -37,6 +37,13 @@ commands: - restore_cached_venv: venv_name: v1-pyspec-01 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}--{{ checksum "deposit_contract/requirements-testing.txt" }}' + save_default_cached_venv: + description: Save a venv into a cache with default keys" + steps: + - save_cached_venv: + venv_name: v1-pyspec-01 + reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}--{{ checksum "deposit_contract/requirements-testing.txt" }}' + venv_path: ./test_libs/pyspec/venv jobs: checkout_specs: docker: @@ -69,10 +76,7 @@ jobs: - run: name: Install pyspec requirements command: make install_test && make install_lint && make install_deposit_contract_test - - save_cached_venv: - venv_name: v1-pyspec-03 - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - venv_path: ./test_libs/pyspec/venv + - save_default_cached_venv test: docker: - image: circleci/python:3.6 From 05dc4b576f89f8e5049be76351278e27f552509b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 May 2019 18:15:33 +0800 Subject: [PATCH 191/308] deposit contract requires `eth2spec.phase0.spec.DepositData` --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ff1891a7c7..e40c7e95c9 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ install_deposit_contract_test: cd ../..; cd $(DEPOSIT_CONTRACT_DIR); \ pip3 install -r requirements-testing.txt -test_deposit_contract: +test_deposit_contract: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; \ cd ../.. && cd $(DEPOSIT_CONTRACT_DIR) && python -m pytest . From 30bb8986c4ef16747de349c0707c9e41bf0235d5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 May 2019 18:25:31 +0800 Subject: [PATCH 192/308] Bump to latest deposit contract dev branch (fad5c32) --- .../contracts/validator_registration.json | 2 +- .../contracts/validator_registration.v.py | 32 +++++++------ .../tests/contracts/test_deposit.py | 46 +++++++++++++++---- 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.json b/deposit_contract/contracts/validator_registration.json index 253eedf6d0..afc86ec1dc 100644 --- a/deposit_contract/contracts/validator_registration.json +++ b/deposit_contract/contracts/validator_registration.json @@ -1 +1 @@ -{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "merkle_tree_index", "indexed": false}], "anonymous": false, "type": "event"}, {"name": "Eth2Genesis", "inputs": [{"type": "bytes32", "name": "deposit_root", "indexed": false}, {"type": "bytes", "name": "deposit_count", "indexed": false}, {"type": "bytes", "name": "time", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "to_little_endian_64", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [{"type": "uint256", "name": "value"}], "constant": true, "payable": false, "type": "function", "gas": 7077}, {"name": "from_little_endian_64", "outputs": [{"type": "uint256", "name": "out"}], "inputs": [{"type": "bytes", "name": "value"}], "constant": true, "payable": false, "type": "function", "gas": 5983}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79251}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 11056}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 456517}, {"name": "chainStarted", "outputs": [{"type": "bool", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 633}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600060c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600060c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600060c052602060c0200155606051600161014051018060405190131561018757600080fd5b809190121561019557600080fd5b602081106101a257600080fd5b600060c052602060c020015460605160016101405101806040519013156101c857600080fd5b80919012156101d657600080fd5b602081106101e357600080fd5b600160c052602060c02001555b81516001018083528114156100aa575b505061160856600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a0526380673289600051141561026b57602060046101403734156100b457600080fd5b67ffffffffffffffff6101405111156100cc57600080fd5b60006101605261014051610180526101a060006008818352015b6101605160086000811215610103578060000360020a820461010a565b8060020a82025b905090506101605260ff61018051166101c052610160516101c0516101605101101561013557600080fd5b6101c051610160510161016052610180517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8600081121561017e578060000360020a8204610185565b8060020a82025b90509050610180525b81516001018083528114156100e6575b505060186008602082066101e001602082840111156101bc57600080fd5b60208061020082610160600060046015f15050818152809050905090508051602001806102a0828460006004600a8704601201f16101f957600080fd5b50506102a05160206001820306601f82010390506103006102a0516008818352015b8261030051111561022b57610247565b6000610300516102c001535b815160010180835281141561021b575b50505060206102805260406102a0510160206001820306601f8201039050610280f3005b639d70e8066000511415610405576020600461014037341561028c57600080fd5b60286004356004016101603760086004356004013511156102ac57600080fd5b60006101c0526101608060200151600082518060209013156102cd57600080fd5b80919012156102db57600080fd5b806020036101000a82049050905090506101e05261020060006008818352015b60ff6101e05116606051606051610200516007038060405190131561031f57600080fd5b809190121561032d57600080fd5b6008028060405190131561034057600080fd5b809190121561034e57600080fd5b6000811215610365578060000360020a820461036c565b8060020a82025b90509050610220526101c051610220516101c05101101561038c57600080fd5b610220516101c051016101c0526101e0517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff860008112156103d5578060000360020a82046103dc565b8060020a82025b905090506101e0525b81516001018083528114156102fb575b50506101c05160005260206000f3005b63c5f2892f600051141561055d57341561041e57600080fd5b6000610140526002546101605261018060006020818352015b60016001610160511614156104b8576000610180516020811061045957600080fd5b600160c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af16104aa57600080fd5b60c051905061014052610526565b6000610140516020826101a001015260208101905061018051602081106104de57600080fd5b600060c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161051c57600080fd5b60c0519050610140525b610160600261053457600080fd5b60028151048152505b8151600101808352811415610437575b50506101405160005260206000f3005b63621fd130600051141561063357341561057657600080fd5b60606101c060246380673289610140526002546101605261015c6000305af161059e57600080fd5b6101e0805160200180610260828460006004600a8704601201f16105c157600080fd5b50506102605160206001820306601f82010390506102c0610260516008818352015b826102c05111156105f35761060f565b60006102c05161028001535b81516001018083528114156105e3575b5050506020610240526040610260510160206001820306601f8201039050610240f3005b63c47e300d60005114156113d557606060046101403760506004356004016101a037603060043560040135111561066957600080fd5b604060243560040161022037602060243560040135111561068957600080fd5b60806044356004016102803760606044356004013511156106a957600080fd5b633b9aca0061034052610340516106bf57600080fd5b61034051340461032052633b9aca006103205110156106dd57600080fd5b6060610440602463806732896103c052610320516103e0526103dc6000305af161070657600080fd5b610460805160200180610360828460006004600a8704601201f161072957600080fd5b50506002546104a05260006104c05260026104e05261050060006020818352015b60006104e05161075957600080fd5b6104e0516104a05160016104a05101101561077357600080fd5b60016104a0510106141515610787576107f3565b6104c06060516001825101806040519013156107a257600080fd5b80919012156107b057600080fd5b8152506104e0805115156107c55760006107df565b60028151600283510204146107d957600080fd5b60028151025b8152505b815160010180835281141561074a575b505060006101a06030806020846105e001018260208501600060046016f15050805182019150506000601060208206610560016020828401111561083657600080fd5b60208061058082610520600060046015f15050818152809050905090506010806020846105e001018260208501600060046013f1505080518201915050806105e0526105e09050602060c0825160208401600060025af161089657600080fd5b60c05190506105405260006000604060208206610680016102805182840111156108bf57600080fd5b6060806106a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af16108ff57600080fd5b60c0519050602082610880010152602081019050600060406020602082066107400161028051828401111561093357600080fd5b606080610760826020602088068803016102800160006004601bf150508181528090509050905060208060208461080001018260208501600060046015f15050805182019150506105205160208261080001015260208101905080610800526108009050602060c0825160208401600060025af16109b057600080fd5b60c051905060208261088001015260208101905080610880526108809050602060c0825160208401600060025af16109e757600080fd5b60c051905061066052600060006105405160208261092001015260208101905061022060208060208461092001018260208501600060046015f150508051820191505080610920526109209050602060c0825160208401600060025af1610a4d57600080fd5b60c0519050602082610aa00101526020810190506000610360600880602084610a2001018260208501600060046012f150508051820191505060006018602082066109a00160208284011115610aa257600080fd5b6020806109c082610520600060046015f1505081815280905090509050601880602084610a2001018260208501600060046014f150508051820191505061066051602082610a2001015260208101905080610a2052610a209050602060c0825160208401600060025af1610b1557600080fd5b60c0519050602082610aa001015260208101905080610aa052610aa09050602060c0825160208401600060025af1610b4c57600080fd5b60c051905061090052610b2060006020818352015b6104c051610b20511215610be1576000610b205160208110610b8257600080fd5b600160c052602060c0200154602082610b4001015260208101905061090051602082610b4001015260208101905080610b4052610b409050602060c0825160208401600060025af1610bd357600080fd5b60c051905061090052610be6565b610bf7565b5b8151600101808352811415610b61575b5050610900516104c05160208110610c0e57600080fd5b600160c052602060c02001556002805460018254011015610c2e57600080fd5b60018154018155506020610c40600463c5f2892f610be052610bfc6000305af1610c5757600080fd5b610c4051610bc0526060610ce060246380673289610c60526104a051610c8052610c7c6000305af1610c8857600080fd5b610d00805160200180610d40828460006004600a8704601201f1610cab57600080fd5b505060a0610dc052610dc051610e00526101a0805160200180610dc051610e0001828460006004600a8704601201f1610ce357600080fd5b5050610dc051610e00015160206001820306601f8201039050610dc051610e0001610da081516040818352015b83610da051101515610d2157610d3e565b6000610da0516020850101535b8151600101808352811415610d10575b505050506020610dc051610e00015160206001820306601f8201039050610dc0510101610dc052610dc051610e2052610220805160200180610dc051610e0001828460006004600a8704601201f1610d9557600080fd5b5050610dc051610e00015160206001820306601f8201039050610dc051610e0001610da081516020818352015b83610da051101515610dd357610df0565b6000610da0516020850101535b8151600101808352811415610dc2575b505050506020610dc051610e00015160206001820306601f8201039050610dc0510101610dc052610dc051610e4052610360805160200180610dc051610e0001828460006004600a8704601201f1610e4757600080fd5b5050610dc051610e00015160206001820306601f8201039050610dc051610e0001610da081516020818352015b83610da051101515610e8557610ea2565b6000610da0516020850101535b8151600101808352811415610e74575b505050506020610dc051610e00015160206001820306601f8201039050610dc0510101610dc052610dc051610e6052610280805160200180610dc051610e0001828460006004600a8704601201f1610ef957600080fd5b5050610dc051610e00015160206001820306601f8201039050610dc051610e0001610da081516060818352015b83610da051101515610f3757610f54565b6000610da0516020850101535b8151600101808352811415610f26575b505050506020610dc051610e00015160206001820306601f8201039050610dc0510101610dc052610dc051610e8052610d40805160200180610dc051610e0001828460006004600a8704601201f1610fab57600080fd5b5050610dc051610e00015160206001820306601f8201039050610dc051610e0001610da081516020818352015b83610da051101515610fe957611006565b6000610da0516020850101535b8151600101808352811415610fd8575b505050506020610dc051610e00015160206001820306601f8201039050610dc0510101610dc0527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce610dc051610e00a1640773594000610320511015156113d357600380546001825401101561107b57600080fd5b60018154018155506201000060035414156113d25742610ec05242610ee052620151806110a757600080fd5b62015180610ee05106610ec05110156110bf57600080fd5b42610ee052620151806110d157600080fd5b62015180610ee05106610ec051036202a30042610ec05242610ee052620151806110fa57600080fd5b62015180610ee05106610ec051101561111257600080fd5b42610ee0526201518061112457600080fd5b62015180610ee05106610ec0510301101561113e57600080fd5b6202a30042610ec05242610ee0526201518061115957600080fd5b62015180610ee05106610ec051101561117157600080fd5b42610ee0526201518061118357600080fd5b62015180610ee05106610ec0510301610ea0526060610f8060246380673289610f0052600254610f2052610f1c6000305af16111be57600080fd5b610fa0805160200180610fe0828460006004600a8704601201f16111e157600080fd5b505060606110c06024638067328961104052610ea0516110605261105c6000305af161120c57600080fd5b6110e0805160200180611120828460006004600a8704601201f161122f57600080fd5b5050610bc0516111e05260606111a0526111a05161120052610fe08051602001806111a0516111e001828460006004600a8704601201f161126f57600080fd5b50506111a0516111e0015160206001820306601f82010390506111a0516111e00161118081516020818352015b83611180511015156112ad576112ca565b6000611180516020850101535b815160010180835281141561129c575b5050505060206111a0516111e0015160206001820306601f82010390506111a05101016111a0526111a051611220526111208051602001806111a0516111e001828460006004600a8704601201f161132157600080fd5b50506111a0516111e0015160206001820306601f82010390506111a0516111e00161118081516020818352015b836111805110151561135f5761137c565b6000611180516020850101535b815160010180835281141561134e575b5050505060206111a0516111e0015160206001820306601f82010390506111a05101016111a0527f08b71ef3f1b58f7a23ffb82e27f12f0888c8403f1ceb0ea7ea26b274e2189d4c6111a0516111e0a160016004555b5b005b63845980e860005114156113fb5734156113ee57600080fd5b60045460005260206000f3005b60006000fd5b61020761160803610207600039610207611608036000f3"} \ No newline at end of file +{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "merkle_tree_index", "indexed": false}], "anonymous": false, "type": "event"}, {"name": "Eth2Genesis", "inputs": [{"type": "bytes32", "name": "deposit_root", "indexed": false}, {"type": "bytes", "name": "deposit_count", "indexed": false}, {"type": "bytes", "name": "time", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "to_little_endian_64", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [{"type": "uint256", "name": "value"}], "constant": true, "payable": false, "type": "function", "gas": 7077}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79221}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 11026}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 457042}, {"name": "chainStarted", "outputs": [{"type": "bool", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 603}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600060c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600060c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600060c052602060c02001555b81516001018083528114156100aa575b505061142e56600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a0526380673289600051141561026b57602060046101403734156100b457600080fd5b67ffffffffffffffff6101405111156100cc57600080fd5b60006101605261014051610180526101a060006008818352015b6101605160086000811215610103578060000360020a820461010a565b8060020a82025b905090506101605260ff61018051166101c052610160516101c0516101605101101561013557600080fd5b6101c051610160510161016052610180517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8600081121561017e578060000360020a8204610185565b8060020a82025b90509050610180525b81516001018083528114156100e6575b505060186008602082066101e001602082840111156101bc57600080fd5b60208061020082610160600060046015f15050818152809050905090508051602001806102a0828460006004600a8704601201f16101f957600080fd5b50506102a05160206001820306601f82010390506103006102a0516008818352015b8261030051111561022b57610247565b6000610300516102c001535b815160010180835281141561021b575b50505060206102805260406102a0510160206001820306601f8201039050610280f3005b63c5f2892f60005114156103c357341561028457600080fd5b6000610140526002546101605261018060006020818352015b600160016101605116141561031e57600061018051602081106102bf57600080fd5b600160c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af161031057600080fd5b60c05190506101405261038c565b6000610140516020826101a0010152602081019050610180516020811061034457600080fd5b600060c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161038257600080fd5b60c0519050610140525b610160600261039a57600080fd5b60028151048152505b815160010180835281141561029d575b50506101405160005260206000f3005b63621fd13060005114156104995734156103dc57600080fd5b60606101c060246380673289610140526002546101605261015c6000305af161040457600080fd5b6101e0805160200180610260828460006004600a8704601201f161042757600080fd5b50506102605160206001820306601f82010390506102c0610260516008818352015b826102c051111561045957610475565b60006102c05161028001535b8151600101808352811415610449575b5050506020610240526040610260510160206001820306601f8201039050610240f3005b63c47e300d600051141561127d57606060046101403760506004356004016101a03760306004356004013511156104cf57600080fd5b60406024356004016102203760206024356004013511156104ef57600080fd5b608060443560040161028037606060443560040135111561050f57600080fd5b63ffffffff6002541061052157600080fd5b60306101a0511461053157600080fd5b6020610220511461054157600080fd5b6060610280511461055157600080fd5b633b9aca00610340526103405161056757600080fd5b61034051340461032052633b9aca0061032051101561058557600080fd5b6060610440602463806732896103c052610320516103e0526103dc6000305af16105ae57600080fd5b610460805160200180610360828460006004600a8704601201f16105d157600080fd5b50506002546104a05260006104c05260026104e05261050060006020818352015b60006104e05161060157600080fd5b6104e0516104a05160016104a05101101561061b57600080fd5b60016104a051010614151561062f5761069b565b6104c060605160018251018060405190131561064a57600080fd5b809190121561065857600080fd5b8152506104e08051151561066d576000610687565b600281516002835102041461068157600080fd5b60028151025b8152505b81516001018083528114156105f2575b505060006101a06030806020846105e001018260208501600060046016f1505080518201915050600060106020820661056001602082840111156106de57600080fd5b60208061058082610520600060046015f15050818152809050905090506010806020846105e001018260208501600060046013f1505080518201915050806105e0526105e09050602060c0825160208401600060025af161073e57600080fd5b60c051905061054052600060006040602082066106800161028051828401111561076757600080fd5b6060806106a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af16107a757600080fd5b60c051905060208261088001015260208101905060006040602060208206610740016102805182840111156107db57600080fd5b606080610760826020602088068803016102800160006004601bf150508181528090509050905060208060208461080001018260208501600060046015f15050805182019150506105205160208261080001015260208101905080610800526108009050602060c0825160208401600060025af161085857600080fd5b60c051905060208261088001015260208101905080610880526108809050602060c0825160208401600060025af161088f57600080fd5b60c051905061066052600060006105405160208261092001015260208101905061022060208060208461092001018260208501600060046015f150508051820191505080610920526109209050602060c0825160208401600060025af16108f557600080fd5b60c0519050602082610aa00101526020810190506000610360600880602084610a2001018260208501600060046012f150508051820191505060006018602082066109a0016020828401111561094a57600080fd5b6020806109c082610520600060046015f1505081815280905090509050601880602084610a2001018260208501600060046014f150508051820191505061066051602082610a2001015260208101905080610a2052610a209050602060c0825160208401600060025af16109bd57600080fd5b60c0519050602082610aa001015260208101905080610aa052610aa09050602060c0825160208401600060025af16109f457600080fd5b60c051905061090052610b2060006020818352015b6104c051610b20511215610a89576000610b205160208110610a2a57600080fd5b600160c052602060c0200154602082610b4001015260208101905061090051602082610b4001015260208101905080610b4052610b409050602060c0825160208401600060025af1610a7b57600080fd5b60c051905061090052610a8e565b610a9f565b5b8151600101808352811415610a09575b5050610900516104c05160208110610ab657600080fd5b600160c052602060c02001556002805460018254011015610ad657600080fd5b60018154018155506060610c4060246380673289610bc0526104a051610be052610bdc6000305af1610b0757600080fd5b610c60805160200180610ca0828460006004600a8704601201f1610b2a57600080fd5b505060a0610d2052610d2051610d60526101a0805160200180610d2051610d6001828460006004600a8704601201f1610b6257600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516040818352015b83610d0051101515610ba057610bbd565b6000610d00516020850101535b8151600101808352811415610b8f575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610d8052610220805160200180610d2051610d6001828460006004600a8704601201f1610c1457600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610c5257610c6f565b6000610d00516020850101535b8151600101808352811415610c41575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610da052610360805160200180610d2051610d6001828460006004600a8704601201f1610cc657600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610d0457610d21565b6000610d00516020850101535b8151600101808352811415610cf3575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610dc052610280805160200180610d2051610d6001828460006004600a8704601201f1610d7857600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516060818352015b83610d0051101515610db657610dd3565b6000610d00516020850101535b8151600101808352811415610da5575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610de052610ca0805160200180610d2051610d6001828460006004600a8704601201f1610e2a57600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610e6857610e85565b6000610d00516020850101535b8151600101808352811415610e57575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d20527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce610d2051610d60a16407735940006103205110151561127b576003805460018254011015610efa57600080fd5b600181540181555062010000600354141561127a5742610e205242610e405262015180610f2657600080fd5b62015180610e405106610e20511015610f3e57600080fd5b42610e405262015180610f5057600080fd5b62015180610e405106610e2051036202a30042610e205242610e405262015180610f7957600080fd5b62015180610e405106610e20511015610f9157600080fd5b42610e405262015180610fa357600080fd5b62015180610e405106610e205103011015610fbd57600080fd5b6202a30042610e205242610e405262015180610fd857600080fd5b62015180610e405106610e20511015610ff057600080fd5b42610e40526201518061100257600080fd5b62015180610e405106610e20510301610e00526020610ee0600463c5f2892f610e8052610e9c6000305af161103657600080fd5b610ee051610e60526060610f8060246380673289610f0052600254610f2052610f1c6000305af161106657600080fd5b610fa0805160200180610fe0828460006004600a8704601201f161108957600080fd5b505060606110c06024638067328961104052610e00516110605261105c6000305af16110b457600080fd5b6110e0805160200180611120828460006004600a8704601201f16110d757600080fd5b5050610e60516111e05260606111a0526111a05161120052610fe08051602001806111a0516111e001828460006004600a8704601201f161111757600080fd5b50506111a0516111e0015160206001820306601f82010390506111a0516111e00161118081516020818352015b836111805110151561115557611172565b6000611180516020850101535b8151600101808352811415611144575b5050505060206111a0516111e0015160206001820306601f82010390506111a05101016111a0526111a051611220526111208051602001806111a0516111e001828460006004600a8704601201f16111c957600080fd5b50506111a0516111e0015160206001820306601f82010390506111a0516111e00161118081516020818352015b836111805110151561120757611224565b6000611180516020850101535b81516001018083528114156111f6575b5050505060206111a0516111e0015160206001820306601f82010390506111a05101016111a0527f08b71ef3f1b58f7a23ffb82e27f12f0888c8403f1ceb0ea7ea26b274e2189d4c6111a0516111e0a160016004555b5b005b63845980e860005114156112a357341561129657600080fd5b60045460005260206000f3005b60006000fd5b61018561142e0361018560003961018561142e036000f3"} \ No newline at end of file diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py index 751d3a2e83..82704f59ef 100644 --- a/deposit_contract/contracts/validator_registration.v.py +++ b/deposit_contract/contracts/validator_registration.v.py @@ -4,6 +4,10 @@ DEPOSIT_CONTRACT_TREE_DEPTH: constant(uint256) = 32 SECONDS_PER_DAY: constant(uint256) = 86400 MAX_64_BIT_VALUE: constant(uint256) = 18446744073709551615 # 2**64 - 1 +PUBKEY_LENGTH: constant(uint256) = 48 # bytes +WITHDRAWAL_CREDENTIALS_LENGTH: constant(uint256) = 32 # bytes +SIGNATURE_LENGTH: constant(uint256) = 96 # bytes +MAX_DEPOSIT_COUNT: constant(uint256) = 4294967295 # 2**DEPOSIT_CONTRACT_TREE_DEPTH - 1 Deposit: event({ pubkey: bytes[48], @@ -25,7 +29,6 @@ def __init__(): for i in range(DEPOSIT_CONTRACT_TREE_DEPTH - 1): self.zerohashes[i+1] = sha256(concat(self.zerohashes[i], self.zerohashes[i])) - self.branch[i+1] = self.zerohashes[i + 1] @public @@ -45,18 +48,6 @@ def to_little_endian_64(value: uint256) -> bytes[8]: return slice(convert(y, bytes32), start=24, len=8) -@public -@constant -def from_little_endian_64(value: bytes[8]) -> uint256: - y: uint256 = 0 - x: uint256 = convert(value, uint256) - for i in range(8): - y = y + shift(bitwise_and(x, 255), 8 * (7-i)) - x = shift(x, -8) - - return y - - @public @constant def get_deposit_root() -> bytes32: @@ -77,7 +68,18 @@ def get_deposit_count() -> bytes[8]: @payable @public -def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96]): +def deposit(pubkey: bytes[PUBKEY_LENGTH], + withdrawal_credentials: bytes[WITHDRAWAL_CREDENTIALS_LENGTH], + signature: bytes[SIGNATURE_LENGTH]): + # Prevent edge case in computing `self.branch` when `self.deposit_count == MAX_DEPOSIT_COUNT` + # NOTE: reaching this point with the constants as currently defined is impossible due to the + # uni-directional nature of transfers from eth1 to eth2 and the total ether supply (< 130M). + assert self.deposit_count < MAX_DEPOSIT_COUNT + + assert len(pubkey) == PUBKEY_LENGTH + assert len(withdrawal_credentials) == WITHDRAWAL_CREDENTIALS_LENGTH + assert len(signature) == SIGNATURE_LENGTH + deposit_amount: uint256 = msg.value / as_wei_value(1, "gwei") assert deposit_amount >= MIN_DEPOSIT_AMOUNT amount: bytes[8] = self.to_little_endian_64(deposit_amount) @@ -115,7 +117,6 @@ def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: byt self.branch[i] = value self.deposit_count += 1 - new_deposit_root: bytes32 = self.get_deposit_root() log.Deposit( pubkey, withdrawal_credentials, @@ -132,6 +133,7 @@ def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: byt as_unitless_number(block.timestamp) % SECONDS_PER_DAY + 2 * SECONDS_PER_DAY ) + new_deposit_root: bytes32 = self.get_deposit_root() log.Eth2Genesis(new_deposit_root, self.to_little_endian_64(self.deposit_count), self.to_little_endian_64(timestamp_day_boundary)) diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py index 92371421e2..7c5b85907a 100644 --- a/deposit_contract/tests/contracts/test_deposit.py +++ b/deposit_contract/tests/contracts/test_deposit.py @@ -77,13 +77,6 @@ def test_to_little_endian_64(registration_contract, value, success, assert_tx_fa ) -def test_from_little_endian_64(registration_contract, assert_tx_failed): - values = [0, 2**64 - 1] + [randint(1, 2**64 - 2) for _ in range(10)] - for value in values: - call = registration_contract.functions.from_little_endian_64((value).to_bytes(8, 'little')) - assert call.call() == value - - @pytest.mark.parametrize( 'success,deposit_amount', [ @@ -108,6 +101,43 @@ def test_deposit_amount(registration_contract, ) +@pytest.mark.parametrize( + 'invalid_pubkey,invalid_withdrawal_credentials,invalid_signature,success', + [ + (False, False, False, True), + (True, False, False, False), + (False, True, False, False), + (False, False, True, False), + ] +) +def test_deposit_inputs(registration_contract, + w3, + assert_tx_failed, + deposit_input, + invalid_pubkey, + invalid_withdrawal_credentials, + invalid_signature, + success): + pubkey = deposit_input[0][2:] if invalid_pubkey else deposit_input[0] + if invalid_withdrawal_credentials: # this one is different to satisfy linter + withdrawal_credentials = deposit_input[1][2:] + else: + withdrawal_credentials = deposit_input[1] + signature = deposit_input[2][2:] if invalid_signature else deposit_input[2] + + call = registration_contract.functions.deposit( + pubkey, + withdrawal_credentials, + signature, + ) + if success: + assert call.transact({"value": FULL_DEPOSIT_AMOUNT * eth_utils.denoms.gwei}) + else: + assert_tx_failed( + lambda: call.transact({"value": FULL_DEPOSIT_AMOUNT * eth_utils.denoms.gwei}) + ) + + def test_deposit_log(registration_contract, a0, w3, deposit_input): log_filter = registration_contract.events.Deposit.createFilter( fromBlock='latest', @@ -151,7 +181,7 @@ def test_deposit_tree(registration_contract, w3, assert_tx_failed, deposit_input assert log["merkle_tree_index"] == i.to_bytes(8, 'little') deposit_data = DepositData( - pubkey=deposit_input[0][:20], + pubkey=deposit_input[0], withdrawal_credentials=deposit_input[1], amount=deposit_amount_list[i], signature=deposit_input[2], From 11196ea87d2d67a532d13c83f7cbaf963be09926 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 May 2019 18:33:35 +0800 Subject: [PATCH 193/308] sequentially execute the jobs --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 23d68113b0..100be47047 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -128,4 +128,4 @@ workflows: - test - deposit_contract: requires: - - test + - lint From 3e6f7a2e961183b0f1d65e4bf1a7f694f24574e9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 May 2019 18:42:09 +0800 Subject: [PATCH 194/308] kick --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 100be47047..164dc98a03 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,13 +35,13 @@ commands: description: "Restore the cache with default keys" steps: - restore_cached_venv: - venv_name: v1-pyspec-01 + venv_name: v1-pyspec-05 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}--{{ checksum "deposit_contract/requirements-testing.txt" }}' save_default_cached_venv: description: Save a venv into a cache with default keys" steps: - save_cached_venv: - venv_name: v1-pyspec-01 + venv_name: v1-pyspec-05 reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}--{{ checksum "deposit_contract/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv jobs: @@ -128,4 +128,5 @@ workflows: - test - deposit_contract: requires: + - install_env - lint From 0ec03db0fb9480e19d6b40098cf3b141fdc4ac15 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 May 2019 18:48:40 +0800 Subject: [PATCH 195/308] test --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e40c7e95c9..ee648f8552 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,9 @@ install_deposit_contract_test: test_deposit_contract: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; \ - cd ../.. && cd $(DEPOSIT_CONTRACT_DIR) && python -m pytest . + cd ../.. && cd $(DEPOSIT_CONTRACT_DIR); \ + pip3 install -r requirements-testing.txt; \ + python -m pytest . # "make pyspec" to create the pyspec for all phases. pyspec: $(PY_SPEC_ALL_TARGETS) From e14f789779ad88903f56acb459a03c3b71bfb79c Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Mon, 27 May 2019 08:07:00 -0400 Subject: [PATCH 196/308] Presentation edits --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 6 +++--- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 8b6c20a179..79675fb4ac 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -104,7 +104,7 @@ def hash_tree_root(obj, typ=None): leaf_root = merkleize_chunks(leaves) return mix_in_length(leaf_root, len(obj)) if is_list_type(typ) else leaf_root elif is_container_typ(typ): - leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in zip(obj.get_field_values(), typ.get_field_types())] + leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields().items()] return merkleize_chunks(chunkify(b''.join(leaves))) else: raise Exception("Type not supported: obj {} type {}".format(obj, typ)) @@ -113,11 +113,11 @@ def signing_root(value, typ): if typ is None: typ = infer_type(obj) assert is_container_typ(typ) - leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in zip(obj.get_field_values(), typ.get_field_types())[:-1]] + leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields().items()] return merkleize_chunks(chunkify(b''.join(leaves))) # Implementation notes: -# - SSZContainer,Vector/BytesN.hash_tree_root/serialize functions are for ease, implementation here +# - Container,Vector/BytesN.hash_tree_root/serialize functions are for ease, implementation here # - uint types have a 'byte_len' attribute # - uint types are not classes. They use NewType(), for performance. # This forces us to check type equivalence by exact reference. diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 78c82be45a..76e518e64e 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -30,7 +30,7 @@ # Note: importing ssz functionality locally, to avoid import loop -class SSZContainer(object): +class Container(object): def __init__(self, **kwargs): cls = self.__class__ @@ -291,7 +291,7 @@ def get_zero_value(typ): return typ() if issubclass(typ, bytes): return b'' - if issubclass(typ, SSZContainer): + if issubclass(typ, Container): return typ(**{f: get_zero_value(t) for f, t in typ.get_fields().items()}), # Type helpers @@ -304,7 +304,7 @@ def infer_type(obj): return uint64 elif isinstance(obj, list): return List[infer_type(obj[0])] - elif isinstance(obj, (Vector, SSZContainer, bool, BytesN, bytes)): + elif isinstance(obj, (Vector, Container, bool, BytesN, bytes)): return obj.__class__ else: raise Exception("Unknown type for {}".format(obj)) From 405b34225fd8b9648d7141dd8628719c1287a59f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 27 May 2019 09:37:30 -0600 Subject: [PATCH 197/308] D be list instead of set in validator eth1 data counting --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index d82bf6e560..b9953882dc 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -224,7 +224,7 @@ epoch_signature = bls_sign( `block.eth1_data` is a mechanism used by block proposers vote on a recent Ethereum 1.0 block hash and an associated deposit root found in the Ethereum 1.0 deposit contract. When consensus is formed, `state.latest_eth1_data` is updated, and validator deposits up to this root can be processed. The deposit root can be calculated by calling the `get_deposit_root()` function of the deposit contract using the post-state of the block hash. -* Let `D` be the set of `Eth1DataVote` objects `vote` in `state.eth1_data_votes` where: +* Let `D` be the list of `Eth1DataVote` objects `vote` in `state.eth1_data_votes` where: * `vote.eth1_data.block_hash` is the hash of an Eth 1.0 block that is (i) part of the canonical chain, (ii) >= `ETH1_FOLLOW_DISTANCE` blocks behind the head, and (iii) newer than `state.latest_eth1_data.block_hash`. * `vote.eth1_data.deposit_count` is the deposit count of the Eth 1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. * `vote.eth1_data.deposit_root` is the deposit root of the Eth 1.0 deposit contract at the block defined by `vote.eth1_data.block_hash`. From d5eab257d0a45ba9dd25be8968fe10766751d5b6 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 18:01:46 +0200 Subject: [PATCH 198/308] improve impl, box less used integer types, use uint64 as default --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 48 +++---- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 127 +++++++++++++----- 2 files changed, 115 insertions(+), 60 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 79675fb4ac..4b8545b17d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -6,18 +6,21 @@ BYTES_PER_LENGTH_OFFSET = 4 + def is_basic_type(typ): return is_uint(typ) or typ == bool + def serialize_basic(value, typ): if is_uint(typ): - return value.to_bytes(typ.byte_len, 'little') + return value.to_bytes(uint_byte_size(typ), 'little') if issubclass(typ, bool): if value: return b'\x01' else: return b'\x00' + def is_fixed_size(typ): if is_basic_type(typ): return True @@ -30,9 +33,9 @@ def is_fixed_size(typ): else: raise Exception("Type not supported: {}".format(typ)) -def serialize(obj, typ=None): - if typ is None: - typ = infer_type(obj) + +@infer_input_type +def serialize(obj, typ): if is_basic_type(typ): return serialize_basic(obj, typ) elif is_list_type(typ) or is_vector_type(typ): @@ -42,6 +45,7 @@ def serialize(obj, typ=None): else: raise Exception("Type not supported: {}".format(typ)) + def encode_series(values, types): # bytes and bytesN are already in the right format. if isinstance(values, bytes): @@ -75,24 +79,28 @@ def encode_series(values, types): # Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts return b''.join(fixed_parts + variable_parts) + # SSZ Hash-tree-root # ----------------------------- + def pack(values, subtype): if isinstance(values, bytes): return values return b''.join([serialize_basic(value, subtype) for value in values]) + def chunkify(bytez): bytez += b'\x00' * (-len(bytez) % 32) return [bytez[i:i + 32] for i in range(0, len(bytez), 32)] + def mix_in_length(root, length): return hash(root + length.to_bytes(32, 'little')) -def hash_tree_root(obj, typ=None): - if typ is None: - typ = infer_type(obj) + +@infer_input_type +def hash_tree_root(obj, typ): if is_basic_type(typ): return merkleize_chunks(chunkify(serialize_basic(obj, typ))) elif is_list_type(typ) or is_vector_type(typ): @@ -104,31 +112,15 @@ def hash_tree_root(obj, typ=None): leaf_root = merkleize_chunks(leaves) return mix_in_length(leaf_root, len(obj)) if is_list_type(typ) else leaf_root elif is_container_typ(typ): - leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields().items()] + leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()] return merkleize_chunks(chunkify(b''.join(leaves))) else: raise Exception("Type not supported: obj {} type {}".format(obj, typ)) -def signing_root(value, typ): - if typ is None: - typ = infer_type(obj) + +@infer_input_type +def signing_root(obj, typ): assert is_container_typ(typ) - leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields().items()] + leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) -# Implementation notes: -# - Container,Vector/BytesN.hash_tree_root/serialize functions are for ease, implementation here -# - uint types have a 'byte_len' attribute -# - uint types are not classes. They use NewType(), for performance. -# This forces us to check type equivalence by exact reference. -# There's no class. The type data comes from an annotation/argument from the context of the value. -# - Vector is not valid to create instances with. Give it a elem-type and length: Vector[FooBar, 123] -# - *The class of* a Vector instance has a `elem_type` (type, may not be a class, see uint) and `length` (int) -# - BytesN is not valid to create instances with. Give it a length: BytesN[123] -# - *The class of* a BytesN instance has a `length` (int) -# Where possible, it is preferable to create helpers that just act on the type, and don't unnecessarily use a value -# E.g. is_basic_type(). This way, we can use them in type-only contexts and have no duplicate logic. -# For every class-instance, you can get the type with my_object.__class__ -# For uints, and other NewType related, you have to rely on type information. It cannot be retrieved from the value. -# Note: we may just want to box integers instead. And then we can do bounds checking too. But it is SLOW and MEMORY INTENSIVE. -# diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 76e518e64e..f117d94008 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,30 +1,82 @@ -from typing import List, Iterable, TypeVar, Type, NewType +from typing import List, Iterable, Type, NewType from typing import Union from inspect import isclass -T = TypeVar('T') -L = TypeVar('L') +# SSZ integers +# ----------------------------- +class uint(int): + byte_len = 0 + def __new__(cls, value, *args, **kwargs): + if value < 0: + raise ValueError("unsigned types must not be negative") + return super().__new__(cls, value) -# SSZ integer types, with 0 computational overhead (NewType) -# ----------------------------- -uint8 = NewType('uint8', int) -uint8.byte_len = 1 -uint16 = NewType('uint16', int) -uint16.byte_len = 2 -uint32 = NewType('uint32', int) -uint32.byte_len = 4 -uint64 = NewType('uint64', int) -uint64.byte_len = 8 -uint128 = NewType('uint128', int) -uint128.byte_len = 16 -uint256 = NewType('uint256', int) -uint256.byte_len = 32 +class uint8(uint): + byte_len = 1 + def __new__(cls, value, *args, **kwargs): + if value.bit_length() > 8: + raise ValueError("value out of bounds for uint8") + return super().__new__(cls, value) + +# Alias for uint8 byte = NewType('byte', uint8) +class uint16(uint): + byte_len = 2 + def __new__(cls, value, *args, **kwargs): + if value.bit_length() > 16: + raise ValueError("value out of bounds for uint16") + return super().__new__(cls, value) + + +class uint32(uint): + byte_len = 4 + def __new__(cls, value, *args, **kwargs): + if value.bit_length() > 32: + raise ValueError("value out of bounds for uint16") + return super().__new__(cls, value) + + +# We simply default to uint64. But do give it a name, for readability +uint64 = NewType('uint64', int) + + +class uint128(uint): + byte_len = 16 + def __new__(cls, value, *args, **kwargs): + if value.bit_length() > 128: + raise ValueError("value out of bounds for uint128") + return super().__new__(cls, value) + + +class uint256(uint): + byte_len = 32 + def __new__(cls, value, *args, **kwargs): + if value.bit_length() > 256: + raise ValueError("value out of bounds for uint256") + return super().__new__(cls, value) + + +def is_uint(typ): + # All integers are uint in the scope of the spec here. + # Since we default to uint64. Bounds can be checked elsewhere. + return issubclass(typ, int) + + +def uint_byte_size(typ): + if issubclass(typ, uint): + return typ.byte_len + elif issubclass(typ, int): + # Default to uint64 + return 8 + else: + raise TypeError("Type %s is not an uint (or int-default uint64) type" % typ) + + # SSZ Container base class # ----------------------------- @@ -57,9 +109,13 @@ def get_field_values(self): return [getattr(self, field) for field in cls.get_field_names()] @classmethod - def get_fields(cls): + def get_fields_dict(cls): return dict(cls.__annotations__) + @classmethod + def get_fields(cls): + return dict(cls.__annotations__).items() + @classmethod def get_field_names(cls): return list(cls.__annotations__.keys()) @@ -70,14 +126,6 @@ def get_field_types(cls): return list(cls.__annotations__.values()) -def is_uint(typ): - # Note: only the type reference exists, - # but it really resolves to 'int' during run-time for zero computational/memory overhead. - # Hence, we check equality to the type references (which are really just 'NewType' instances), - # and don't use any sub-classing like we normally would. - return typ == uint8 or typ == uint16 or typ == uint32 or typ == uint64 \ - or typ == uint128 or typ == uint256 or typ == byte - # SSZ vector # ----------------------------- @@ -138,7 +186,7 @@ def __ne__(self, other): class Vector(metaclass=VectorMeta): - def __init__(self, *args: Iterable[T]): + def __init__(self, *args: Iterable): cls = self.__class__ if not hasattr(cls, 'elem_type'): @@ -275,6 +323,7 @@ def hash_tree_root(self): from .ssz_impl import hash_tree_root return hash_tree_root(self, self.__class__) + # SSZ Defaults # ----------------------------- @@ -292,7 +341,8 @@ def get_zero_value(typ): if issubclass(typ, bytes): return b'' if issubclass(typ, Container): - return typ(**{f: get_zero_value(t) for f, t in typ.get_fields().items()}), + return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}), + # Type helpers # ----------------------------- @@ -309,17 +359,30 @@ def infer_type(obj): else: raise Exception("Unknown type for {}".format(obj)) + +def infer_input_type(fn): + """ + Decorator to run infer_type on the obj if typ argument is None + """ + def infer_helper(obj, typ=None): + if typ is None: + typ = infer_type(obj) + return fn(obj, typ) + return infer_helper + + def is_list_type(typ): return (hasattr(typ, '_name') and typ._name == 'List') or typ == bytes def is_vector_type(typ): - return hasattr(typ, '_name') and typ._name == 'Vector' + return issubclass(typ, Vector) def is_container_typ(typ): - return hasattr(typ, 'get_fields') + return issubclass(typ, Container) def read_list_elem_typ(list_typ: Type[List[T]]) -> T: - assert list_typ.__args__ is not None + if list_typ.__args__ is None or len(list_typ.__args__) != 1: + raise TypeError("Supplied list-type is invalid, no element type found.") return list_typ.__args__[0] def read_vector_elem_typ(vector_typ: Type[Vector[T, L]]) -> T: @@ -333,4 +396,4 @@ def read_elem_typ(typ): elif is_vector_type(typ): return read_vector_elem_typ(typ) else: - raise Exception("Unexpected type: {}".format(typ)) + raise TypeError("Unexpected type: {}".format(typ)) From f3088884b3c6441271245982fd9c6050d3a3d581 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 20:29:53 +0200 Subject: [PATCH 199/308] Minor adjustments + getting the beacon spec doc ready --- specs/core/0_beacon-chain.md | 128 +++++++++--------- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 6 + 3 files changed, 71 insertions(+), 65 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 8442819c44..cae9dc55d9 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -268,11 +268,11 @@ The types are defined topologically to aid in facilitating an executable version #### `Fork` ```python -class Fork(SSZContainer): +class Fork(Container): # Previous fork version - previous_version: bytes4 + previous_version: Bytes4 # Current fork version - current_version: bytes4 + current_version: Bytes4 # Fork epoch number epoch: uint64 ``` @@ -280,41 +280,41 @@ class Fork(SSZContainer): #### `Crosslink` ```python -class Crosslink(SSZContainer): +class Crosslink(Container): # Shard number shard: uint64 # Epoch number epoch: uint64 # Root of the previous crosslink - parent_root: bytes32 + parent_root: Bytes32 # Root of the crosslinked shard data since the previous crosslink - data_root: bytes32 + data_root: Bytes32 ``` #### `Eth1Data` ```python -class Eth1Data(SSZContainer): +class Eth1Data(Container): # Root of the deposit tree - deposit_root: bytes32 + deposit_root: Bytes32 # Total number of deposits deposit_count: uint64 # Block hash - block_hash: bytes32 + block_hash: Bytes32 ``` #### `AttestationData` ```python -class AttestationData(SSZContainer): +class AttestationData(Container): # LMD GHOST vote - beacon_block_root: bytes32 + beacon_block_root: Bytes32 # FFG vote source_epoch: uint64 - source_root: bytes32 + source_root: Bytes32 target_epoch: uint64 - target_root: bytes32 + target_root: Bytes32 # Crosslink vote crosslink: Crosslink @@ -323,7 +323,7 @@ class AttestationData(SSZContainer): #### `AttestationDataAndCustodyBit` ```python -class AttestationDataAndCustodyBit(SSZContainer): +class AttestationDataAndCustodyBit(Container): # Attestation data data: AttestationData # Custody bit @@ -333,48 +333,48 @@ class AttestationDataAndCustodyBit(SSZContainer): #### `IndexedAttestation` ```python -class IndexedAttestation(SSZContainer): +class IndexedAttestation(Container): # Validator indices custody_bit_0_indices: List[uint64] custody_bit_1_indices: List[uint64] # Attestation data data: AttestationData # Aggregate signature - signature: bytes96 + signature: Bytes96 ``` #### `DepositData` ```python -class DepositData(SSZContainer): +class DepositData(Container): # BLS pubkey - pubkey: bytes48 + pubkey: Bytes48 # Withdrawal credentials - withdrawal_credentials: bytes32 + withdrawal_credentials: Bytes32 # Amount in Gwei amount: uint64 # Container self-signature - signature: bytes96 + signature: Bytes96 ``` #### `BeaconBlockHeader` ```python -class BeaconBlockHeader(SSZContainer): +class BeaconBlockHeader(Container): slot: uint64 - parent_root: bytes32 - state_root: bytes32 - body_root: bytes32 - signature: bytes96 + parent_root: Bytes32 + state_root: Bytes32 + body_root: Bytes32 + signature: Bytes96 ``` #### `Validator` ```python -class Validator(SSZContainer): +class Validator(Container): # BLS public key - pubkey: bytes48 + pubkey: Bytes48 # Withdrawal credentials - withdrawal_credentials: bytes32 + withdrawal_credentials: Bytes32 # Epoch when became eligible for activation activation_eligibility_epoch: uint64 # Epoch when validator activated @@ -392,7 +392,7 @@ class Validator(SSZContainer): #### `PendingAttestation` ```python -class PendingAttestation(SSZContainer): +class PendingAttestation(Container): # Attester aggregation bitfield aggregation_bitfield: bytes # Attestation data @@ -406,11 +406,11 @@ class PendingAttestation(SSZContainer): #### `HistoricalBatch` ```python -class HistoricalBatch(SSZContainer): +class HistoricalBatch(Container): # Block roots - block_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] + block_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] # State roots - state_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] ``` ### Beacon operations @@ -418,7 +418,7 @@ class HistoricalBatch(SSZContainer): #### `ProposerSlashing` ```python -class ProposerSlashing(SSZContainer): +class ProposerSlashing(Container): # Proposer index proposer_index: uint64 # First block header @@ -430,7 +430,7 @@ class ProposerSlashing(SSZContainer): #### `AttesterSlashing` ```python -class AttesterSlashing(SSZContainer): +class AttesterSlashing(Container): # First attestation attestation_1: IndexedAttestation # Second attestation @@ -440,7 +440,7 @@ class AttesterSlashing(SSZContainer): #### `Attestation` ```python -class Attestation(SSZContainer): +class Attestation(Container): # Attester aggregation bitfield aggregation_bitfield: bytes # Attestation data @@ -448,15 +448,15 @@ class Attestation(SSZContainer): # Custody bitfield custody_bitfield: bytes # BLS aggregate signature - signature: bytes96 + signature: Bytes96 ``` #### `Deposit` ```python -class Deposit(SSZContainer): +class Deposit(Container): # Branch in the deposit tree - proof: Vector[bytes32, DEPOSIT_CONTRACT_TREE_DEPTH] + proof: Vector[Bytes32, DEPOSIT_CONTRACT_TREE_DEPTH] # Index in the deposit tree index: uint64 # Data @@ -466,19 +466,19 @@ class Deposit(SSZContainer): #### `VoluntaryExit` ```python -class VoluntaryExit(SSZContainer): +class VoluntaryExit(Container): # Minimum epoch for processing exit epoch: uint64 # Index of the exiting validator validator_index: uint64 # Validator signature - signature: bytes96 + signature: Bytes96 ``` #### `Transfer` ```python -class Transfer(SSZContainer): +class Transfer(Container): # Sender index sender: uint64 # Recipient index @@ -490,9 +490,9 @@ class Transfer(SSZContainer): # Inclusion slot slot: uint64 # Sender withdrawal pubkey - pubkey: bytes48 + pubkey: Bytes48 # Sender signature - signature: bytes96 + signature: Bytes96 ``` ### Beacon blocks @@ -500,10 +500,10 @@ class Transfer(SSZContainer): #### `BeaconBlockBody` ```python -class BeaconBlockBody(SSZContainer): - randao_reveal: bytes96 +class BeaconBlockBody(Container): + randao_reveal: Bytes96 eth1_data: Eth1Data - graffiti: bytes32 + graffiti: Bytes32 proposer_slashings: List[ProposerSlashing] attester_slashings: List[AttesterSlashing] attestations: List[Attestation] @@ -515,13 +515,13 @@ class BeaconBlockBody(SSZContainer): #### `BeaconBlock` ```python -class BeaconBlock(SSZContainer): +class BeaconBlock(Container): # Header slot: uint64 - parent_root: bytes32 - state_root: bytes32 + parent_root: Bytes32 + state_root: Bytes32 body: BeaconBlockBody - signature: bytes96 + signature: Bytes96 ``` ### Beacon state @@ -529,7 +529,7 @@ class BeaconBlock(SSZContainer): #### `BeaconState` ```python -class BeaconState(SSZContainer): +class BeaconState(Container): # Misc slot: uint64 genesis_time: uint64 @@ -540,7 +540,7 @@ class BeaconState(SSZContainer): balances: List[uint64] # Randomness and committees - latest_randao_mixes: Vector[bytes32, LATEST_RANDAO_MIXES_LENGTH] + latest_randao_mixes: Vector[Bytes32, LATEST_RANDAO_MIXES_LENGTH] latest_start_shard: uint64 # Finality @@ -548,23 +548,23 @@ class BeaconState(SSZContainer): current_epoch_attestations: List[PendingAttestation] previous_justified_epoch: uint64 current_justified_epoch: uint64 - previous_justified_root: bytes32 - current_justified_root: bytes32 + previous_justified_root: Bytes32 + current_justified_root: Bytes32 justification_bitfield: uint64 finalized_epoch: uint64 - finalized_root: bytes32 + finalized_root: Bytes32 # Recent state current_crosslinks: Vector[Crosslink, SHARD_COUNT] previous_crosslinks: Vector[Crosslink, SHARD_COUNT] - latest_block_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] - latest_state_roots: Vector[bytes32, SLOTS_PER_HISTORICAL_ROOT] - latest_active_index_roots: Vector[bytes32, LATEST_ACTIVE_INDEX_ROOTS_LENGTH] + latest_block_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] + latest_state_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] + latest_active_index_roots: Vector[Bytes32, LATEST_ACTIVE_INDEX_ROOTS_LENGTH] # Balances slashed at every withdrawal period latest_slashed_balances: Vector[uint64, LATEST_SLASHED_EXIT_LENGTH] # `latest_block_header.state_root == ZERO_HASH` temporarily latest_block_header: BeaconBlockHeader - historical_roots: List[bytes32] + historical_roots: List[Bytes32] # Ethereum 1.0 chain data latest_eth1_data: Eth1Data @@ -583,9 +583,9 @@ We define the following Python custom types for type hinting and readability: | `Shard` | `uint64` | a shard number | | `ValidatorIndex` | `uint64` | a validator registry index | | `Gwei` | `uint64` | an amount in Gwei | -| `Bytes32` | `bytes32` | 32 bytes of binary data | -| `BLSPubkey` | `bytes48` | a BLS12-381 public key | -| `BLSSignature` | `bytes96` | a BLS12-381 signature | +| `Bytes32` | `Bytes32` | 32 bytes of binary data | +| `BLSPubkey` | `Bytes48` | a BLS12-381 public key | +| `BLSSignature` | `Bytes96` | a BLS12-381 signature | ## Helper functions @@ -595,7 +595,7 @@ We define the following Python custom types for type hinting and readability: ```python def xor(bytes1: Bytes32, bytes2: Bytes32) -> Bytes32: - return bytes(a ^ b for a, b in zip(bytes1, bytes2)) + return Bytes32(a ^ b for a, b in zip(bytes1, bytes2)) ``` ### `hash` @@ -610,7 +610,7 @@ The `hash` function is SHA256. ### `signing_root` -`def signing_root(object: SSZContainer) -> Bytes32` is a function defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers) to compute signing messages. +`def signing_root(object: Container) -> Bytes32` is a function defined in the [SimpleSerialize spec](../simple-serialize.md#self-signed-containers) to compute signing messages. ### `slot_to_epoch` diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 4b8545b17d..655cd9da42 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -29,7 +29,7 @@ def is_fixed_size(typ): elif is_vector_type(typ): return is_fixed_size(read_vector_elem_typ(typ)) elif is_container_typ(typ): - return all([is_fixed_size(t) for t in typ.get_field_types()]) + return all(is_fixed_size(t) for t in typ.get_field_types()) else: raise Exception("Type not supported: {}".format(typ)) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index f117d94008..1d8aed0ded 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -108,6 +108,9 @@ def get_field_values(self): cls = self.__class__ return [getattr(self, field) for field in cls.get_field_names()] + def __repr__(self): + return {field: getattr(self, field) for field in self.get_field_names()} + @classmethod def get_fields_dict(cls): return dict(cls.__annotations__) @@ -217,6 +220,9 @@ def hash_tree_root(self): from .ssz_impl import hash_tree_root return hash_tree_root(self, self.__class__) + def __repr__(self): + return {'length': self.__class__.length, 'items': self.items} + def __getitem__(self, key): return self.items[key] From 132d3c976a0529d358363d0106960b05da7285dd Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 21:14:59 +0200 Subject: [PATCH 200/308] fix spec builder --- scripts/phase0/build_spec.py | 18 ++++++----- scripts/phase0/function_puller.py | 50 +++++++++++++++++++++---------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index b188dada25..59d9e9f9df 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -16,7 +16,10 @@ def build_phase0_spec(sourcefile, outfile): hash_tree_root, signing_root, ) -from eth2spec.utils.ssz.ssz_typing import * +from eth2spec.utils.ssz.ssz_typing import ( + uint8, uint16, uint32, uint64, uint128, uint256, + Container, Vector, BytesN +) from eth2spec.utils.bls_stub import ( bls_aggregate_pubkeys, bls_verify, @@ -24,19 +27,18 @@ def build_phase0_spec(sourcefile, outfile): ) from eth2spec.utils.hash_function import hash - -# stub, will get overwritten by real var -SLOTS_PER_EPOCH = 64 - +# Note: 'int' type defaults to being interpreted as a uint64 by SSZ implementation. Slot = NewType('Slot', int) # uint64 Epoch = NewType('Epoch', int) # uint64 Shard = NewType('Shard', int) # uint64 ValidatorIndex = NewType('ValidatorIndex', int) # uint64 Gwei = NewType('Gwei', int) # uint64 + +Bytes4 = BytesN[4] Bytes32 = BytesN[32] -BLSPubkey = NewType('BLSPubkey', BytesN[48]) -BLSSignature = NewType('BLSSignature', BytesN[96]) -Store = None +Bytes48 = BytesN[48] +Bytes96 = BytesN[96] + """) code_lines += function_puller.get_spec(sourcefile) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 5c1cf859bb..0047d53212 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -1,4 +1,3 @@ -import sys from typing import List @@ -6,9 +5,11 @@ def get_spec(file_name: str) -> List[str]: code_lines = [] pulling_from = None current_name = None + # list of current type definition being parsed, or None otherwise current_typedef = None + # list of (name, definition lines list) tuples. type_defs = [] - for linenum, line in enumerate(open(sys.argv[1]).readlines()): + for linenum, line in enumerate(open(file_name).readlines()): line = line.rstrip() if pulling_from is None and len(line) > 0 and line[0] == '#' and line[-1] == '`': current_name = line[line[:-1].rfind('`') + 1: -1] @@ -19,22 +20,24 @@ def get_spec(file_name: str) -> List[str]: if pulling_from is None: pulling_from = linenum else: - if current_typedef is not None: - assert code_lines[-1] == '}' - code_lines[-1] = '' - code_lines.append('') pulling_from = None - current_typedef = None + if current_typedef is not None: + type_defs.append((current_name, current_typedef)) + current_typedef = None else: - if pulling_from == linenum and line == '{': - code_lines.append('class %s(SSZContainer):' % current_name) - current_typedef = current_name - type_defs.append(current_name) - elif pulling_from is not None: + if pulling_from is not None: # Add some whitespace between functions - if line[:3] == 'def': + if line[:3] == 'def' or line[:5] == 'class': code_lines.append('') code_lines.append('') + # Check for SSZ type definitions + if len(line) > 18 and line[:6] == 'class ' and line[-12:] == '(Container):': + name = line[6:-12] + # Check consistency with markdown header + assert name == current_name + current_typedef = [] + if current_typedef is not None: + current_typedef.append(line) code_lines.append(line) elif pulling_from is None and len(line) > 0 and line[0] == '|': row = line[1:].split('|') @@ -54,11 +57,26 @@ def get_spec(file_name: str) -> List[str]: # Build type-def re-initialization code_lines.append('\n') code_lines.append('ssz_types = [\n') - for ssz_type_name in type_defs: - code_lines.append(f' {ssz_type_name},\n') + for (ssz_type_name, _) in type_defs: + code_lines.append(f' {ssz_type_name},') code_lines.append(']') code_lines.append('\n') - code_lines.append('def get_ssz_type_by_name(name: str) -> SSZType:') + code_lines.append('def init_SSZ_types():') + code_lines.append(' global_vars = globals()') + for ssz_type_name, ssz_type in type_defs: + code_lines.append('') + for type_line in ssz_type: + if len(type_line) > 0: + code_lines.append(' ' + type_line) + code_lines.append('\n') + for (ssz_type_name, _) in type_defs: + code_lines.append(f' global_vars["{ssz_type_name}"] = {ssz_type_name},') + code_lines.append(' global_vars["ssz_types"] = [') + for (ssz_type_name, _) in type_defs: + code_lines.append(f' {ssz_type_name},') + code_lines.append(' ]') + code_lines.append('\n') + code_lines.append('def get_ssz_type_by_name(name: str) -> Container:') code_lines.append(' return globals()[name]') code_lines.append('') return code_lines From d63b553a2da3f5293d7202183c18cfdf28816b7f Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 21:45:47 +0200 Subject: [PATCH 201/308] efficiency bugfix in bytes encoding, improve typing doc + bugfix --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 4 ++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 655cd9da42..46d842e168 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -14,7 +14,7 @@ def is_basic_type(typ): def serialize_basic(value, typ): if is_uint(typ): return value.to_bytes(uint_byte_size(typ), 'little') - if issubclass(typ, bool): + if is_bool_type(typ): if value: return b'\x01' else: @@ -39,7 +39,7 @@ def serialize(obj, typ): if is_basic_type(typ): return serialize_basic(obj, typ) elif is_list_type(typ) or is_vector_type(typ): - return encode_series(list(obj), [read_elem_typ(typ)]*len(obj)) + return encode_series(obj, [read_elem_typ(typ)]*len(obj)) elif is_container_typ(typ): return encode_series(obj.get_field_values(), typ.get_field_types()) else: diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 1d8aed0ded..93a272c8c5 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -376,12 +376,20 @@ def infer_helper(obj, typ=None): return fn(obj, typ) return infer_helper +def is_bool_type(typ): + return issubclass(typ, bool) def is_list_type(typ): + """ + Checks if the given type is a kind of list. Can be bytes. + """ return (hasattr(typ, '_name') and typ._name == 'List') or typ == bytes def is_vector_type(typ): - return issubclass(typ, Vector) + """ + Checks if the given type is a kind of vector. Can be BytesN. + """ + return issubclass(typ, Vector) or issubclass(typ, BytesN) def is_container_typ(typ): return issubclass(typ, Container) From 87b3466eae90ebe557899083ca50e3babc29bb6f Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 21:46:14 +0200 Subject: [PATCH 202/308] update encoder and decoder for reading from parsed data --- test_libs/pyspec/eth2spec/debug/decode.py | 45 +++++++++++++---------- test_libs/pyspec/eth2spec/debug/encode.py | 25 +++++++------ 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index e9aa8bc2b4..261692bedb 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,28 +1,35 @@ -from eth2spec.utils.minimal_ssz import hash_tree_root +from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_typing import * -def decode(json, typ): - if isinstance(typ, str) and typ[:4] == 'uint': - return json - elif typ == 'bool': - assert json in (True, False) - return json - elif isinstance(typ, list): - return [decode(element, typ[0]) for element in json] - elif isinstance(typ, str) and typ[:4] == 'byte': - return bytes.fromhex(json[2:]) - elif hasattr(typ, 'fields'): +def decode(data, typ): + if is_uint(typ): + return data + elif is_bool_type(typ): + assert data in (True, False) + return data + elif issubclass(typ, list): + elem_typ = read_list_elem_typ(typ) + return [decode(element, elem_typ) for element in data] + elif issubclass(typ, Vector): + elem_typ = read_vector_elem_typ(typ) + return Vector(decode(element, elem_typ) for element in data) + elif issubclass(typ, bytes): + return bytes.fromhex(data[2:]) + elif issubclass(typ, BytesN): + return BytesN(bytes.fromhex(data[2:])) + elif is_container_typ(typ): temp = {} - for field, subtype in typ.fields.items(): - temp[field] = decode(json[field], subtype) - if field + "_hash_tree_root" in json: - assert(json[field + "_hash_tree_root"][2:] == + for field, subtype in typ.get_fields(): + temp[field] = decode(data[field], subtype) + if field + "_hash_tree_root" in data: + assert(data[field + "_hash_tree_root"][2:] == hash_tree_root(temp[field], subtype).hex()) ret = typ(**temp) - if "hash_tree_root" in json: - assert(json["hash_tree_root"][2:] == + if "hash_tree_root" in data: + assert(data["hash_tree_root"][2:] == hash_tree_root(ret, typ).hex()) return ret else: - print(json, typ) + print(data, typ) raise Exception("Type not recognized") diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index b38e5fe98b..3c0658c8fc 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,24 +1,27 @@ -from eth2spec.utils.minimal_ssz import hash_tree_root +from eth2spec.utils.ssz.ssz_impl import hash_tree_root +from eth2spec.utils.ssz.ssz_typing import * def encode(value, typ, include_hash_tree_roots=False): - if isinstance(typ, str) and typ[:4] == 'uint': - if typ[4:] == '128' or typ[4:] == '256': + if is_uint(typ): + if issubclass(typ, uint) and typ.byte_len > 8: return str(value) return value - elif typ == 'bool': + elif is_bool_type(typ): assert value in (True, False) return value - elif isinstance(typ, list): - return [encode(element, typ[0], include_hash_tree_roots) for element in value] - elif isinstance(typ, str) and typ[:4] == 'byte': + elif issubclass(typ, list) or issubclass(typ, Vector): + elem_typ = read_elem_typ(typ) + return [encode(element, elem_typ, include_hash_tree_roots) for element in value] + elif issubclass(typ, bytes): return '0x' + value.hex() - elif hasattr(typ, 'fields'): + elif is_container_typ(typ): ret = {} - for field, subtype in typ.fields.items(): - ret[field] = encode(getattr(value, field), subtype, include_hash_tree_roots) + for field, subtype in typ.get_fields(): + field_value = getattr(value, field) + ret[field] = encode(field_value, subtype, include_hash_tree_roots) if include_hash_tree_roots: - ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(getattr(value, field), subtype).hex() + ret[field + "_hash_tree_root"] = '0x' + hash_tree_root(field_value, subtype).hex() if include_hash_tree_roots: ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex() return ret From c68944bd53cbe8759c233a188f3e58943012e84c Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 22:18:34 +0200 Subject: [PATCH 203/308] separate type (direct) from kinds (alias incl) --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 26 +++++++-------- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 33 +++++++++++++++---- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 46d842e168..c3cc579bdd 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -8,11 +8,11 @@ def is_basic_type(typ): - return is_uint(typ) or typ == bool + return is_uint_type(typ) or is_bool_type(typ) def serialize_basic(value, typ): - if is_uint(typ): + if is_uint_type(typ): return value.to_bytes(uint_byte_size(typ), 'little') if is_bool_type(typ): if value: @@ -24,11 +24,11 @@ def serialize_basic(value, typ): def is_fixed_size(typ): if is_basic_type(typ): return True - elif is_list_type(typ): + elif is_list_kind(typ): return False - elif is_vector_type(typ): - return is_fixed_size(read_vector_elem_typ(typ)) - elif is_container_typ(typ): + elif is_vector_kind(typ): + return is_fixed_size(read_vector_elem_type(typ)) + elif is_container_type(typ): return all(is_fixed_size(t) for t in typ.get_field_types()) else: raise Exception("Type not supported: {}".format(typ)) @@ -38,9 +38,9 @@ def is_fixed_size(typ): def serialize(obj, typ): if is_basic_type(typ): return serialize_basic(obj, typ) - elif is_list_type(typ) or is_vector_type(typ): - return encode_series(obj, [read_elem_typ(typ)]*len(obj)) - elif is_container_typ(typ): + elif is_list_kind(typ) or is_vector_kind(typ): + return encode_series(obj, [read_elem_type(typ)]*len(obj)) + elif is_container_type(typ): return encode_series(obj.get_field_values(), typ.get_field_types()) else: raise Exception("Type not supported: {}".format(typ)) @@ -103,15 +103,15 @@ def mix_in_length(root, length): def hash_tree_root(obj, typ): if is_basic_type(typ): return merkleize_chunks(chunkify(serialize_basic(obj, typ))) - elif is_list_type(typ) or is_vector_type(typ): - subtype = read_elem_typ(typ) + elif is_list_kind(typ) or is_vector_kind(typ): + subtype = read_elem_type(typ) if is_basic_type(subtype): leaves = chunkify(pack(obj, subtype)) else: leaves = [hash_tree_root(elem, subtype) for elem in obj] leaf_root = merkleize_chunks(leaves) return mix_in_length(leaf_root, len(obj)) if is_list_type(typ) else leaf_root - elif is_container_typ(typ): + elif is_container_type(typ): leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()] return merkleize_chunks(chunkify(b''.join(leaves))) else: @@ -120,7 +120,7 @@ def hash_tree_root(obj, typ): @infer_input_type def signing_root(obj, typ): - assert is_container_typ(typ) + assert is_container_type(typ) leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 93a272c8c5..d16c66abb5 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -61,7 +61,7 @@ def __new__(cls, value, *args, **kwargs): return super().__new__(cls, value) -def is_uint(typ): +def is_uint_type(typ): # All integers are uint in the scope of the spec here. # Since we default to uint64. Bounds can be checked elsewhere. return issubclass(typ, int) @@ -380,29 +380,48 @@ def is_bool_type(typ): return issubclass(typ, bool) def is_list_type(typ): + """ + Checks if the given type is a list. + """ + return (hasattr(typ, '_name') and typ._name == 'List') + +def is_bytes_type(typ): + # Do not accept subclasses of bytes here, to avoid confusion with BytesN + return typ == bytes + +def is_list_kind(typ): """ Checks if the given type is a kind of list. Can be bytes. """ - return (hasattr(typ, '_name') and typ._name == 'List') or typ == bytes + return is_list_type(typ) or is_bytes_type(typ) def is_vector_type(typ): + """ + Checks if the given type is a vector. + """ + return issubclass(typ, Vector) + +def is_bytesn_type(typ): + return issubclass(typ, BytesN) + +def is_vector_kind(typ): """ Checks if the given type is a kind of vector. Can be BytesN. """ - return issubclass(typ, Vector) or issubclass(typ, BytesN) + return is_vector_type(typ) or is_bytesn_type(typ) -def is_container_typ(typ): +def is_container_type(typ): return issubclass(typ, Container) -def read_list_elem_typ(list_typ: Type[List[T]]) -> T: +def read_list_elem_type(list_typ: Type[List[T]]) -> T: if list_typ.__args__ is None or len(list_typ.__args__) != 1: raise TypeError("Supplied list-type is invalid, no element type found.") return list_typ.__args__[0] -def read_vector_elem_typ(vector_typ: Type[Vector[T, L]]) -> T: +def read_vector_elem_type(vector_typ: Type[Vector[T, L]]) -> T: return vector_typ.elem_type -def read_elem_typ(typ): +def read_elem_type(typ): if typ == bytes: return byte elif is_list_type(typ): From 0f79ed709bd4846299fb4379ed22d1de47f94e02 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 22:19:18 +0200 Subject: [PATCH 204/308] update yaml encoder/decoder and obj randomizer for typed ssz usage --- test_libs/pyspec/eth2spec/debug/decode.py | 16 +-- test_libs/pyspec/eth2spec/debug/encode.py | 11 +- .../pyspec/eth2spec/debug/random_value.py | 112 +++++++++--------- 3 files changed, 68 insertions(+), 71 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index 261692bedb..c9657dc287 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -3,22 +3,22 @@ def decode(data, typ): - if is_uint(typ): + if is_uint_type(typ): return data elif is_bool_type(typ): assert data in (True, False) return data - elif issubclass(typ, list): - elem_typ = read_list_elem_typ(typ) + elif is_list_type(typ): + elem_typ = read_list_elem_type(typ) return [decode(element, elem_typ) for element in data] - elif issubclass(typ, Vector): - elem_typ = read_vector_elem_typ(typ) + elif is_vector_type(typ): + elem_typ = read_vector_elem_type(typ) return Vector(decode(element, elem_typ) for element in data) - elif issubclass(typ, bytes): + elif is_bytes_type(typ): return bytes.fromhex(data[2:]) - elif issubclass(typ, BytesN): + elif is_bytesn_type(typ): return BytesN(bytes.fromhex(data[2:])) - elif is_container_typ(typ): + elif is_container_type(typ): temp = {} for field, subtype in typ.get_fields(): temp[field] = decode(data[field], subtype) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 3c0658c8fc..832203e350 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -3,19 +3,20 @@ def encode(value, typ, include_hash_tree_roots=False): - if is_uint(typ): + if is_uint_type(typ): + # Larger uints are boxed and the class declares their byte length if issubclass(typ, uint) and typ.byte_len > 8: return str(value) return value elif is_bool_type(typ): assert value in (True, False) return value - elif issubclass(typ, list) or issubclass(typ, Vector): - elem_typ = read_elem_typ(typ) + elif is_list_type(typ) or is_vector_type(typ): + elem_typ = read_elem_type(typ) return [encode(element, elem_typ, include_hash_tree_roots) for element in value] - elif issubclass(typ, bytes): + elif issubclass(typ, bytes): # both bytes and BytesN return '0x' + value.hex() - elif is_container_typ(typ): + elif is_container_type(typ): ret = {} for field, subtype in typ.get_fields(): field_value = getattr(value, field) diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index f28181943c..5abd730865 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -2,10 +2,11 @@ from typing import Any from enum import Enum +from eth2spec.utils.ssz.ssz_typing import * +from eth2spec.utils.ssz.ssz_impl import is_basic_type -UINT_SIZES = [8, 16, 32, 64, 128, 256] - -basic_types = ["uint%d" % v for v in UINT_SIZES] + ['bool', 'byte'] +# in bytes +UINT_SIZES = [1, 2, 4, 8, 16, 32] random_mode_names = ["random", "zero", "max", "nil", "one", "lengthy"] @@ -49,60 +50,61 @@ def get_random_ssz_object(rng: Random, """ if chaos: mode = rng.choice(list(RandomizationMode)) - if isinstance(typ, str): - # Bytes array - if typ == 'bytes': - if mode == RandomizationMode.mode_nil_count: - return b'' - if mode == RandomizationMode.mode_max_count: - return get_random_bytes_list(rng, max_bytes_length) - if mode == RandomizationMode.mode_one_count: - return get_random_bytes_list(rng, 1) - if mode == RandomizationMode.mode_zero: - return b'\x00' - if mode == RandomizationMode.mode_max: - return b'\xff' - return get_random_bytes_list(rng, rng.randint(0, max_bytes_length)) - elif typ[:5] == 'bytes' and len(typ) > 5: - length = int(typ[5:]) - # Sanity, don't generate absurdly big random values - # If a client is aiming to performance-test, they should create a benchmark suite. - assert length <= max_bytes_length - if mode == RandomizationMode.mode_zero: - return b'\x00' * length - if mode == RandomizationMode.mode_max: - return b'\xff' * length - return get_random_bytes_list(rng, length) - # Basic types - else: - if mode == RandomizationMode.mode_zero: - return get_min_basic_value(typ) - if mode == RandomizationMode.mode_max: - return get_max_basic_value(typ) - return get_random_basic_value(rng, typ) + # Bytes array + if is_bytes_type(typ): + if mode == RandomizationMode.mode_nil_count: + return b'' + if mode == RandomizationMode.mode_max_count: + return get_random_bytes_list(rng, max_bytes_length) + if mode == RandomizationMode.mode_one_count: + return get_random_bytes_list(rng, 1) + if mode == RandomizationMode.mode_zero: + return b'\x00' + if mode == RandomizationMode.mode_max: + return b'\xff' + return get_random_bytes_list(rng, rng.randint(0, max_bytes_length)) + elif is_bytesn_type(typ): + length = typ.length + # Sanity, don't generate absurdly big random values + # If a client is aiming to performance-test, they should create a benchmark suite. + assert length <= max_bytes_length + if mode == RandomizationMode.mode_zero: + return b'\x00' * length + if mode == RandomizationMode.mode_max: + return b'\xff' * length + return get_random_bytes_list(rng, length) + # Basic types + elif is_basic_type(typ): + if mode == RandomizationMode.mode_zero: + return get_min_basic_value(typ) + if mode == RandomizationMode.mode_max: + return get_max_basic_value(typ) + return get_random_basic_value(rng, typ) # Vector: - elif isinstance(typ, list) and len(typ) == 2: + elif is_vector_type(typ): + elem_typ = read_vector_elem_type(typ) return [ - get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) - for _ in range(typ[1]) + get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) + for _ in range(typ.length) ] # List: - elif isinstance(typ, list) and len(typ) == 1: + elif is_list_type(typ): + elem_typ = read_list_elem_type(typ) length = rng.randint(0, max_list_length) if mode == RandomizationMode.mode_one_count: length = 1 if mode == RandomizationMode.mode_max_count: length = max_list_length return [ - get_random_ssz_object(rng, typ[0], max_bytes_length, max_list_length, mode, chaos) + get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) for _ in range(length) ] # Container: - elif hasattr(typ, 'fields'): + elif is_container_type(typ): return typ(**{ field: get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode, chaos) - for field, subtype in typ.fields.items() + for field, subtype in typ.get_fields() }) else: print(typ) @@ -114,39 +116,33 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: def get_random_basic_value(rng: Random, typ: str) -> Any: - if typ == 'bool': + if is_bool_type(typ): return rng.choice((True, False)) - if typ[:4] == 'uint': - size = int(typ[4:]) + if is_uint_type(typ): + size = uint_byte_size(typ) assert size in UINT_SIZES - return rng.randint(0, 2**size - 1) - if typ == 'byte': - return rng.randint(0, 8) + return rng.randint(0, 256**size - 1) else: raise ValueError("Not a basic type") def get_min_basic_value(typ: str) -> Any: - if typ == 'bool': + if is_bool_type(typ): return False - if typ[:4] == 'uint': - size = int(typ[4:]) + if is_uint_type(typ): + size = uint_byte_size(typ) assert size in UINT_SIZES return 0 - if typ == 'byte': - return 0x00 else: raise ValueError("Not a basic type") def get_max_basic_value(typ: str) -> Any: - if typ == 'bool': + if is_bool_type(typ): return True - if typ[:4] == 'uint': - size = int(typ[4:]) + if is_uint_type(typ): + size = uint_byte_size(typ) assert size in UINT_SIZES - return 2**size - 1 - if typ == 'byte': - return 0xff + return 256**size - 1 else: raise ValueError("Not a basic type") From d023d2d20f06474f7322d113049e3365a733a74a Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 23:40:05 +0200 Subject: [PATCH 205/308] lots of bugfixes --- test_libs/pyspec/eth2spec/debug/encode.py | 4 +- .../pyspec/eth2spec/debug/random_value.py | 6 +- .../pyspec/eth2spec/utils/merkle_minimal.py | 7 +- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 5 +- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 90 ++++++++++++++----- 5 files changed, 81 insertions(+), 31 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 832203e350..a3c3c1189a 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -4,6 +4,8 @@ def encode(value, typ, include_hash_tree_roots=False): if is_uint_type(typ): + if hasattr(typ, '__supertype__'): + typ = typ.__supertype__ # Larger uints are boxed and the class declares their byte length if issubclass(typ, uint) and typ.byte_len > 8: return str(value) @@ -14,7 +16,7 @@ def encode(value, typ, include_hash_tree_roots=False): elif is_list_type(typ) or is_vector_type(typ): elem_typ = read_elem_type(typ) return [encode(element, elem_typ, include_hash_tree_roots) for element in value] - elif issubclass(typ, bytes): # both bytes and BytesN + elif isinstance(typ, type) and issubclass(typ, bytes): # both bytes and BytesN return '0x' + value.hex() elif is_container_type(typ): ret = {} diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 5abd730865..ab9c4c8859 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -115,7 +115,7 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: return bytes(rng.getrandbits(8) for _ in range(length)) -def get_random_basic_value(rng: Random, typ: str) -> Any: +def get_random_basic_value(rng: Random, typ) -> Any: if is_bool_type(typ): return rng.choice((True, False)) if is_uint_type(typ): @@ -126,7 +126,7 @@ def get_random_basic_value(rng: Random, typ: str) -> Any: raise ValueError("Not a basic type") -def get_min_basic_value(typ: str) -> Any: +def get_min_basic_value(typ) -> Any: if is_bool_type(typ): return False if is_uint_type(typ): @@ -137,7 +137,7 @@ def get_min_basic_value(typ: str) -> Any: raise ValueError("Not a basic type") -def get_max_basic_value(typ: str) -> Any: +def get_max_basic_value(typ) -> Any: if is_bool_type(typ): return True if is_uint_type(typ): diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index e3e5d35d8c..420f0b5f1f 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -34,10 +34,13 @@ def get_merkle_proof(tree, item_index): def next_power_of_two(v: int) -> int: """ - Get the next power of 2. (for 64 bit range ints) + Get the next power of 2. (for 64 bit range ints). + 0 is a special case, to have non-empty defaults. Examples: - 0 -> 0, 1 -> 1, 2 -> 2, 3 -> 4, 32 -> 32, 33 -> 64 + 0 -> 1, 1 -> 1, 2 -> 2, 3 -> 4, 32 -> 32, 33 -> 64 """ + if v == 0: + return 1 # effectively fill the bitstring (1 less, do not want to with ones, then increment for next power of 2. v -= 1 v |= v >> (1 << 0) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index c3cc579bdd..826714c965 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -112,7 +112,7 @@ def hash_tree_root(obj, typ): leaf_root = merkleize_chunks(leaves) return mix_in_length(leaf_root, len(obj)) if is_list_type(typ) else leaf_root elif is_container_type(typ): - leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()] + leaves = [hash_tree_root(field_value, field_typ) for field_value, field_typ in obj.get_typed_values()] return merkleize_chunks(chunkify(b''.join(leaves))) else: raise Exception("Type not supported: obj {} type {}".format(obj, typ)) @@ -121,6 +121,7 @@ def hash_tree_root(obj, typ): @infer_input_type def signing_root(obj, typ): assert is_container_type(typ) - leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()[:-1]] + # ignore last field + leaves = [hash_tree_root(field_value, field_typ) for field_value, field_typ in obj.get_typed_values()[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index d16c66abb5..0121ded9ec 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,6 +1,6 @@ -from typing import List, Iterable, Type, NewType -from typing import Union from inspect import isclass +from typing import List, Iterable, TypeVar, Type, NewType +from typing import Union # SSZ integers @@ -64,17 +64,25 @@ def __new__(cls, value, *args, **kwargs): def is_uint_type(typ): # All integers are uint in the scope of the spec here. # Since we default to uint64. Bounds can be checked elsewhere. - return issubclass(typ, int) + + # However, some are wrapped in a NewType + if hasattr(typ, '__supertype__'): + # get the type that the NewType is wrapping + typ = typ.__supertype__ + + return isinstance(typ, type) and issubclass(typ, int) def uint_byte_size(typ): - if issubclass(typ, uint): - return typ.byte_len - elif issubclass(typ, int): - # Default to uint64 - return 8 - else: - raise TypeError("Type %s is not an uint (or int-default uint64) type" % typ) + if hasattr(typ, '__supertype__'): + typ = typ.__supertype__ + if isinstance(typ, type): + if issubclass(typ, uint): + return typ.byte_len + elif issubclass(typ, int): + # Default to uint64 + return 8 + raise TypeError("Type %s is not an uint (or int-default uint64) type" % typ) # SSZ Container base class @@ -86,7 +94,7 @@ class Container(object): def __init__(self, **kwargs): cls = self.__class__ - for f, t in cls.get_fields().items(): + for f, t in cls.get_fields(): if f not in kwargs: setattr(self, f, get_zero_value(t)) else: @@ -117,7 +125,10 @@ def get_fields_dict(cls): @classmethod def get_fields(cls): - return dict(cls.__annotations__).items() + return list(dict(cls.__annotations__).items()) + + def get_typed_values(self): + return list(zip(self.get_field_values(), self.get_field_types())) @classmethod def get_field_names(cls): @@ -134,6 +145,9 @@ def get_field_types(cls): def _is_vector_instance_of(a, b): + # Other must not be a BytesN + if issubclass(b, bytes): + return False if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): # Vector (b) is not an instance of Vector[X, Y] (a) return False @@ -146,6 +160,9 @@ def _is_vector_instance_of(a, b): def _is_equal_vector_type(a, b): + # Other must not be a BytesN + if issubclass(b, bytes): + return False if not hasattr(a, 'elem_type') or not hasattr(a, 'length'): if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): # Vector == Vector @@ -237,6 +254,9 @@ def __len__(self): def _is_bytes_n_instance_of(a, b): + # Other has to be a Bytes derivative class to be a BytesN + if not issubclass(b, bytes): + return False if not hasattr(b, 'length'): # BytesN (b) is not an instance of BytesN[X] (a) return False @@ -249,6 +269,9 @@ def _is_bytes_n_instance_of(a, b): def _is_equal_bytes_n_type(a, b): + # Other has to be a Bytes derivative class to be a BytesN + if not issubclass(b, bytes): + return False if not hasattr(a, 'length'): if not hasattr(b, 'length'): # BytesN == BytesN @@ -267,7 +290,7 @@ def __new__(cls, class_name, parents, attrs): out = type.__new__(cls, class_name, parents, attrs) if 'length' in attrs: setattr(out, 'length', attrs['length']) - out._name = 'Vector' + out._name = 'BytesN' out.elem_type = byte return out @@ -318,7 +341,7 @@ def __new__(cls, *args): else: bytesval = b'\x00' * cls.length if len(bytesval) != cls.length: - raise TypeError("bytesN[%d] cannot be initialized with value of %d bytes" % (cls.length, len(bytesval))) + raise TypeError("BytesN[%d] cannot be initialized with value of %d bytes" % (cls.length, len(bytesval))) return super().__new__(cls, bytesval) def serialize(self): @@ -334,7 +357,7 @@ def hash_tree_root(self): # ----------------------------- def get_zero_value(typ): - if is_uint(typ): + if is_uint_type(typ): return 0 if issubclass(typ, bool): return False @@ -354,7 +377,7 @@ def get_zero_value(typ): # ----------------------------- def infer_type(obj): - if is_uint(obj.__class__): + if is_uint_type(obj.__class__): return obj.__class__ elif isinstance(obj, int): return uint64 @@ -370,39 +393,50 @@ def infer_input_type(fn): """ Decorator to run infer_type on the obj if typ argument is None """ + def infer_helper(obj, typ=None): if typ is None: typ = infer_type(obj) return fn(obj, typ) + return infer_helper + def is_bool_type(typ): - return issubclass(typ, bool) + if hasattr(typ, '__supertype__'): + typ = typ.__supertype__ + return isinstance(typ, type) and issubclass(typ, bool) + def is_list_type(typ): """ Checks if the given type is a list. """ - return (hasattr(typ, '_name') and typ._name == 'List') + return hasattr(typ, '_name') and typ._name == 'List' + def is_bytes_type(typ): # Do not accept subclasses of bytes here, to avoid confusion with BytesN return typ == bytes + def is_list_kind(typ): """ Checks if the given type is a kind of list. Can be bytes. """ return is_list_type(typ) or is_bytes_type(typ) + def is_vector_type(typ): """ Checks if the given type is a vector. """ - return issubclass(typ, Vector) + return isinstance(typ, type) and issubclass(typ, Vector) + def is_bytesn_type(typ): - return issubclass(typ, BytesN) + return isinstance(typ, type) and issubclass(typ, BytesN) + def is_vector_kind(typ): """ @@ -410,23 +444,33 @@ def is_vector_kind(typ): """ return is_vector_type(typ) or is_bytesn_type(typ) + def is_container_type(typ): - return issubclass(typ, Container) + return isinstance(typ, type) and issubclass(typ, Container) + + +T = TypeVar('T') +L = TypeVar('L') + def read_list_elem_type(list_typ: Type[List[T]]) -> T: if list_typ.__args__ is None or len(list_typ.__args__) != 1: raise TypeError("Supplied list-type is invalid, no element type found.") return list_typ.__args__[0] + def read_vector_elem_type(vector_typ: Type[Vector[T, L]]) -> T: return vector_typ.elem_type + def read_elem_type(typ): if typ == bytes: return byte elif is_list_type(typ): - return read_list_elem_typ(typ) + return read_list_elem_type(typ) elif is_vector_type(typ): - return read_vector_elem_typ(typ) + return read_vector_elem_type(typ) + elif issubclass(typ, bytes): + return byte else: raise TypeError("Unexpected type: {}".format(typ)) From b4c4df6a09e6feff9f1bdc1595f9703e836bf233 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 23:41:29 +0200 Subject: [PATCH 206/308] bugfix in sss_types global building in script --- scripts/phase0/function_puller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 0047d53212..d7765a4a44 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -70,10 +70,10 @@ def get_spec(file_name: str) -> List[str]: code_lines.append(' ' + type_line) code_lines.append('\n') for (ssz_type_name, _) in type_defs: - code_lines.append(f' global_vars["{ssz_type_name}"] = {ssz_type_name},') + code_lines.append(f' global_vars["{ssz_type_name}"] = {ssz_type_name}') code_lines.append(' global_vars["ssz_types"] = [') for (ssz_type_name, _) in type_defs: - code_lines.append(f' {ssz_type_name},') + code_lines.append(f' "{ssz_type_name}",') code_lines.append(' ]') code_lines.append('\n') code_lines.append('def get_ssz_type_by_name(name: str) -> Container:') From 54b14b5ac305ecb3fcc25086b6a63c81dafc61a9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 23:41:48 +0200 Subject: [PATCH 207/308] update ssz-static generator --- test_generators/ssz_static/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/ssz_static/main.py b/test_generators/ssz_static/main.py index 1234294db9..abb1676138 100644 --- a/test_generators/ssz_static/main.py +++ b/test_generators/ssz_static/main.py @@ -2,7 +2,7 @@ from eth2spec.debug import random_value, encode from eth2spec.phase0 import spec -from eth2spec.utils.minimal_ssz import ( +from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, serialize, From c99fa52d9fdfe5f4bc2fe6b043f8106a0feeaf80 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 27 May 2019 23:56:17 +0200 Subject: [PATCH 208/308] fix dev branch build script, missing quotes --- scripts/phase0/function_puller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index 750f195904..e54df3ef09 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -66,7 +66,7 @@ def get_spec(file_name: str) -> List[str]: code_lines.append('\n') code_lines.append('ssz_types = [\n') for (ssz_type_name, _) in type_defs: - code_lines.append(f' {ssz_type_name},\n') + code_lines.append(f' "{ssz_type_name}",\n') code_lines.append(']') code_lines.append('\n') code_lines.append('def get_ssz_type_by_name(name: str) -> SSZType:') From 47f3df6d0a6296f950289057f3082c6c95d907be Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 27 May 2019 16:40:00 -0600 Subject: [PATCH 209/308] fix deposit test for new index handling --- .../block_processing/test_process_deposit.py | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/test_libs/pyspec/tests/block_processing/test_process_deposit.py b/test_libs/pyspec/tests/block_processing/test_process_deposit.py index bbfb390efb..62b058a707 100644 --- a/test_libs/pyspec/tests/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/tests/block_processing/test_process_deposit.py @@ -83,33 +83,45 @@ def test_success_top_up(state): return pre_state, deposit, post_state -def test_wrong_index(state): +def test_wrong_deposit_for_deposit_count(state): pre_state = deepcopy(state) deposit_data_leaves = [ZERO_HASH] * len(pre_state.validator_registry) - index = len(deposit_data_leaves) - pubkey = pubkeys[index] - privkey = privkeys[index] - deposit, root, deposit_data_leaves = build_deposit( + # build root for deposit_1 + index_1 = len(deposit_data_leaves) + pubkey_1 = pubkeys[index_1] + privkey_1 = privkeys[index_1] + deposit_1, root_1, deposit_data_leaves = build_deposit( pre_state, deposit_data_leaves, - pubkey, - privkey, + pubkey_1, + privkey_1, spec.MAX_EFFECTIVE_BALANCE, ) + deposit_count_1 = len(deposit_data_leaves) - # mess up deposit_index - deposit.index = pre_state.deposit_index + 1 + # build root for deposit_2 + index_2 = len(deposit_data_leaves) + pubkey_2 = pubkeys[index_2] + privkey_2 = privkeys[index_2] + deposit_2, root_2, deposit_data_leaves = build_deposit( + pre_state, + deposit_data_leaves, + pubkey_2, + privkey_2, + spec.MAX_EFFECTIVE_BALANCE, + ) - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) + # state has root for deposit_2 but is at deposit_count for deposit_1 + pre_state.latest_eth1_data.deposit_root = root_2 + pre_state.latest_eth1_data.deposit_count = deposit_count_1 post_state = deepcopy(pre_state) with pytest.raises(AssertionError): - process_deposit(post_state, deposit) + process_deposit(post_state, deposit_2) - return pre_state, deposit, None + return pre_state, deposit_2, None def test_bad_merkle_proof(state): From 9e61cc2ed3124c2118b820543f5424d680153854 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 28 May 2019 00:45:13 +0200 Subject: [PATCH 210/308] bugfix: typing edge-case: python bool is subclass of int. --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 0121ded9ec..326dcb748a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -70,7 +70,7 @@ def is_uint_type(typ): # get the type that the NewType is wrapping typ = typ.__supertype__ - return isinstance(typ, type) and issubclass(typ, int) + return isinstance(typ, type) and issubclass(typ, int) and not issubclass(typ, bool) def uint_byte_size(typ): From 5e28adf5561172cde56b5af09e8de481ff9fbd0c Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 28 May 2019 00:51:27 +0200 Subject: [PATCH 211/308] bugfix: don't forget about var-length bytes getting a length mixin --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 826714c965..96eaff481c 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -110,7 +110,7 @@ def hash_tree_root(obj, typ): else: leaves = [hash_tree_root(elem, subtype) for elem in obj] leaf_root = merkleize_chunks(leaves) - return mix_in_length(leaf_root, len(obj)) if is_list_type(typ) else leaf_root + return mix_in_length(leaf_root, len(obj)) if is_list_kind(typ) else leaf_root elif is_container_type(typ): leaves = [hash_tree_root(field_value, field_typ) for field_value, field_typ in obj.get_typed_values()] return merkleize_chunks(chunkify(b''.join(leaves))) From 56698602ab325871d703bacf3bb12a392421cbf1 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Tue, 28 May 2019 10:48:41 +1000 Subject: [PATCH 212/308] Updated all absolute URLs to the eth2.0-specs repo so that they point to the master branch (instead of dev). --- specs/validator/beacon_node_oapi.yaml | 30 +++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/specs/validator/beacon_node_oapi.yaml b/specs/validator/beacon_node_oapi.yaml index 940f82137f..74be21fac8 100644 --- a/specs/validator/beacon_node_oapi.yaml +++ b/specs/validator/beacon_node_oapi.yaml @@ -308,7 +308,7 @@ components: description: "Globally, the estimated most recent slot number, or current target slot number." BeaconBlock: - description: "The [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblock) object from the Eth2.0 spec." + description: "The [`BeaconBlock`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#beaconblock) object from the Eth2.0 spec." allOf: - $ref: '#/components/schemas/BeaconBlockCommon' - type: object @@ -316,7 +316,7 @@ components: body: $ref: '#/components/schemas/BeaconBlockBody' BeaconBlockHeader: - description: "The [`BeaconBlockHeader`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblockheader) object from the Eth2.0 spec." + description: "The [`BeaconBlockHeader`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#beaconblockheader) object from the Eth2.0 spec." allOf: - $ref: '#/components/schemas/BeaconBlockCommon' - type: object @@ -352,7 +352,7 @@ components: description: "The BLS signature of the `BeaconBlock` made by the validator of the block." BeaconBlockBody: type: object - description: "The [`BeaconBlockBody`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#beaconblockbody) object from the Eth2.0 spec." + description: "The [`BeaconBlockBody`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#beaconblockbody) object from the Eth2.0 spec." properties: randao_reveal: type: string @@ -362,7 +362,7 @@ components: eth1_data: title: Eth1Data type: object - description: "The [`Eth1Data`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#eth1data) object from the Eth2.0 spec." + description: "The [`Eth1Data`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#eth1data) object from the Eth2.0 spec." properties: deposit_root: type: string @@ -387,7 +387,7 @@ components: items: title: ProposerSlashings type: object - description: "The [`ProposerSlashing`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#proposerslashing) object from the Eth2.0 spec." + description: "The [`ProposerSlashing`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#proposerslashing) object from the Eth2.0 spec." properties: proposer_index: type: integer @@ -402,7 +402,7 @@ components: items: title: AttesterSlashings type: object - description: "The [`AttesterSlashing`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attesterslashing) object from the Eth2.0 spec." + description: "The [`AttesterSlashing`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attesterslashing) object from the Eth2.0 spec." properties: attestation_1: $ref: '#/components/schemas/IndexedAttestation' @@ -413,7 +413,7 @@ components: items: title: Attestation type: object - description: "The [`Attestation`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attestation) object from the Eth2.0 spec." + description: "The [`Attestation`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attestation) object from the Eth2.0 spec." properties: aggregation_bitfield: type: string @@ -437,7 +437,7 @@ components: items: title: Deposit type: object - description: "The [`Deposit`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#deposit) object from the Eth2.0 spec." + description: "The [`Deposit`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#deposit) object from the Eth2.0 spec." properties: proof: type: array @@ -455,7 +455,7 @@ components: data: title: DepositData type: object - description: "The [`DepositData`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#depositdata) object from the Eth2.0 spec." + description: "The [`DepositData`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#depositdata) object from the Eth2.0 spec." properties: pubkey: $ref: '#/components/schemas/pubkey' @@ -478,7 +478,7 @@ components: items: title: VoluntaryExit type: object - description: "The [`VoluntaryExit`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#voluntaryexit) object from the Eth2.0 spec." + description: "The [`VoluntaryExit`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#voluntaryexit) object from the Eth2.0 spec." properties: epoch: type: integer @@ -498,7 +498,7 @@ components: items: title: Transfer type: object - description: "The [`Transfer`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#transfer) object from the Eth2.0 spec." + description: "The [`Transfer`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#transfer) object from the Eth2.0 spec." properties: sender: type: integer @@ -533,7 +533,7 @@ components: Fork: type: object - description: "The [`Fork`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#Fork) object from the Eth2.0 spec." + description: "The [`Fork`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#Fork) object from the Eth2.0 spec." properties: previous_version: type: string @@ -551,7 +551,7 @@ components: description: "Fork epoch number." IndexedAttestation: type: object - description: "The [`IndexedAttestation`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#indexedattestation) object from the Eth2.0 spec." + description: "The [`IndexedAttestation`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#indexedattestation) object from the Eth2.0 spec." properties: custody_bit_0_indices: type: array @@ -575,7 +575,7 @@ components: $ref: '#/components/schemas/AttestationData' AttestationData: type: object - description: "The [`AttestationData`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#attestationdata) object from the Eth2.0 spec." + description: "The [`AttestationData`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#attestationdata) object from the Eth2.0 spec." properties: beacon_block_root: type: string @@ -603,7 +603,7 @@ components: crosslink: title: CrossLink type: object - description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec, contains data from epochs [`start_epoch`, `end_epoch`)." + description: "The [`Crosslink`](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md#crosslink) object from the Eth2.0 spec, contains data from epochs [`start_epoch`, `end_epoch`)." properties: shard: type: integer From c32328fdf2d13796343cc5cd33bd1c5a1c9043e7 Mon Sep 17 00:00:00 2001 From: Luke Anderson Date: Tue, 28 May 2019 10:49:40 +1000 Subject: [PATCH 213/308] Fixed swagger URL so that the version number isn't specified (defaults to latest). --- specs/validator/0_beacon-node-validator-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-node-validator-api.md b/specs/validator/0_beacon-node-validator-api.md index c53ce37aca..2a5fe7fcda 100644 --- a/specs/validator/0_beacon-node-validator-api.md +++ b/specs/validator/0_beacon-node-validator-api.md @@ -25,4 +25,4 @@ This specification is derived from a proposal and discussion on Issues [#1011](h The API specification has been written in [OpenAPI 3.0](https://swagger.io/docs/specification/about/) and is provided in the [beacon_node_oapi.yaml](beacon_node_oapi.yaml) file alongside this document. For convenience, this specification has been uploaded to [SwaggerHub](https://swagger.io/tools/swaggerhub/) at the following URL: -[https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator/0.1](https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator/0.1) +[https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator](https://app.swaggerhub.com/apis/spble/beacon_node_api_for_validator) From 00ffaf4d71e6f26667f37616418618cef7fc0559 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 May 2019 12:37:41 +0800 Subject: [PATCH 214/308] Fix ssz path --- test_generators/operations/deposits.py | 2 +- test_libs/pyspec/tests/helpers.py | 2 +- test_libs/pyspec/tests/test_sanity.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test_generators/operations/deposits.py b/test_generators/operations/deposits.py index 075ccbd5ba..297a27859b 100644 --- a/test_generators/operations/deposits.py +++ b/test_generators/operations/deposits.py @@ -5,7 +5,7 @@ from gen_base import gen_suite, gen_typing from preset_loader import loader from eth2spec.debug.encode import encode -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.utils.merkle_minimal import get_merkle_root, calc_merkle_tree_from_leaves, get_merkle_proof from typing import List, Tuple diff --git a/test_libs/pyspec/tests/helpers.py b/test_libs/pyspec/tests/helpers.py index a4849bfbbc..ca40bf1d8c 100644 --- a/test_libs/pyspec/tests/helpers.py +++ b/test_libs/pyspec/tests/helpers.py @@ -3,7 +3,7 @@ from py_ecc import bls import eth2spec.phase0.spec as spec -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.phase0.spec import ( # constants ZERO_HASH, diff --git a/test_libs/pyspec/tests/test_sanity.py b/test_libs/pyspec/tests/test_sanity.py index 1c05e6b53a..b4d7a8e8b3 100644 --- a/test_libs/pyspec/tests/test_sanity.py +++ b/test_libs/pyspec/tests/test_sanity.py @@ -5,7 +5,7 @@ from py_ecc import bls import eth2spec.phase0.spec as spec -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.phase0.spec import ( # constants ZERO_HASH, From f0ceefc36d8d54670be1b8e48a0b5128d0d6208b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 May 2019 13:57:42 +0800 Subject: [PATCH 215/308] Make `test_libs/pyspec/tests` pass 1. Use `typing_inspect` library to check if it's `typing.List[Any]`. Use it in `is_list_type`. 2. Add `__hash__`, `__eq__`, `__repr__` methods to some classes. --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 56 +++++++++++++------ test_libs/pyspec/requirements.txt | 1 + 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 326dcb748a..3662c52b5b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,6 +1,7 @@ from inspect import isclass from typing import List, Iterable, TypeVar, Type, NewType from typing import Union +from typing_inspect import get_origin # SSZ integers @@ -119,6 +120,18 @@ def get_field_values(self): def __repr__(self): return {field: getattr(self, field) for field in self.get_field_names()} + def __str__(self): + output = [] + for field in self.get_field_names(): + output.append(f'{field}: {getattr(self, field)}') + return "\n".join(output) + + def __eq__(self, other): + return self.hash_tree_root() == other.hash_tree_root() + + def __hash__(self): + return hash(self.hash_tree_root()) + @classmethod def get_fields_dict(cls): return dict(cls.__annotations__) @@ -203,6 +216,9 @@ def __eq__(self, other): def __ne__(self, other): return not _is_equal_vector_type(self, other) + def __hash__(self): + return hash(self.__class__) + class Vector(metaclass=VectorMeta): @@ -252,6 +268,9 @@ def __iter__(self): def __len__(self): return len(self.items) + def __eq__(self, other): + return self.hash_tree_root() == other.hash_tree_root() + def _is_bytes_n_instance_of(a, b): # Other has to be a Bytes derivative class to be a BytesN @@ -309,6 +328,9 @@ def __eq__(self, other): def __ne__(self, other): return not _is_equal_bytes_n_type(self, other) + def __hash__(self): + return hash(self.__class__) + def parse_bytes(val): if val is None: @@ -355,23 +377,25 @@ def hash_tree_root(self): # SSZ Defaults # ----------------------------- - def get_zero_value(typ): + result = None if is_uint_type(typ): - return 0 - if issubclass(typ, bool): - return False - if issubclass(typ, list): - return [] - if issubclass(typ, Vector): - return typ() - if issubclass(typ, BytesN): - return typ() - if issubclass(typ, bytes): - return b'' - if issubclass(typ, Container): - return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}), - + result = 0 + elif is_list_type(typ): + result = [] + elif issubclass(typ, bool): + result = False + elif issubclass(typ, Vector): + result = typ() + elif issubclass(typ, BytesN): + result = typ() + elif issubclass(typ, bytes): + result = b'' + elif issubclass(typ, Container): + result = typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) + else: + return Exception("Type not supported: {}".format(typ)) + return result # Type helpers # ----------------------------- @@ -412,7 +436,7 @@ def is_list_type(typ): """ Checks if the given type is a list. """ - return hasattr(typ, '_name') and typ._name == 'List' + return get_origin(typ) is List or get_origin(typ) is list def is_bytes_type(typ): diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index 78d41708dc..3b38930bda 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -2,3 +2,4 @@ eth-utils>=1.3.0,<2 eth-typing>=2.1.0,<3.0.0 pycryptodome==3.7.3 py_ecc>=1.6.0 +typing_inspect==0.4.0 From fd05b6e364d2622562a9587631d7bb5f83886eb0 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 28 May 2019 09:58:51 +0200 Subject: [PATCH 216/308] Minor bugfixes --- scripts/build_spec.py | 1 + specs/core/1_shard-data-chains.md | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index a1d390a95a..c359dd08c8 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -40,6 +40,7 @@ signing_root, type_of, empty, + serialize, ) from eth2spec.utils.bls_stub import ( diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index f26b220d74..2d6d2a4079 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -145,7 +145,7 @@ def get_period_committee(state: BeaconState, ```python def get_switchover_epoch(state: BeaconState, epoch: Epoch, index: ValidatorIndex): earlier_start_epoch = epoch - (epoch % PERSISTENT_COMMITTEE_PERIOD) - PERSISTENT_COMMITTEE_PERIOD * 2 - return (bytes_to_int(hash(generate_seed(state, earlier_start_epoch) + bytes3(index))[0:8]) + return (bytes_to_int(hash(generate_seed(state, earlier_start_epoch) + int_to_bytes(index, length=3)[0:8])) % PERSISTENT_COMMITTEE_PERIOD) ``` @@ -255,11 +255,16 @@ def compute_crosslink_data_root(blocks: List[ShardBlock]) -> Bytes32: def hash_tree_root_of_bytes(data: bytes) -> bytes: return hash_tree_root([data[i:i + 32] for i in range(0, len(data), 32)]) + def zpad(data: bytes, length: int) -> bytes: + return data + b'\x00' * (length - len(data)) + return hash( hash_tree_root(pad_to_power_of_2([ - hash_tree_root_of_bytes(zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_BODY)) for block in blocks - ])) + - hash_tree_root(pad_to_power_of_2([ + hash_tree_root_of_bytes( + zpad(serialize(get_shard_header(block)), BYTES_PER_SHARD_BLOCK_BODY) + ) for block in blocks + ])) + + hash_tree_root(pad_to_power_of_2([ hash_tree_root_of_bytes(block.body) for block in blocks ])) ) From 19601df57237384f30123b45bde0ec8becbe7f63 Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Tue, 28 May 2019 09:30:35 -0400 Subject: [PATCH 217/308] Starting work on partials --- .../pyspec/eth2spec/utils/ssz/__init__.py | 1 + .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 6 +- .../pyspec/eth2spec/utils/ssz/ssz_partials.py | 81 +++++++++++++++++++ .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 23 +++--- 4 files changed, 98 insertions(+), 13 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py diff --git a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py b/test_libs/pyspec/eth2spec/utils/ssz/__init__.py index 752d77c437..6543bd9e51 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/__init__.py @@ -1 +1,2 @@ from .ssz_impl import * +from .ssz_partials import * diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 655cd9da42..3bca0b820a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -35,7 +35,7 @@ def is_fixed_size(typ): @infer_input_type -def serialize(obj, typ): +def serialize(obj, typ=None): if is_basic_type(typ): return serialize_basic(obj, typ) elif is_list_type(typ) or is_vector_type(typ): @@ -100,7 +100,7 @@ def mix_in_length(root, length): @infer_input_type -def hash_tree_root(obj, typ): +def hash_tree_root(obj, typ=None): if is_basic_type(typ): return merkleize_chunks(chunkify(serialize_basic(obj, typ))) elif is_list_type(typ) or is_vector_type(typ): @@ -119,7 +119,7 @@ def hash_tree_root(obj, typ): @infer_input_type -def signing_root(obj, typ): +def signing_root(obj, typ=None): assert is_container_typ(typ) leaves = [hash_tree_root(elem, subtyp) for elem, subtyp in obj.get_fields()[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py new file mode 100644 index 0000000000..00a22279c8 --- /dev/null +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py @@ -0,0 +1,81 @@ +from ..merkle_minimal import hash, next_power_of_two +from .ssz_typing import * +from .ssz_impl import * + +ZERO_CHUNK = b'\x00' * 32 + +def last_power_of_two(x): + return next_power_of_two(x+1) // 2 + +def concat_generalized_indices(x, y): + return x * last_power_of_two(y) + y - last_power_of_two(y) + +def rebase(objs, new_root): + return {concat_generalized_indices(new_root, k): v for k,v in objs.items()} + +def constrict_generalized_index(x, q): + depth = last_power_of_two(x // q) + o = depth + x - q * depth + if concat_generalized_indices(q, o) != x: + return None + return o + +def unrebase(objs, q): + o = {} + for k,v in objs.items(): + new_k = constrict_generalized_index(k, q) + if new_k is not None: + o[new_k] = v + return o + +def filler(starting_position, chunk_count): + at, skip, end = chunk_count, 1, next_power_of_two(chunk_count) + value = ZERO_CHUNK + o = {} + while at < end: + while at % (skip*2) == 0: + skip *= 2 + value = hash(value + value) + o[starting_position + at] = value + at += skip + return o + +def merkle_tree_of_chunks(chunks, root): + starting_index = root * next_power_of_two(len(chunks)) + o = {starting_index+i: chunk for i,chunk in enumerate(chunks)} + o = {**o, **filler(starting_index, len(chunks))} + return o + +def is_bottom_layer_type(typ): + return ( + is_basic_type(typ) or + (is_list_type(typ) or is_vector_type(typ)) and is_basic_type(read_elem_typ(typ)) + ) + +@infer_input_type +def get_fields(obj, typ=None): + if is_container_typ(typ): + return obj.get_fields() + elif is_list_type(typ) or is_vector_type(typ): + subtype = read_elem_typ(typ) + return zip([subtype] * len(obj), obj) + else: + raise Exception("Invalid type") + +@infer_input_type +def ssz_all(obj, typ=None, root=1): + if is_list_type(typ): + o = {root * 2 + 1: len(obj).to_bytes(32, 'little')} + base = root * 2 + else: + o = {} + base = root + if is_bottom_layer_type(typ): + data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, read_elem_typ(typ)) + return {**o, **merkle_tree_of_chunks(chunkify(data), base)} + else: + fields = get_fields(obj, typ=typ) + sub_base = base * next_power_of_two(len(fields)) + for i, (elem, subtype) in enumerate(fields): + o = {**o, **ssz_all(elem, typ=subtype, root=sub_base+i)} + return {**o, **filter(sub_base, len(fields))} diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 1d8aed0ded..411c219766 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,7 +1,10 @@ -from typing import List, Iterable, Type, NewType +from typing import List, Iterable, Type, NewType, TypeVar from typing import Union from inspect import isclass +T = TypeVar('T') +L = TypeVar('L') + # SSZ integers # ----------------------------- @@ -64,13 +67,13 @@ def __new__(cls, value, *args, **kwargs): def is_uint(typ): # All integers are uint in the scope of the spec here. # Since we default to uint64. Bounds can be checked elsewhere. - return issubclass(typ, int) + return (isinstance(typ, int.__class__) and issubclass(typ, int)) or typ == uint64 def uint_byte_size(typ): - if issubclass(typ, uint): + if isinstance(typ, int.__class__) and issubclass(typ, uint): return typ.byte_len - elif issubclass(typ, int): + elif typ in (int, uint64): # Default to uint64 return 8 else: @@ -109,7 +112,7 @@ def get_field_values(self): return [getattr(self, field) for field in cls.get_field_names()] def __repr__(self): - return {field: getattr(self, field) for field in self.get_field_names()} + return repr({field: getattr(self, field) for field in self.get_field_names()}) @classmethod def get_fields_dict(cls): @@ -221,7 +224,7 @@ def hash_tree_root(self): return hash_tree_root(self, self.__class__) def __repr__(self): - return {'length': self.__class__.length, 'items': self.items} + return repr({'length': self.__class__.length, 'items': self.items}) def __getitem__(self, key): return self.items[key] @@ -370,10 +373,10 @@ def infer_input_type(fn): """ Decorator to run infer_type on the obj if typ argument is None """ - def infer_helper(obj, typ=None): + def infer_helper(obj, *args, typ=None, **kwargs): if typ is None: typ = infer_type(obj) - return fn(obj, typ) + return fn(obj, *args, typ=typ, **kwargs) return infer_helper @@ -381,10 +384,10 @@ def is_list_type(typ): return (hasattr(typ, '_name') and typ._name == 'List') or typ == bytes def is_vector_type(typ): - return issubclass(typ, Vector) + return isinstance(typ, int.__class__) and issubclass(typ, Vector) def is_container_typ(typ): - return issubclass(typ, Container) + return isinstance(typ, int.__class__) and issubclass(typ, Container) def read_list_elem_typ(list_typ: Type[List[T]]) -> T: if list_typ.__args__ is None or len(list_typ.__args__) != 1: From 8737984e19773d11233873713bcc7b21874ba3b9 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 28 May 2019 16:05:25 +0200 Subject: [PATCH 218/308] introduce back assert, modified to allow valid but non-effective calls --- specs/core/0_beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e873aa4ed2..9dc052ec42 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1248,6 +1248,7 @@ def state_transition(state: BeaconState, block: BeaconBlock, validate_state_root ```python def process_slots(state: BeaconState, slot: Slot) -> None: + assert state.slot <= slot while state.slot < slot: process_slot(state) # Process epoch on the first slot of the next epoch From 053d6e7805a61698022bdae397af4450e52519fc Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Tue, 28 May 2019 14:33:12 -0400 Subject: [PATCH 219/308] Simplified hash tree root --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 45 ++++++++++++------- .../pyspec/eth2spec/utils/ssz/ssz_partials.py | 16 ------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 56789337ea..21dba1034d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -52,7 +52,7 @@ def encode_series(values, types): return values # Recursively serialize - parts = [(is_fixed_size(types[i]), serialize(values[i], types[i])) for i in range(len(values))] + parts = [(is_fixed_size(types[i]), serialize(values[i], typ=types[i])) for i in range(len(values))] # Compute and check lengths fixed_lengths = [len(serialized) if constant_size else BYTES_PER_LENGTH_OFFSET @@ -99,29 +99,42 @@ def mix_in_length(root, length): return hash(root + length.to_bytes(32, 'little')) +def is_bottom_layer_kind(typ): + return ( + is_basic_type(typ) or + (is_list_kind(typ) or is_vector_kind(typ)) and is_basic_type(read_elem_type(typ)) + ) + + @infer_input_type -def hash_tree_root(obj, typ=None): - if is_basic_type(typ): - return merkleize_chunks(chunkify(serialize_basic(obj, typ))) +def get_typed_values(obj, typ=None): + if is_container_type(typ): + return obj.get_typed_values() elif is_list_kind(typ) or is_vector_kind(typ): - subtype = read_elem_type(typ) - if is_basic_type(subtype): - leaves = chunkify(pack(obj, subtype)) - else: - leaves = [hash_tree_root(elem, subtype) for elem in obj] - leaf_root = merkleize_chunks(leaves) - return mix_in_length(leaf_root, len(obj)) if is_list_kind(typ) else leaf_root - elif is_container_type(typ): - leaves = [hash_tree_root(field_value, field_typ) for field_value, field_typ in obj.get_typed_values()] - return merkleize_chunks(chunkify(b''.join(leaves))) + elem_type = read_elem_type(typ) + return zip(obj, [elem_type] * len(obj)) + else: + raise Exception("Invalid type") + + +@infer_input_type +def hash_tree_root(obj, typ=None): + if is_bottom_layer_kind(typ): + data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, read_elem_type(typ)) + leaves = chunkify(data) + else: + fields = get_typed_values(obj, typ=typ) + leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in fields] + if is_list_kind(typ): + return mix_in_length(merkleize_chunks(leaves), len(obj)) else: - raise Exception("Type not supported: obj {} type {}".format(obj, typ)) + return merkleize_chunks(leaves) @infer_input_type def signing_root(obj, typ): assert is_container_type(typ) # ignore last field - leaves = [hash_tree_root(field_value, field_typ) for field_value, field_typ in obj.get_typed_values()[:-1]] + leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in obj.get_typed_values()[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py index d8b9bae6f5..b30ebfb4c6 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py @@ -46,22 +46,6 @@ def merkle_tree_of_chunks(chunks, root): o = {**o, **filler(starting_index, len(chunks))} return o -def is_bottom_layer_type(typ): - return ( - is_basic_type(typ) or - (is_list_type(typ) or is_vector_type(typ)) and is_basic_type(read_elem_typ(typ)) - ) - -@infer_input_type -def get_typed_values(obj, typ=None): - if is_container_typ(typ): - return obj.get_typed_values() - elif is_list_type(typ) or is_vector_type(typ): - elem_type = read_elem_typ(typ) - return zip(obj, [elem_type] * len(obj)) - else: - raise Exception("Invalid type") - @infer_input_type def ssz_all(obj, typ=None, root=1): if is_list_type(typ): From 12a7e264532800670d12c31e08dc2d0d69f47b31 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Tue, 28 May 2019 20:57:18 -0700 Subject: [PATCH 220/308] Update README.md --- specs/test_formats/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/test_formats/README.md b/specs/test_formats/README.md index d245fcfa46..f0acb444ee 100644 --- a/specs/test_formats/README.md +++ b/specs/test_formats/README.md @@ -25,7 +25,8 @@ Test formats: - [`bls`](./bls/README.md) - [`operations`](./operations/README.md) - [`shuffling`](./shuffling/README.md) -- [`ssz`](./ssz/README.md) +- [`ssz_generic`](./ssz_generic/README.md) +- [`ssz_static`](./ssz_static/README.md) - More formats are planned, see tracking issues for CI/testing ## Glossary From 1c416541e1478af60ee3695f263f6499275a374c Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 29 May 2019 23:40:46 +0300 Subject: [PATCH 221/308] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 104f3eb921..b394d59e11 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -17,7 +17,7 @@ - [Initial values](#initial-values) - [Time parameters](#time-parameters) - [State list lengths](#state-list-lengths) - - [Reward and penalty quotients](#reward-and-penalty-quotients) + - [Rewards and penalties](#rewards-and-penalties) - [Max operations per block](#max-operations-per-block) - [Signature domains](#signature-domains) - [Data structures](#data-structures) @@ -103,7 +103,7 @@ - [Helper functions](#helper-functions-1) - [Justification and finalization](#justification-and-finalization) - [Crosslinks](#crosslinks) - - [Rewards and penalties](#rewards-and-penalties) + - [Rewards and penalties](#rewards-and-penalties-1) - [Registry updates](#registry-updates) - [Slashings](#slashings) - [Final updates](#final-updates) @@ -220,17 +220,17 @@ These configurations are updated for releases, but may be out of sync during `de | `LATEST_ACTIVE_INDEX_ROOTS_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | | `LATEST_SLASHED_EXIT_LENGTH` | `2**13` (= 8,192) | epochs | ~36 days | -### Reward and penalty quotients +### Rewards and penalties | Name | Value | | - | - | -| `BASE_REWARD_QUOTIENT` | `2**5` (= 32) | +| `BASE_REWARD_FACTOR` | `2**5` (= 32) | | `WHISTLEBLOWING_REWARD_QUOTIENT` | `2**9` (= 512) | | `PROPOSER_REWARD_QUOTIENT` | `2**3` (= 8) | | `INACTIVITY_PENALTY_QUOTIENT` | `2**25` (= 33,554,432) | | `MIN_SLASHING_PENALTY_QUOTIENT` | `2**5` (= 32) | -* **The `BASE_REWARD_QUOTIENT` is NOT final. Once all other protocol details are finalized, it will be adjusted to target a theoretical maximum total issuance of `2**21` ETH per year if `2**27` ETH is validating (and therefore `2**20` per year if `2**25` ETH is validating, etc.)** +* **The `BASE_REWARD_FACTOR` is NOT final. Once all other protocol details are finalized, it will be adjusted to target a theoretical maximum total issuance of `2**21` ETH per year if `2**27` ETH is validating (and therefore `2**20` per year if `2**25` ETH is validating, etc.)** * The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**12 epochs` (~18 days) is the time it takes the inactivity penalty to reduce the balance of non-participating [validators](#dfn-validator) to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline [validators](#dfn-validator) after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)` so after `INVERSE_SQRT_E_DROP_TIME` epochs it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. ### Max operations per block @@ -1415,7 +1415,7 @@ def process_crosslinks(state: BeaconState) -> None: def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: total_balance = get_total_active_balance(state) effective_balance = state.validator_registry[index].effective_balance - return effective_balance * BASE_REWARD_QUOTIENT // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH + return effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH ``` ```python From 6ec59867ef8eb56329e8ae10437f72edc0f0976d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 May 2019 09:53:46 +0800 Subject: [PATCH 222/308] Fix linter errors --- scripts/phase0/build_spec.py | 12 ++++++++++-- specs/core/0_beacon-chain.md | 3 ++- .../test/block_processing/test_process_transfer.py | 10 ++++++++-- test_libs/pyspec/eth2spec/test/conftest.py | 1 + test_libs/pyspec/eth2spec/test/helpers/block.py | 1 - test_libs/pyspec/eth2spec/test/sanity/test_slots.py | 1 - 6 files changed, 21 insertions(+), 7 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index 36d039c2ca..457a13da02 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -13,9 +13,17 @@ def build_phase0_spec(sourcefile, outfile): NewType, Tuple, ) -from eth2spec.utils.minimal_ssz import * +from eth2spec.utils.minimal_ssz import ( + SSZType, + hash_tree_root, + signing_root, +) from eth2spec.utils.hash_function import hash -from eth2spec.utils.bls import * +from eth2spec.utils.bls import ( + bls_aggregate_pubkeys, + bls_verify, + bls_verify_multiple, +) # stub, will get overwritten by real var diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 9dc052ec42..a01ba785de 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1776,7 +1776,8 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: validator_pubkeys = [v.pubkey for v in state.validator_registry] if pubkey not in validator_pubkeys: # Verify the deposit signature (proof of possession). - # Invalid signatures are allowed by the deposit contract, and hence included on-chain, but must not be processed. + # Invalid signatures are allowed by the deposit contract, + # and hence included on-chain, but must not be processed. # Note: deposits are valid across forks, hence the deposit domain is retrieved directly from `bls_domain` if not bls_verify( pubkey, signing_root(deposit.data), deposit.data.signature, bls_domain(DOMAIN_DEPOSIT) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py index 83af755743..bd435d67a1 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py @@ -89,7 +89,7 @@ def test_invalid_signature(state): transfer = get_valid_transfer(state) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - + yield from run_transfer_processing(state, transfer, False) @@ -140,7 +140,13 @@ def test_insufficient_balance(state): def test_no_dust_sender(state): sender_index = get_active_validator_indices(state, get_current_epoch(state))[-1] balance = state.balances[sender_index] - transfer = get_valid_transfer(state, sender_index=sender_index, amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, fee=0, signed=True) + transfer = get_valid_transfer( + state, + sender_index=sender_index, + amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, + fee=0, + signed=True, + ) # un-activate so validator can transfer state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH diff --git a/test_libs/pyspec/eth2spec/test/conftest.py b/test_libs/pyspec/eth2spec/test/conftest.py index dadb0d5d06..5e8ec708a1 100644 --- a/test_libs/pyspec/eth2spec/test/conftest.py +++ b/test_libs/pyspec/eth2spec/test/conftest.py @@ -3,6 +3,7 @@ # We import pytest only when it's present, i.e. when we are running tests. # The test-cases themselves can be generated without installing pytest. + def module_exists(module_name): try: __import__(module_name) diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index c557e02452..715cf82dbe 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -77,4 +77,3 @@ def build_empty_block(state, slot=None, signed=False): def build_empty_block_for_next_slot(state, signed=False): return build_empty_block(state, state.slot + 1, signed=signed) - diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_slots.py b/test_libs/pyspec/eth2spec/test/sanity/test_slots.py index 92e0251ca5..4c3897a6c2 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_slots.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_slots.py @@ -55,4 +55,3 @@ def test_over_epoch_boundary(state): yield 'slots', slots process_slots(state, state.slot + slots) yield 'post', state - From b280e0a8d45acf9e14bbfc286b7e97f9959c11f4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 May 2019 10:08:19 +0800 Subject: [PATCH 223/308] Sync with ethereum/deposit_contract#43 --- deposit_contract/contracts/validator_registration.json | 2 +- deposit_contract/contracts/validator_registration.v.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deposit_contract/contracts/validator_registration.json b/deposit_contract/contracts/validator_registration.json index afc86ec1dc..08d57f80a5 100644 --- a/deposit_contract/contracts/validator_registration.json +++ b/deposit_contract/contracts/validator_registration.json @@ -1 +1 @@ -{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "merkle_tree_index", "indexed": false}], "anonymous": false, "type": "event"}, {"name": "Eth2Genesis", "inputs": [{"type": "bytes32", "name": "deposit_root", "indexed": false}, {"type": "bytes", "name": "deposit_count", "indexed": false}, {"type": "bytes", "name": "time", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "to_little_endian_64", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [{"type": "uint256", "name": "value"}], "constant": true, "payable": false, "type": "function", "gas": 7077}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79221}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 11026}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 457042}, {"name": "chainStarted", "outputs": [{"type": "bool", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 603}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600060c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600060c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600060c052602060c02001555b81516001018083528114156100aa575b505061142e56600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a0526380673289600051141561026b57602060046101403734156100b457600080fd5b67ffffffffffffffff6101405111156100cc57600080fd5b60006101605261014051610180526101a060006008818352015b6101605160086000811215610103578060000360020a820461010a565b8060020a82025b905090506101605260ff61018051166101c052610160516101c0516101605101101561013557600080fd5b6101c051610160510161016052610180517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8600081121561017e578060000360020a8204610185565b8060020a82025b90509050610180525b81516001018083528114156100e6575b505060186008602082066101e001602082840111156101bc57600080fd5b60208061020082610160600060046015f15050818152809050905090508051602001806102a0828460006004600a8704601201f16101f957600080fd5b50506102a05160206001820306601f82010390506103006102a0516008818352015b8261030051111561022b57610247565b6000610300516102c001535b815160010180835281141561021b575b50505060206102805260406102a0510160206001820306601f8201039050610280f3005b63c5f2892f60005114156103c357341561028457600080fd5b6000610140526002546101605261018060006020818352015b600160016101605116141561031e57600061018051602081106102bf57600080fd5b600160c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af161031057600080fd5b60c05190506101405261038c565b6000610140516020826101a0010152602081019050610180516020811061034457600080fd5b600060c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161038257600080fd5b60c0519050610140525b610160600261039a57600080fd5b60028151048152505b815160010180835281141561029d575b50506101405160005260206000f3005b63621fd13060005114156104995734156103dc57600080fd5b60606101c060246380673289610140526002546101605261015c6000305af161040457600080fd5b6101e0805160200180610260828460006004600a8704601201f161042757600080fd5b50506102605160206001820306601f82010390506102c0610260516008818352015b826102c051111561045957610475565b60006102c05161028001535b8151600101808352811415610449575b5050506020610240526040610260510160206001820306601f8201039050610240f3005b63c47e300d600051141561127d57606060046101403760506004356004016101a03760306004356004013511156104cf57600080fd5b60406024356004016102203760206024356004013511156104ef57600080fd5b608060443560040161028037606060443560040135111561050f57600080fd5b63ffffffff6002541061052157600080fd5b60306101a0511461053157600080fd5b6020610220511461054157600080fd5b6060610280511461055157600080fd5b633b9aca00610340526103405161056757600080fd5b61034051340461032052633b9aca0061032051101561058557600080fd5b6060610440602463806732896103c052610320516103e0526103dc6000305af16105ae57600080fd5b610460805160200180610360828460006004600a8704601201f16105d157600080fd5b50506002546104a05260006104c05260026104e05261050060006020818352015b60006104e05161060157600080fd5b6104e0516104a05160016104a05101101561061b57600080fd5b60016104a051010614151561062f5761069b565b6104c060605160018251018060405190131561064a57600080fd5b809190121561065857600080fd5b8152506104e08051151561066d576000610687565b600281516002835102041461068157600080fd5b60028151025b8152505b81516001018083528114156105f2575b505060006101a06030806020846105e001018260208501600060046016f1505080518201915050600060106020820661056001602082840111156106de57600080fd5b60208061058082610520600060046015f15050818152809050905090506010806020846105e001018260208501600060046013f1505080518201915050806105e0526105e09050602060c0825160208401600060025af161073e57600080fd5b60c051905061054052600060006040602082066106800161028051828401111561076757600080fd5b6060806106a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af16107a757600080fd5b60c051905060208261088001015260208101905060006040602060208206610740016102805182840111156107db57600080fd5b606080610760826020602088068803016102800160006004601bf150508181528090509050905060208060208461080001018260208501600060046015f15050805182019150506105205160208261080001015260208101905080610800526108009050602060c0825160208401600060025af161085857600080fd5b60c051905060208261088001015260208101905080610880526108809050602060c0825160208401600060025af161088f57600080fd5b60c051905061066052600060006105405160208261092001015260208101905061022060208060208461092001018260208501600060046015f150508051820191505080610920526109209050602060c0825160208401600060025af16108f557600080fd5b60c0519050602082610aa00101526020810190506000610360600880602084610a2001018260208501600060046012f150508051820191505060006018602082066109a0016020828401111561094a57600080fd5b6020806109c082610520600060046015f1505081815280905090509050601880602084610a2001018260208501600060046014f150508051820191505061066051602082610a2001015260208101905080610a2052610a209050602060c0825160208401600060025af16109bd57600080fd5b60c0519050602082610aa001015260208101905080610aa052610aa09050602060c0825160208401600060025af16109f457600080fd5b60c051905061090052610b2060006020818352015b6104c051610b20511215610a89576000610b205160208110610a2a57600080fd5b600160c052602060c0200154602082610b4001015260208101905061090051602082610b4001015260208101905080610b4052610b409050602060c0825160208401600060025af1610a7b57600080fd5b60c051905061090052610a8e565b610a9f565b5b8151600101808352811415610a09575b5050610900516104c05160208110610ab657600080fd5b600160c052602060c02001556002805460018254011015610ad657600080fd5b60018154018155506060610c4060246380673289610bc0526104a051610be052610bdc6000305af1610b0757600080fd5b610c60805160200180610ca0828460006004600a8704601201f1610b2a57600080fd5b505060a0610d2052610d2051610d60526101a0805160200180610d2051610d6001828460006004600a8704601201f1610b6257600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516040818352015b83610d0051101515610ba057610bbd565b6000610d00516020850101535b8151600101808352811415610b8f575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610d8052610220805160200180610d2051610d6001828460006004600a8704601201f1610c1457600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610c5257610c6f565b6000610d00516020850101535b8151600101808352811415610c41575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610da052610360805160200180610d2051610d6001828460006004600a8704601201f1610cc657600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610d0457610d21565b6000610d00516020850101535b8151600101808352811415610cf3575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610dc052610280805160200180610d2051610d6001828460006004600a8704601201f1610d7857600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516060818352015b83610d0051101515610db657610dd3565b6000610d00516020850101535b8151600101808352811415610da5575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610de052610ca0805160200180610d2051610d6001828460006004600a8704601201f1610e2a57600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610e6857610e85565b6000610d00516020850101535b8151600101808352811415610e57575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d20527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce610d2051610d60a16407735940006103205110151561127b576003805460018254011015610efa57600080fd5b600181540181555062010000600354141561127a5742610e205242610e405262015180610f2657600080fd5b62015180610e405106610e20511015610f3e57600080fd5b42610e405262015180610f5057600080fd5b62015180610e405106610e2051036202a30042610e205242610e405262015180610f7957600080fd5b62015180610e405106610e20511015610f9157600080fd5b42610e405262015180610fa357600080fd5b62015180610e405106610e205103011015610fbd57600080fd5b6202a30042610e205242610e405262015180610fd857600080fd5b62015180610e405106610e20511015610ff057600080fd5b42610e40526201518061100257600080fd5b62015180610e405106610e20510301610e00526020610ee0600463c5f2892f610e8052610e9c6000305af161103657600080fd5b610ee051610e60526060610f8060246380673289610f0052600254610f2052610f1c6000305af161106657600080fd5b610fa0805160200180610fe0828460006004600a8704601201f161108957600080fd5b505060606110c06024638067328961104052610e00516110605261105c6000305af16110b457600080fd5b6110e0805160200180611120828460006004600a8704601201f16110d757600080fd5b5050610e60516111e05260606111a0526111a05161120052610fe08051602001806111a0516111e001828460006004600a8704601201f161111757600080fd5b50506111a0516111e0015160206001820306601f82010390506111a0516111e00161118081516020818352015b836111805110151561115557611172565b6000611180516020850101535b8151600101808352811415611144575b5050505060206111a0516111e0015160206001820306601f82010390506111a05101016111a0526111a051611220526111208051602001806111a0516111e001828460006004600a8704601201f16111c957600080fd5b50506111a0516111e0015160206001820306601f82010390506111a0516111e00161118081516020818352015b836111805110151561120757611224565b6000611180516020850101535b81516001018083528114156111f6575b5050505060206111a0516111e0015160206001820306601f82010390506111a05101016111a0527f08b71ef3f1b58f7a23ffb82e27f12f0888c8403f1ceb0ea7ea26b274e2189d4c6111a0516111e0a160016004555b5b005b63845980e860005114156112a357341561129657600080fd5b60045460005260206000f3005b60006000fd5b61018561142e0361018560003961018561142e036000f3"} \ No newline at end of file +{"abi": [{"name": "Deposit", "inputs": [{"type": "bytes", "name": "pubkey", "indexed": false}, {"type": "bytes", "name": "withdrawal_credentials", "indexed": false}, {"type": "bytes", "name": "amount", "indexed": false}, {"type": "bytes", "name": "signature", "indexed": false}, {"type": "bytes", "name": "merkle_tree_index", "indexed": false}], "anonymous": false, "type": "event"}, {"name": "Eth2Genesis", "inputs": [{"type": "bytes32", "name": "deposit_root", "indexed": false}, {"type": "bytes", "name": "deposit_count", "indexed": false}, {"type": "bytes", "name": "time", "indexed": false}], "anonymous": false, "type": "event"}, {"outputs": [], "inputs": [], "constant": false, "payable": false, "type": "constructor"}, {"name": "to_little_endian_64", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [{"type": "uint256", "name": "value"}], "constant": true, "payable": false, "type": "function", "gas": 7077}, {"name": "get_deposit_root", "outputs": [{"type": "bytes32", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 79221}, {"name": "get_deposit_count", "outputs": [{"type": "bytes", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 11026}, {"name": "deposit", "outputs": [], "inputs": [{"type": "bytes", "name": "pubkey"}, {"type": "bytes", "name": "withdrawal_credentials"}, {"type": "bytes", "name": "signature"}], "constant": false, "payable": true, "type": "function", "gas": 445994}, {"name": "chainStarted", "outputs": [{"type": "bool", "name": "out"}], "inputs": [], "constant": true, "payable": false, "type": "function", "gas": 603}], "bytecode": "0x600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a052341561009e57600080fd5b6101406000601f818352015b600061014051602081106100bd57600080fd5b600060c052602060c020015460208261016001015260208101905061014051602081106100e957600080fd5b600060c052602060c020015460208261016001015260208101905080610160526101609050602060c0825160208401600060025af161012757600080fd5b60c0519050606051600161014051018060405190131561014657600080fd5b809190121561015457600080fd5b6020811061016157600080fd5b600060c052602060c02001555b81516001018083528114156100aa575b505061140756600035601c52740100000000000000000000000000000000000000006020526f7fffffffffffffffffffffffffffffff6040527fffffffffffffffffffffffffffffffff8000000000000000000000000000000060605274012a05f1fffffffffffffffffffffffffdabf41c006080527ffffffffffffffffffffffffed5fa0e000000000000000000000000000000000060a0526380673289600051141561026b57602060046101403734156100b457600080fd5b67ffffffffffffffff6101405111156100cc57600080fd5b60006101605261014051610180526101a060006008818352015b6101605160086000811215610103578060000360020a820461010a565b8060020a82025b905090506101605260ff61018051166101c052610160516101c0516101605101101561013557600080fd5b6101c051610160510161016052610180517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8600081121561017e578060000360020a8204610185565b8060020a82025b90509050610180525b81516001018083528114156100e6575b505060186008602082066101e001602082840111156101bc57600080fd5b60208061020082610160600060046015f15050818152809050905090508051602001806102a0828460006004600a8704601201f16101f957600080fd5b50506102a05160206001820306601f82010390506103006102a0516008818352015b8261030051111561022b57610247565b6000610300516102c001535b815160010180835281141561021b575b50505060206102805260406102a0510160206001820306601f8201039050610280f3005b63c5f2892f60005114156103c357341561028457600080fd5b6000610140526002546101605261018060006020818352015b600160016101605116141561031e57600061018051602081106102bf57600080fd5b600160c052602060c02001546020826102200101526020810190506101405160208261022001015260208101905080610220526102209050602060c0825160208401600060025af161031057600080fd5b60c05190506101405261038c565b6000610140516020826101a0010152602081019050610180516020811061034457600080fd5b600060c052602060c02001546020826101a0010152602081019050806101a0526101a09050602060c0825160208401600060025af161038257600080fd5b60c0519050610140525b610160600261039a57600080fd5b60028151048152505b815160010180835281141561029d575b50506101405160005260206000f3005b63621fd13060005114156104995734156103dc57600080fd5b60606101c060246380673289610140526002546101605261015c6000305af161040457600080fd5b6101e0805160200180610260828460006004600a8704601201f161042757600080fd5b50506102605160206001820306601f82010390506102c0610260516008818352015b826102c051111561045957610475565b60006102c05161028001535b8151600101808352811415610449575b5050506020610240526040610260510160206001820306601f8201039050610240f3005b63c47e300d600051141561125657606060046101403760506004356004016101a03760306004356004013511156104cf57600080fd5b60406024356004016102203760206024356004013511156104ef57600080fd5b608060443560040161028037606060443560040135111561050f57600080fd5b63ffffffff6002541061052157600080fd5b60306101a0511461053157600080fd5b6020610220511461054157600080fd5b6060610280511461055157600080fd5b633b9aca00610340526103405161056757600080fd5b61034051340461032052633b9aca0061032051101561058557600080fd5b6060610440602463806732896103c052610320516103e0526103dc6000305af16105ae57600080fd5b610460805160200180610360828460006004600a8704601201f16105d157600080fd5b50506002546104a05260006104c0526104a05160016104a0510110156105f657600080fd5b60016104a051016104e05261050060006020818352015b600160016104e05116141561062157610674565b6104c060605160018251018060405190131561063c57600080fd5b809190121561064a57600080fd5b8152506104e0600261065b57600080fd5b60028151048152505b815160010180835281141561060d575b505060006101a06030806020846105e001018260208501600060046016f1505080518201915050600060106020820661056001602082840111156106b757600080fd5b60208061058082610520600060046015f15050818152809050905090506010806020846105e001018260208501600060046013f1505080518201915050806105e0526105e09050602060c0825160208401600060025af161071757600080fd5b60c051905061054052600060006040602082066106800161028051828401111561074057600080fd5b6060806106a0826020602088068803016102800160006004601bf1505081815280905090509050602060c0825160208401600060025af161078057600080fd5b60c051905060208261088001015260208101905060006040602060208206610740016102805182840111156107b457600080fd5b606080610760826020602088068803016102800160006004601bf150508181528090509050905060208060208461080001018260208501600060046015f15050805182019150506105205160208261080001015260208101905080610800526108009050602060c0825160208401600060025af161083157600080fd5b60c051905060208261088001015260208101905080610880526108809050602060c0825160208401600060025af161086857600080fd5b60c051905061066052600060006105405160208261092001015260208101905061022060208060208461092001018260208501600060046015f150508051820191505080610920526109209050602060c0825160208401600060025af16108ce57600080fd5b60c0519050602082610aa00101526020810190506000610360600880602084610a2001018260208501600060046012f150508051820191505060006018602082066109a0016020828401111561092357600080fd5b6020806109c082610520600060046015f1505081815280905090509050601880602084610a2001018260208501600060046014f150508051820191505061066051602082610a2001015260208101905080610a2052610a209050602060c0825160208401600060025af161099657600080fd5b60c0519050602082610aa001015260208101905080610aa052610aa09050602060c0825160208401600060025af16109cd57600080fd5b60c051905061090052610b2060006020818352015b6104c051610b20511215610a62576000610b205160208110610a0357600080fd5b600160c052602060c0200154602082610b4001015260208101905061090051602082610b4001015260208101905080610b4052610b409050602060c0825160208401600060025af1610a5457600080fd5b60c051905061090052610a67565b610a78565b5b81516001018083528114156109e2575b5050610900516104c05160208110610a8f57600080fd5b600160c052602060c02001556002805460018254011015610aaf57600080fd5b60018154018155506060610c4060246380673289610bc0526104a051610be052610bdc6000305af1610ae057600080fd5b610c60805160200180610ca0828460006004600a8704601201f1610b0357600080fd5b505060a0610d2052610d2051610d60526101a0805160200180610d2051610d6001828460006004600a8704601201f1610b3b57600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516040818352015b83610d0051101515610b7957610b96565b6000610d00516020850101535b8151600101808352811415610b68575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610d8052610220805160200180610d2051610d6001828460006004600a8704601201f1610bed57600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610c2b57610c48565b6000610d00516020850101535b8151600101808352811415610c1a575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610da052610360805160200180610d2051610d6001828460006004600a8704601201f1610c9f57600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610cdd57610cfa565b6000610d00516020850101535b8151600101808352811415610ccc575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610dc052610280805160200180610d2051610d6001828460006004600a8704601201f1610d5157600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516060818352015b83610d0051101515610d8f57610dac565b6000610d00516020850101535b8151600101808352811415610d7e575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d2052610d2051610de052610ca0805160200180610d2051610d6001828460006004600a8704601201f1610e0357600080fd5b5050610d2051610d60015160206001820306601f8201039050610d2051610d6001610d0081516020818352015b83610d0051101515610e4157610e5e565b6000610d00516020850101535b8151600101808352811415610e30575b505050506020610d2051610d60015160206001820306601f8201039050610d20510101610d20527fdc5fc95703516abd38fa03c3737ff3b52dc52347055c8028460fdf5bbe2f12ce610d2051610d60a164077359400061032051101515611254576003805460018254011015610ed357600080fd5b60018154018155506201000060035414156112535742610e205242610e405262015180610eff57600080fd5b62015180610e405106610e20511015610f1757600080fd5b42610e405262015180610f2957600080fd5b62015180610e405106610e2051036202a30042610e205242610e405262015180610f5257600080fd5b62015180610e405106610e20511015610f6a57600080fd5b42610e405262015180610f7c57600080fd5b62015180610e405106610e205103011015610f9657600080fd5b6202a30042610e205242610e405262015180610fb157600080fd5b62015180610e405106610e20511015610fc957600080fd5b42610e405262015180610fdb57600080fd5b62015180610e405106610e20510301610e00526020610ee0600463c5f2892f610e8052610e9c6000305af161100f57600080fd5b610ee051610e60526060610f8060246380673289610f0052600254610f2052610f1c6000305af161103f57600080fd5b610fa0805160200180610fe0828460006004600a8704601201f161106257600080fd5b505060606110c06024638067328961104052610e00516110605261105c6000305af161108d57600080fd5b6110e0805160200180611120828460006004600a8704601201f16110b057600080fd5b5050610e60516111e05260606111a0526111a05161120052610fe08051602001806111a0516111e001828460006004600a8704601201f16110f057600080fd5b50506111a0516111e0015160206001820306601f82010390506111a0516111e00161118081516020818352015b836111805110151561112e5761114b565b6000611180516020850101535b815160010180835281141561111d575b5050505060206111a0516111e0015160206001820306601f82010390506111a05101016111a0526111a051611220526111208051602001806111a0516111e001828460006004600a8704601201f16111a257600080fd5b50506111a0516111e0015160206001820306601f82010390506111a0516111e00161118081516020818352015b83611180511015156111e0576111fd565b6000611180516020850101535b81516001018083528114156111cf575b5050505060206111a0516111e0015160206001820306601f82010390506111a05101016111a0527f08b71ef3f1b58f7a23ffb82e27f12f0888c8403f1ceb0ea7ea26b274e2189d4c6111a0516111e0a160016004555b5b005b63845980e8600051141561127c57341561126f57600080fd5b60045460005260206000f3005b60006000fd5b61018561140703610185600039610185611407036000f3"} \ No newline at end of file diff --git a/deposit_contract/contracts/validator_registration.v.py b/deposit_contract/contracts/validator_registration.v.py index 82704f59ef..1d475311ae 100644 --- a/deposit_contract/contracts/validator_registration.v.py +++ b/deposit_contract/contracts/validator_registration.v.py @@ -88,12 +88,12 @@ def deposit(pubkey: bytes[PUBKEY_LENGTH], # add deposit to merkle tree i: int128 = 0 - power_of_two: uint256 = 2 + size: uint256 = index + 1 for _ in range(DEPOSIT_CONTRACT_TREE_DEPTH): - if (index+1) % power_of_two != 0: + if bitwise_and(size, 1) == 1: break i += 1 - power_of_two *= 2 + size /= 2 zero_bytes_32: bytes32 pubkey_root: bytes32 = sha256(concat(pubkey, slice(zero_bytes_32, start=0, len=16))) From cc5b172da3d981f974e999dc21018bba4f95bcac Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 30 May 2019 12:38:55 +1000 Subject: [PATCH 224/308] Test deposit top-up with inconsistent withdrawal credentials (#1133) * Simplify deposits * Avoid divisions by zero Possible fix to avoid four cases of divisions by zero: * `return state.validator_registry[index].effective_balance // adjusted_quotient // BASE_REWARDS_PER_EPOCH` * `rewards[index] += get_base_reward(state, index) * attesting_balance // total_balance` * `validator.effective_balance * min(total_penalties * 3, total_balance) // total_balance` * `rewards[index] += base_reward * attesting_balance // committee_balance` See also #1107. * fix deposit test for new index handling * tests: deposit with inconsistent withdrawal credentials * Update README.md * Update 0_beacon-chain.md * Fix linter errors * Update test_process_deposit.py * fix deposit test * fix lint --- .../block_processing/test_process_deposit.py | 18 ++++++++++++++++++ .../pyspec/eth2spec/test/helpers/deposits.py | 16 +++++++++++----- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py index 0ef2e509a7..0430dd12fe 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py @@ -94,6 +94,22 @@ def test_invalid_sig_top_up(state): yield from run_deposit_processing(state, deposit, validator_index, valid=True, effective=True) +@spec_state_test +def test_invalid_withdrawal_credentials_top_up(state): + validator_index = 0 + amount = spec.MAX_EFFECTIVE_BALANCE // 4 + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(b"junk")[1:] + deposit = prepare_state_and_deposit( + state, + validator_index, + amount, + withdrawal_credentials=withdrawal_credentials + ) + + # inconsistent withdrawal credentials, in top-ups, are allowed! + yield from run_deposit_processing(state, deposit, validator_index, valid=True, effective=True) + + @spec_state_test def test_wrong_index(state): validator_index = len(state.validator_registry) @@ -122,6 +138,7 @@ def test_wrong_deposit_for_deposit_count(state): pubkey_1, privkey_1, spec.MAX_EFFECTIVE_BALANCE, + withdrawal_credentials=b'\x00'*32, signed=True, ) deposit_count_1 = len(deposit_data_leaves) @@ -136,6 +153,7 @@ def test_wrong_deposit_for_deposit_count(state): pubkey_2, privkey_2, spec.MAX_EFFECTIVE_BALANCE, + withdrawal_credentials=b'\x00'*32, signed=True, ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index c5deb124e6..2db3ae03c3 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -8,11 +8,10 @@ from eth2spec.utils.minimal_ssz import signing_root -def build_deposit_data(state, pubkey, privkey, amount, signed=False): +def build_deposit_data(state, pubkey, privkey, amount, withdrawal_credentials, signed=False): deposit_data = DepositData( pubkey=pubkey, - # insecurely use pubkey as withdrawal key as well - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(pubkey)[1:], + withdrawal_credentials=withdrawal_credentials, amount=amount, ) if signed: @@ -37,8 +36,9 @@ def build_deposit(state, pubkey, privkey, amount, + withdrawal_credentials, signed): - deposit_data = build_deposit_data(state, pubkey, privkey, amount, signed) + deposit_data = build_deposit_data(state, pubkey, privkey, amount, withdrawal_credentials, signed) item = deposit_data.hash_tree_root() index = len(deposit_data_leaves) @@ -57,7 +57,7 @@ def build_deposit(state, return deposit, root, deposit_data_leaves -def prepare_state_and_deposit(state, validator_index, amount, signed=False): +def prepare_state_and_deposit(state, validator_index, amount, withdrawal_credentials=None, signed=False): """ Prepare the state for the deposit, and create a deposit for the given validator, depositing the given amount. """ @@ -67,12 +67,18 @@ def prepare_state_and_deposit(state, validator_index, amount, signed=False): pubkey = pubkeys[validator_index] privkey = privkeys[validator_index] + + # insecurely use pubkey as withdrawal key if no credentials provided + if withdrawal_credentials is None: + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(pubkey)[1:] + deposit, root, deposit_data_leaves = build_deposit( state, deposit_data_leaves, pubkey, privkey, amount, + withdrawal_credentials, signed ) From c277703423c1b8af0662ef0831fe88f4a4666f04 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 May 2019 13:26:36 +0800 Subject: [PATCH 225/308] Add `compile_deposit_contract` command --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index e80efcd7ea..47be51e0aa 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,11 @@ lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ flake8 --max-line-length=120 ./eth2spec; +compile_deposit_contract: + cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; \ + cd ../..; cd $(DEPOSIT_CONTRACT_DIR); \ + python tool/compile_deposit_contract.py contracts/validator_registration.v.py; + install_deposit_contract_test: cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; \ cd ../..; cd $(DEPOSIT_CONTRACT_DIR); \ From 0424fab024a8580cd486d0708ca79d0ae7ffb889 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 May 2019 13:39:42 +0800 Subject: [PATCH 226/308] Use `eth2spec.utils.hash_function.hash` --- deposit_contract/tests/contracts/test_deposit.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py index 7c5b85907a..3d0b6f8453 100644 --- a/deposit_contract/tests/contracts/test_deposit.py +++ b/deposit_contract/tests/contracts/test_deposit.py @@ -13,17 +13,15 @@ FULL_DEPOSIT_AMOUNT, MIN_DEPOSIT_AMOUNT, ) + +from eth2spec.phase0.spec import ( + DepositData, +) +from eth2spec.utils.hash_function import hash from eth2spec.utils.minimal_ssz import ( SSZType, hash_tree_root, ) -from eth2spec.phase0.spec import ( - DepositData, -) - - -def hash(data): - return sha256(data).digest() def compute_merkle_root(leaf_nodes): From a81fa54c73d8b1cfcc6a96a54513c7d84e556674 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 May 2019 14:12:05 +0800 Subject: [PATCH 227/308] Add README.md --- deposit_contract/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 deposit_contract/README.md diff --git a/deposit_contract/README.md b/deposit_contract/README.md new file mode 100644 index 0000000000..266eb93e42 --- /dev/null +++ b/deposit_contract/README.md @@ -0,0 +1,23 @@ +# Deposit contract + +## How to compile the contract? + +Under the `eth2.0-specs` directory, execute: + +```sh +make compile_deposit_contract +``` + +The ABI and bytecode will be updated at [`contracts/validator_registration.json`](./contracts/validator_registration.json). + +## How to set up the testing environment? + +```sh +make install_deposit_contract_test +``` + +## How to run tests? + +```sh +make test_deposit_contract +``` From ebdf74c6b808fb5583ace4a03a013eee35c33324 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 May 2019 15:57:39 +0800 Subject: [PATCH 228/308] kick the CI --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 694c7bc921..ed3e16f63e 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -12,6 +12,7 @@ class uint(int): byte_len = 0 + def __new__(cls, value, *args, **kwargs): if value < 0: raise ValueError("unsigned types must not be negative") From 8fae0f8c78deb30a10be45457d0f8b668349e62e Mon Sep 17 00:00:00 2001 From: Vitalik Buterin Date: Thu, 30 May 2019 09:42:42 -0400 Subject: [PATCH 229/308] Added support for SSZ partials --- .../pyspec/eth2spec/utils/ssz/ssz_impl.py | 15 +- .../pyspec/eth2spec/utils/ssz/ssz_partials.py | 208 +++++++++++++++++- 2 files changed, 213 insertions(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 21dba1034d..0ef89f97f6 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -14,11 +14,22 @@ def is_basic_type(typ): def serialize_basic(value, typ): if is_uint_type(typ): return value.to_bytes(uint_byte_size(typ), 'little') - if is_bool_type(typ): + elif is_bool_type(typ): if value: return b'\x01' else: return b'\x00' + else: + raise Exception("Type not supported: {}".format(typ)) + +def deserialize_basic(value, typ): + if is_uint_type(typ): + return typ(int.from_bytes(value, 'little')) + elif is_bool_type(typ): + assert value in (b'\x00', b'\x01') + return True if value == b'\x01' else False + else: + raise Exception("Type not supported: {}".format(typ)) def is_fixed_size(typ): @@ -112,7 +123,7 @@ def get_typed_values(obj, typ=None): return obj.get_typed_values() elif is_list_kind(typ) or is_vector_kind(typ): elem_type = read_elem_type(typ) - return zip(obj, [elem_type] * len(obj)) + return list(zip(obj, [elem_type] * len(obj))) else: raise Exception("Invalid type") diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py index b30ebfb4c6..6a243f4f35 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py @@ -36,7 +36,7 @@ def filler(starting_position, chunk_count): while at % (skip*2) == 0: skip *= 2 value = hash(value + value) - o[starting_position + at] = value + o[(starting_position + at) // skip] = value at += skip return o @@ -47,19 +47,211 @@ def merkle_tree_of_chunks(chunks, root): return o @infer_input_type -def ssz_all(obj, typ=None, root=1): - if is_list_type(typ): +def ssz_leaves(obj, typ=None, root=1): + if is_list_kind(typ): o = {root * 2 + 1: len(obj).to_bytes(32, 'little')} base = root * 2 else: o = {} base = root - if is_bottom_layer_type(typ): - data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, read_elem_typ(typ)) - return {**o, **merkle_tree_of_chunks(chunkify(data), base)} + if is_bottom_layer_kind(typ): + data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, read_elem_type(typ)) + q = {**o, **merkle_tree_of_chunks(chunkify(data), base)} + #print(obj, root, typ, base, list(q.keys())) + return(q) else: fields = get_typed_values(obj, typ=typ) sub_base = base * next_power_of_two(len(fields)) for i, (elem, elem_type) in enumerate(fields): - o = {**o, **ssz_all(elem, typ=elem_type, root=sub_base+i)} - return {**o, **filter(sub_base, len(fields))} + o = {**o, **ssz_leaves(elem, typ=elem_type, root=sub_base+i)} + q = {**o, **filler(sub_base, len(fields))} + #print(obj, root, typ, base, list(q.keys())) + return(q) + +def fill(objects): + objects = {k:v for k,v in objects.items()} + keys = sorted(objects.keys())[::-1] + pos = 0 + while pos < len(keys): + k = keys[pos] + if k in objects and k^1 in objects and k//2 not in objects: + objects[k//2] = hash(objects[k&-2] + objects[k|1]) + keys.append(k // 2) + pos += 1 + return objects + +@infer_input_type +def ssz_full(obj, typ=None): + return fill(ssz_leaves(obj, typ=typ)) + +def get_basic_type_size(typ): + if is_uint_type(typ): + return uint_byte_size(typ) + elif is_bool_type(typ): + return 1 + else: + raise Exception("Type not basic: {}".format(typ)) + +def get_bottom_layer_element_position(typ, base, length, index): + """ + Returns the generalized index and the byte range of the index'th value + in the list with the given base generalized index and given length + """ + assert index < (1 if is_basic_type(typ) else length) + elem_typ = typ if is_basic_type(typ) else read_elem_type(typ) + elem_size = get_basic_type_size(elem_typ) + chunk_index = index * elem_size // 32 + chunk_count = (1 if is_basic_type(typ) else length) * elem_size // 32 + generalized_index = base * next_power_of_two(chunk_count) + chunk_index + start = elem_size * index % 32 + return generalized_index, start, start+elem_size + +@infer_input_type +def get_generalized_indices(obj, path, typ=None, root=1): + if len(path) == 0: + return [root] if is_basic_type(typ) else list(ssz_leaves(obj, typ=typ, root=root).keys()) + if path[0] == '__len__': + return [root * 2 + 1] if is_list_type(typ) else [] + base = root * 2 if is_list_kind(typ) else root + if is_bottom_layer_kind(typ): + length = 1 if is_basic_type(typ) else len(obj) + index, _, _ = get_bottom_layer_element_position(typ, base, length, path[0]) + return [index] + else: + if is_container_type(typ): + fields = typ.get_field_names() + field_count, index = len(fields), fields.index(path[0]) + elem_type = typ.get_field_types()[index] + child = obj.get_field_values()[index] + else: + field_count, index, elem_type, child = len(obj), path[0], read_elem_type(typ), obj[path[0]] + return get_generalized_indices( + child, + path[1:], + typ=elem_type, + root=base * next_power_of_two(field_count) + index + ) + +def get_branch_indices(tree_index): + o = [tree_index, tree_index ^ 1] + while o[-1] > 1: + o.append((o[-1] // 2) ^ 1) + return o[:-1] + +def remove_redundant_indices(obj): + return {k:v for k,v in obj.items() if not (k*2 in obj and k*2+1 in obj)} + +def merge(*args): + o = {} + for arg in args: + o = {**o, **arg} + return fill(o) + +@infer_input_type +def get_nodes_along_path(obj, path, typ=None): + indices = get_generalized_indices(obj, path, typ=typ) + return remove_redundant_indices(merge(*({i:obj.objects[i] for i in get_branch_indices(index)} for index in indices))) + +class OutOfRangeException(Exception): + pass + +class SSZPartial(): + def __init__(self, typ, objects, root=1): + assert not is_basic_type(typ) + self.objects = objects + self.typ = typ + self.root = root + if is_container_type(self.typ): + for field in self.typ.get_field_names(): + try: + setattr(self, field, self.getter(field)) + except OutOfRangeException: + pass + + def getter(self, index): + base = self.root * 2 if is_list_kind(self.typ) else self.root + if is_bottom_layer_kind(self.typ): + tree_index, start, end = get_bottom_layer_element_position( + self.typ, base, len(self), index + ) + if tree_index not in self.objects: + raise OutOfRangeException("Do not have required data") + else: + return deserialize_basic( + self.objects[tree_index][start:end], + self.typ if is_basic_type(self.typ) else read_elem_type(self.typ) + ) + else: + if is_container_type(self.typ): + fields = self.typ.get_field_names() + field_count, index = len(fields), fields.index(index) + elem_type = self.typ.get_field_types()[index] + else: + field_count, index, elem_type = len(self), index, read_elem_type(self.typ) + tree_index = base * next_power_of_two(field_count) + index + if tree_index not in self.objects: + raise OutOfRangeException("Do not have required data") + if is_basic_type(elem_type): + return deserialize_basic(self.objects[tree_index][:get_basic_type_size(elem_type)], elem_type) + else: + return ssz_partial(elem_type, self.objects, root=tree_index) + + def __getitem__(self, index): + return self.getter(index) + + def __iter__(self): + return (self[i] for i in range(len(self))) + + def __len__(self): + if is_list_kind(self.typ): + if self.root*2+1 not in self.objects: + raise OutOfRangeException("Do not have required data: {}".format(self.root*2+1)) + return int.from_bytes(self.objects[self.root*2+1], 'little') + elif is_vector_kind(self.typ): + return self.typ.length + elif is_container_type(self.typ): + return len(self.typ.get_fields()) + else: + raise Exception("Unsupported type: {}".format(self.typ)) + + def full_value(self): + if is_bytes_type(self.typ) or is_bytesn_type(self.typ): + return bytes([self.getter(i) for i in range(len(self))]) + elif is_list_kind(self.typ): + return [self[i] for i in range(len(self))] + elif is_vector_kind(self.typ): + return self.typ(*(self[i] for i in range(len(self)))) + elif is_container_type(self.typ): + full_value = lambda x: x.full_value() if hasattr(x, 'full_value') else x + return self.typ(**{field: full_value(self.getter(field)) for field in self.typ.get_field_names()}) + elif is_basic_type(self.typ): + return self.getter(0) + else: + raise Exception("Unsupported type: {}".format(self.typ)) + + def hash_tree_root(self): + o = {**self.objects} + keys = sorted(o.keys())[::-1] + pos = 0 + while pos < len(keys): + k = keys[pos] + if k in o and k^1 in o and k//2 not in o: + o[k//2] = hash(o[k&-2] + o[k|1]) + keys.append(k // 2) + pos += 1 + return o[self.root] + + def __str__(self): + return str(self.full_value()) + +def ssz_partial(typ, objects, root=1): + ssz_type = ( + Container if is_container_type(typ) else + typ if (is_vector_type(typ) or is_bytesn_type(typ)) else object + ) + class Partial(SSZPartial, ssz_type): + pass + if is_container_type(typ): + Partial.__annotations__ = typ.__annotations__ + o = Partial(typ, objects, root=root) + return o From ed71efc061954fb87871fa60824ad236e08b7f40 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 31 May 2019 10:41:39 +0200 Subject: [PATCH 230/308] state is kwarg --- .../test_process_proposer_slashing.py | 4 ++-- test_libs/pyspec/eth2spec/test/context.py | 17 ++++++++++------- .../epoch_processing/test_process_crosslinks.py | 6 ++---- .../test_process_registry_updates.py | 10 +++++----- .../eth2spec/test/helpers/attestations.py | 1 + 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py index b35241859c..47f9ed1de8 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py @@ -40,17 +40,17 @@ def run_proposer_slashing_processing(spec, state, proposer_slashing, valid=True) ) -@with_all_phases @spec_state_test +@with_all_phases def test_success(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) yield from run_proposer_slashing_processing(spec, state, proposer_slashing) -@with_all_phases @always_bls @spec_state_test +@with_all_phases def test_invalid_sig_1(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=False, signed_2=True) yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 6b92f21512..dcdd8140ba 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -4,10 +4,14 @@ from .helpers.genesis import create_genesis_state -from .utils import spectest, with_args, with_tags +from .utils import spectest, with_tags -# Provides a genesis state as first argument to the function decorated with this -with_state = with_args(lambda: [create_genesis_state(spec.SLOTS_PER_EPOCH * 8)]) + +def with_state(fn): + def entry(*args, **kw): + kw['state'] = create_genesis_state(spec=spec_phase0, num_validators=spec_phase0.SLOTS_PER_EPOCH * 8) + return fn(*args, **kw) + return entry # BLS is turned off by default *for performance purposes during TESTING*. @@ -88,18 +92,17 @@ def with_phase0(fn): Decorator to use phase 0's spec and helpers """ def entry(*args, **kw): - args = (spec_phase0, *args) - print(args) + kw['spec'] = spec_phase0 return fn(*args, **kw) return entry def with_phase1(fn): """ - Decorator to use phase 0's spec and helpers + Decorator to use phase 1's spec and helpers """ def entry(*args, **kw): - args = (spec_phase1, *args) + kw['spec'] = spec_phase1 return fn(*args, **kw) return entry diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py index 54e8d5ef2b..65d9586788 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py @@ -24,13 +24,13 @@ def run_process_crosslinks(spec, state, valid=True): """ # transition state to slot before state transition slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = build_empty_block_for_next_slot(state) + block = build_empty_block_for_next_slot(spec, state) block.slot = slot sign_block(spec, state, block) spec.state_transition(state, block) # cache state before epoch transition - spec.spec.process_slot(state) + spec.process_slot(state) yield 'pre', state spec.process_crosslinks(state) @@ -91,7 +91,6 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): # ensure rewarded for index in spec.get_crosslink_committee( - spec, state, attestation.data.target_epoch, attestation.data.crosslink.shard): @@ -144,7 +143,6 @@ def test_double_late_crosslink(spec, state): assert state.previous_crosslinks[shard] == state.current_crosslinks[shard] # ensure no reward, only penalties for the failed crosslink for index in spec.get_crosslink_committee( - spec, state, attestation_2.data.target_epoch, attestation_2.data.crosslink.shard): diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py index a3249a406b..e6679f8448 100644 --- a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py @@ -35,13 +35,13 @@ def run_process_registry_updates(spec, state, valid=True): @spec_state_test def test_activation(spec, state): index = 0 - assert spec.is_active_validator(state.validator_registry[index], spec.spec.get_current_epoch(state)) + assert spec.is_active_validator(state.validator_registry[index], spec.get_current_epoch(state)) # Mock a new deposit state.validator_registry[index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH state.validator_registry[index].activation_epoch = spec.FAR_FUTURE_EPOCH state.validator_registry[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE - assert not spec.is_active_validator(state.validator_registry[index], spec.spec.get_current_epoch(state)) + assert not spec.is_active_validator(state.validator_registry[index], spec.get_current_epoch(state)) for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): next_epoch(spec, state) @@ -52,7 +52,7 @@ def test_activation(spec, state): assert state.validator_registry[index].activation_epoch != spec.FAR_FUTURE_EPOCH assert spec.is_active_validator( state.validator_registry[index], - spec.spec.get_current_epoch(state), + spec.get_current_epoch(state), ) @@ -60,7 +60,7 @@ def test_activation(spec, state): @spec_state_test def test_ejection(spec, state): index = 0 - assert spec.is_active_validator(state.validator_registry[index], spec.spec.get_current_epoch(state)) + assert spec.is_active_validator(state.validator_registry[index], spec.get_current_epoch(state)) assert state.validator_registry[index].exit_epoch == spec.FAR_FUTURE_EPOCH # Mock an ejection @@ -74,5 +74,5 @@ def test_ejection(spec, state): assert state.validator_registry[index].exit_epoch != spec.FAR_FUTURE_EPOCH assert not spec.is_active_validator( state.validator_registry[index], - spec.spec.get_current_epoch(state), + spec.get_current_epoch(state), ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index a4eac3fb70..369738ed4a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -89,6 +89,7 @@ def sign_aggregate_attestation(spec, state, attestation_data, participants: List privkey = privkeys[validator_index] signatures.append( get_attestation_signature( + spec, state, attestation_data, privkey From ec9f8f15ed7c11c637f4ea7addd8f764093a9457 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 31 May 2019 11:32:53 +0200 Subject: [PATCH 231/308] Adds Custody Tests --- ...est_process_early_derived_secret_reveal.py | 122 ++++++++++++++++++ .../pyspec/eth2spec/test/helpers/custody.py | 38 ++++++ 2 files changed, 160 insertions(+) create mode 100644 test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py create mode 100644 test_libs/pyspec/eth2spec/test/helpers/custody.py diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py new file mode 100644 index 0000000000..0f9057bb6a --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py @@ -0,0 +1,122 @@ +from eth2spec.test.helpers.custody import get_valid_early_derived_secret_reveal +from eth2spec.test.context import with_phase1, spec_state_test, expect_assertion_error + + +def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, valid=True): + """ + Run ``process_randao_key_reveal``, yielding: + - pre-state ('pre') + - randao_key_reveal ('randao_key_reveal') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + yield 'pre', state + yield 'randao_key_reveal', randao_key_reveal + + if not valid: + expect_assertion_error(lambda: spec.process_early_derived_secret_reveal(state, randao_key_reveal)) + yield 'post', None + return + + spec.process_early_derived_secret_reveal(state, randao_key_reveal) + + slashed_validator = state.validator_registry[randao_key_reveal.revealed_index] + + if randao_key_reveal.epoch >= spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING: + assert slashed_validator.slashed + assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH + assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH + # lost whistleblower reward + # FIXME: Currently broken because get_base_reward in genesis epoch is 0 + assert ( + state.balances[randao_key_reveal.revealed_index] < + state.balances[randao_key_reveal.revealed_index] + ) + yield 'post', state + + +@with_phase1 +@spec_state_test +def test_success(spec, state): + randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state) + + yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal) + + +@with_phase1 +@spec_state_test +def test_reveal_from_current_epoch(spec, state): + randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state)) + + yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) + + +# @with_phase1 +# @spec_state_test +# def test_reveal_from_past_epoch(state): +# randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state) - 1) +# +# yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) + + +@with_phase1 +@spec_state_test +def test_reveal_with_custody_padding(spec, state): + randao_key_reveal = get_valid_early_derived_secret_reveal( + spec, + state, + spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING, + ) + yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, True) + + +@with_phase1 +@spec_state_test +def test_reveal_with_custody_padding_minus_one(spec, state): + randao_key_reveal = get_valid_early_derived_secret_reveal( + spec, + state, + spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING - 1, + ) + yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, True) + + +# @with_phase1 +# @spec_state_test +# def test_double_reveal(spec, state): +# randao_key_reveal1 = get_valid_early_derived_secret_reveal( +# spec, +# state, +# spec.get_current_epoch(state) + spec.RANDAO_PENALTY_EPOCHS + 1, +# ) +# pre_state, intermediate_state = run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal1) + +# randao_key_reveal2 = get_valid_early_derived_secret_reveal( +# spec, +# intermediate_state, +# spec.get_current_epoch(pre_state) + spec.RANDAO_PENALTY_EPOCHS + 1, +# ) +# _, post_state = run_early_derived_secret_reveal_processing(spec, intermediate_state, randao_key_reveal2, False) + +# return pre_state, [randao_key_reveal1, randao_key_reveal2], post_state + + +@with_phase1 +@spec_state_test +def test_revealer_is_slashed(spec, state): + randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state)) + state.validator_registry[randao_key_reveal.revealed_index].slashed = True + + yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) + + +@with_phase1 +@spec_state_test +def test_far_future_epoch(spec, state): + randao_key_reveal = get_valid_early_derived_secret_reveal( + spec, + state, + spec.get_current_epoch(state) + spec.EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS, + ) + + yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) diff --git a/test_libs/pyspec/eth2spec/test/helpers/custody.py b/test_libs/pyspec/eth2spec/test/helpers/custody.py new file mode 100644 index 0000000000..67df12fcdb --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/custody.py @@ -0,0 +1,38 @@ +from eth2spec.test.helpers.keys import privkeys +from eth2spec.utils.bls import bls_sign + + +def get_valid_early_derived_secret_reveal(spec, state, epoch=None): + current_epoch = spec.get_current_epoch(state) + revealed_index = spec.get_active_validator_indices(state, current_epoch)[-1] + masker_index = spec.get_active_validator_indices(state, current_epoch)[0] + + if epoch is None: + epoch = current_epoch + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING + + reveal = bls_sign( + message_hash=spec.hash_tree_root(epoch), + privkey=privkeys[revealed_index], + domain=spec.get_domain( + state=state, + domain_type=spec.DOMAIN_RANDAO, + message_epoch=epoch, + ), + ) + mask = bls_sign( + message_hash=spec.hash_tree_root(epoch), + privkey=privkeys[masker_index], + domain=spec.get_domain( + state=state, + domain_type=spec.DOMAIN_RANDAO, + message_epoch=epoch, + ), + ) + + return spec.EarlyDerivedSecretReveal( + revealed_index=revealed_index, + epoch=epoch, + reveal=reveal, + masker_index=masker_index, + mask=mask, + ) From 0c57cfcaa7b62191ad327121a440c288ad930668 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 31 May 2019 11:33:47 +0200 Subject: [PATCH 232/308] state builds from spec version --- .../test_process_proposer_slashing.py | 4 +- test_libs/pyspec/eth2spec/test/context.py | 5 +- ...est_process_early_derived_secret_reveal.py | 116 ------------------ 3 files changed, 6 insertions(+), 119 deletions(-) delete mode 100644 test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py index 47f9ed1de8..b35241859c 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py @@ -40,17 +40,17 @@ def run_proposer_slashing_processing(spec, state, proposer_slashing, valid=True) ) -@spec_state_test @with_all_phases +@spec_state_test def test_success(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) yield from run_proposer_slashing_processing(spec, state, proposer_slashing) +@with_all_phases @always_bls @spec_state_test -@with_all_phases def test_invalid_sig_1(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=False, signed_2=True) yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index dcdd8140ba..d0869ba686 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -9,7 +9,10 @@ def with_state(fn): def entry(*args, **kw): - kw['state'] = create_genesis_state(spec=spec_phase0, num_validators=spec_phase0.SLOTS_PER_EPOCH * 8) + try: + kw['state'] = create_genesis_state(spec=kw['spec'], num_validators=spec_phase0.SLOTS_PER_EPOCH * 8) + except KeyError: + raise TypeError('Spec decorator must come before state decorator to inject spec into state.') return fn(*args, **kw) return entry diff --git a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py deleted file mode 100644 index ad436ff29e..0000000000 --- a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_process_early_derived_secret_reveal.py +++ /dev/null @@ -1,116 +0,0 @@ -from copy import deepcopy -import pytest - -# mark entire file as 'randao_key_reveals' -pytestmark = pytest.mark.randao_key_reveals - - -def run_early_derived_secret_reveal_processing(state, randao_key_reveal, valid=True): - """ - Run ``process_randao_key_reveal`` returning the pre and post state. - If ``valid == False``, run expecting ``AssertionError`` - """ - post_state = deepcopy(state) - - if not valid: - with pytest.raises(AssertionError): - spec.process_early_derived_secret_reveal(post_state, randao_key_reveal) - return state, None - - spec.process_early_derived_secret_reveal(post_state, randao_key_reveal) - - slashed_validator = post_state.validator_registry[randao_key_reveal.revealed_index] - - if randao_key_reveal.epoch >= spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING: - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - # lost whistleblower reward - # FIXME: Currently broken because get_base_reward in genesis epoch is 0 - assert ( - post_state.balances[randao_key_reveal.revealed_index] < - state.balances[randao_key_reveal.revealed_index] - ) - - return state, post_state - - -def test_success(state): - randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state) - - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal) - - return pre_state, randao_key_reveal, post_state - - -def test_reveal_from_current_epoch(state): - randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state)) - - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) - - return pre_state, randao_key_reveal, post_state - - -@pytest.mark.skip(reason="Not currently possible as we are testing at epoch 0") -def test_reveal_from_past_epoch(state): - randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state) - 1) - - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) - - return pre_state, randao_key_reveal, post_state - - -def test_reveal_with_custody_padding(state): - randao_key_reveal = helpers.get_valid_early_derived_secret_reveal( - state, - spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING, - ) - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) - - return pre_state, randao_key_reveal, post_state - - -def test_reveal_with_custody_padding_minus_one(state): - randao_key_reveal = helpers.get_valid_early_derived_secret_reveal( - state, - spec.get_current_epoch(state) + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING - 1, - ) - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, True) - - return pre_state, randao_key_reveal, post_state - - -def test_double_reveal(state): - randao_key_reveal1 = helpers.get_valid_early_derived_secret_reveal( - state, - spec.get_current_epoch(state) + spec.RANDAO_PENALTY_EPOCHS + 1, - ) - pre_state, intermediate_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal1) - - randao_key_reveal2 = helpers.get_valid_early_derived_secret_reveal( - intermediate_state, - spec.get_current_epoch(pre_state) + spec.RANDAO_PENALTY_EPOCHS + 1, - ) - _, post_state = run_early_derived_secret_reveal_processing(intermediate_state, randao_key_reveal2, False) - - return pre_state, [randao_key_reveal1, randao_key_reveal2], post_state - - -def test_revealer_is_slashed(state): - randao_key_reveal = helpers.get_valid_early_derived_secret_reveal(state, spec.get_current_epoch(state)) - state.validator_registry[randao_key_reveal.revealed_index].slashed = True - - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) - - return pre_state, randao_key_reveal, post_state - - -def test_far_future_epoch(state): - randao_key_reveal = helpers.get_valid_early_derived_secret_reveal( - state, - spec.get_current_epoch(state) + spec.EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS, - ) - - pre_state, post_state = run_early_derived_secret_reveal_processing(state, randao_key_reveal, False) - - return pre_state, randao_key_reveal, post_state From 9e8a969f22dc187932a3c4f9ba8d8f68d467f0d1 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 31 May 2019 12:44:34 +0200 Subject: [PATCH 233/308] Removes old tests & fixes phase1 tests --- Makefile | 9 +- ...est_process_early_derived_secret_reveal.py | 47 +- test_libs/pyspec/eth2spec/test/conftest.py | 2 +- test_libs/pyspec/tests/phase0/__init__.py | 0 .../test_process_attestation.py | 180 -------- .../test_process_attester_slashing.py | 107 ----- .../test_process_block_header.py | 65 --- .../block_processing/test_process_deposit.py | 128 ------ .../test_process_proposer_slashing.py | 87 ---- .../block_processing/test_process_transfer.py | 132 ------ .../block_processing/test_voluntary_exit.py | 150 ------- test_libs/pyspec/tests/phase0/conftest.py | 37 -- .../test_process_crosslinks.py | 201 --------- .../test_process_registry_updates.py | 104 ----- test_libs/pyspec/tests/phase0/helpers.py | 400 ----------------- .../pyspec/tests/phase0/test_finality.py | 234 ---------- test_libs/pyspec/tests/phase0/test_sanity.py | 402 ------------------ test_libs/pyspec/tests/phase1/__init__.py | 0 .../test_phase0_block_processing.py | 7 - test_libs/pyspec/tests/phase1/conftest.py | 39 -- .../test_phase0_epoch_processing.py | 2 - test_libs/pyspec/tests/phase1/helpers.py | 39 -- .../pyspec/tests/phase1/test_finality.py | 4 - test_libs/pyspec/tests/phase1/test_sanity.py | 4 - 24 files changed, 27 insertions(+), 2353 deletions(-) delete mode 100644 test_libs/pyspec/tests/phase0/__init__.py delete mode 100644 test_libs/pyspec/tests/phase0/block_processing/test_process_attestation.py delete mode 100644 test_libs/pyspec/tests/phase0/block_processing/test_process_attester_slashing.py delete mode 100644 test_libs/pyspec/tests/phase0/block_processing/test_process_block_header.py delete mode 100644 test_libs/pyspec/tests/phase0/block_processing/test_process_deposit.py delete mode 100644 test_libs/pyspec/tests/phase0/block_processing/test_process_proposer_slashing.py delete mode 100644 test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py delete mode 100644 test_libs/pyspec/tests/phase0/block_processing/test_voluntary_exit.py delete mode 100644 test_libs/pyspec/tests/phase0/conftest.py delete mode 100644 test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py delete mode 100644 test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py delete mode 100644 test_libs/pyspec/tests/phase0/helpers.py delete mode 100644 test_libs/pyspec/tests/phase0/test_finality.py delete mode 100644 test_libs/pyspec/tests/phase0/test_sanity.py delete mode 100644 test_libs/pyspec/tests/phase1/__init__.py delete mode 100644 test_libs/pyspec/tests/phase1/block_processing_phase1/test_phase0_block_processing.py delete mode 100644 test_libs/pyspec/tests/phase1/conftest.py delete mode 100644 test_libs/pyspec/tests/phase1/epoch_processing/test_phase0_epoch_processing.py delete mode 100644 test_libs/pyspec/tests/phase1/helpers.py delete mode 100644 test_libs/pyspec/tests/phase1/test_finality.py delete mode 100644 test_libs/pyspec/tests/phase1/test_sanity.py diff --git a/Makefile b/Makefile index b80e81238d..af71f44fbd 100644 --- a/Makefile +++ b/Makefile @@ -43,8 +43,7 @@ test: $(PY_SPEC_ALL_TARGETS) citest: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; \ - python -m pytest --junitxml=test-reports/eth2spec/test_results_phase0.xml tests/phase0; \ - python -m pytest --junitxml=test-reports/eth2spec/test_results_phase0.xml tests/phase1 + python -m pytest --junitxml=test-reports/eth2spec/test_results_phase0.xml eth2spec install_lint: cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.5.0 @@ -57,12 +56,6 @@ lint: $(PY_SPEC_ALL_TARGETS) # "make pyspec" to create the pyspec for all phases. pyspec: $(PY_SPEC_ALL_TARGETS) -# "make phase0" to create pyspec for phase0 -phase0: $(PY_SPEC_PHASE_0_TARGETS) - -# "make phase1" to create pyspec for phase1 -phase1: $(PY_SPEC_PHASE_1_TARGETS) - $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $@ diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py index 0f9057bb6a..4b4608d4da 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py @@ -28,10 +28,10 @@ def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, v assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH # lost whistleblower reward # FIXME: Currently broken because get_base_reward in genesis epoch is 0 - assert ( - state.balances[randao_key_reveal.revealed_index] < - state.balances[randao_key_reveal.revealed_index] - ) + # assert ( + # state.balances[randao_key_reveal.revealed_index] < + # state.balances[randao_key_reveal.revealed_index] + # ) yield 'post', state @@ -81,24 +81,27 @@ def test_reveal_with_custody_padding_minus_one(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, True) -# @with_phase1 -# @spec_state_test -# def test_double_reveal(spec, state): -# randao_key_reveal1 = get_valid_early_derived_secret_reveal( -# spec, -# state, -# spec.get_current_epoch(state) + spec.RANDAO_PENALTY_EPOCHS + 1, -# ) -# pre_state, intermediate_state = run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal1) - -# randao_key_reveal2 = get_valid_early_derived_secret_reveal( -# spec, -# intermediate_state, -# spec.get_current_epoch(pre_state) + spec.RANDAO_PENALTY_EPOCHS + 1, -# ) -# _, post_state = run_early_derived_secret_reveal_processing(spec, intermediate_state, randao_key_reveal2, False) - -# return pre_state, [randao_key_reveal1, randao_key_reveal2], post_state +@with_phase1 +@spec_state_test +def test_double_reveal(spec, state): + randao_key_reveal1 = get_valid_early_derived_secret_reveal( + spec, + state, + spec.get_current_epoch(state) + spec.RANDAO_PENALTY_EPOCHS + 1, + ) + res = dict(run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal1)) + pre_state = res['pre'] + yield 'pre', pre_state + intermediate_state = res['post'] + + randao_key_reveal2 = get_valid_early_derived_secret_reveal( + spec, + intermediate_state, + spec.get_current_epoch(pre_state) + spec.RANDAO_PENALTY_EPOCHS + 1, + ) + post_state = dict(run_early_derived_secret_reveal_processing(spec, intermediate_state, randao_key_reveal2, False))['post'] + yield 'randao_key_reveal', [randao_key_reveal1, randao_key_reveal2] + yield 'post', post_state @with_phase1 diff --git a/test_libs/pyspec/eth2spec/test/conftest.py b/test_libs/pyspec/eth2spec/test/conftest.py index b2acb0573c..5713c34706 100644 --- a/test_libs/pyspec/eth2spec/test/conftest.py +++ b/test_libs/pyspec/eth2spec/test/conftest.py @@ -1,5 +1,5 @@ from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.phase0 import spec as spec_phase1 +from eth2spec.phase1 import spec as spec_phase1 # We import pytest only when it's present, i.e. when we are running tests. # The test-cases themselves can be generated without installing pytest. diff --git a/test_libs/pyspec/tests/phase0/__init__.py b/test_libs/pyspec/tests/phase0/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_attestation.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_attestation.py deleted file mode 100644 index 122195fc17..0000000000 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_attestation.py +++ /dev/null @@ -1,180 +0,0 @@ -from copy import deepcopy -import pytest - - -# mark entire file as 'attestations' -pytestmark = pytest.mark.attestations - - -def run_attestation_processing(state, attestation, valid=True): - """ - Run ``spec.process_attestation`` returning the pre and post state. - If ``valid == False``, run expecting ``AssertionError`` - """ - post_state = deepcopy(state) - - if not valid: - with pytest.raises(AssertionError): - spec.process_attestation(post_state, attestation) - return state, None - - spec.process_attestation(post_state, attestation) - - current_epoch = spec.get_current_epoch(state) - if attestation.data.target_epoch == current_epoch: - assert len(post_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1 - else: - assert len(post_state.previous_epoch_attestations) == len(state.previous_epoch_attestations) + 1 - - return state, post_state - - -def test_success(state): - attestation = helpers.get_valid_attestation(state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - pre_state, post_state = run_attestation_processing(state, attestation) - - return pre_state, attestation, post_state - - -def test_success_prevous_epoch(state): - attestation = helpers.get_valid_attestation(state) - block = helpers.build_empty_block_for_next_slot(state) - block.slot = state.slot + spec.SLOTS_PER_EPOCH - spec.state_transition(state, block) - - pre_state, post_state = run_attestation_processing(state, attestation) - - return pre_state, attestation, post_state - - -def test_success_since_max_epochs_per_crosslink(state): - for _ in range(spec.MAX_EPOCHS_PER_CROSSLINK + 2): - helpers.next_epoch(state) - - attestation = helpers.get_valid_attestation(state) - data = attestation.data - assert data.crosslink.end_epoch - data.crosslink.start_epoch == spec.MAX_EPOCHS_PER_CROSSLINK - - for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): - helpers.next_slot(state) - - pre_state, post_state = run_attestation_processing(state, attestation) - - return pre_state, attestation, post_state - - -def test_before_inclusion_delay(state): - attestation = helpers.get_valid_attestation(state) - # do not increment slot to allow for inclusion delay - - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state - - -def test_after_epoch_slots(state): - attestation = helpers.get_valid_attestation(state) - block = helpers.build_empty_block_for_next_slot(state) - # increment past latest inclusion slot - block.slot = state.slot + spec.SLOTS_PER_EPOCH + 1 - spec.state_transition(state, block) - - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state - - -def test_bad_source_epoch(state): - attestation = helpers.get_valid_attestation(state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - attestation.data.source_epoch += 10 - - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state - - -def test_bad_source_root(state): - attestation = helpers.get_valid_attestation(state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - attestation.data.source_root = b'\x42' * 32 - - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state - - -def test_non_zero_crosslink_data_root(state): - attestation = helpers.get_valid_attestation(state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - attestation.data.crosslink.data_root = b'\x42' * 32 - - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state - - -def test_bad_previous_crosslink(state): - helpers.next_epoch(state) - attestation = helpers.get_valid_attestation(state) - for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): - helpers.next_slot(state) - - attestation.data.crosslink.parent_root = b'\x27' * 32 - - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state - - -def test_bad_crosslink_start_epoch(state): - helpers.next_epoch(state) - attestation = helpers.get_valid_attestation(state) - for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): - helpers.next_slot(state) - - attestation.data.crosslink.start_epoch += 1 - - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state - - -def test_bad_crosslink_end_epoch(state): - helpers.next_epoch(state) - attestation = helpers.get_valid_attestation(state) - for _ in range(spec.MIN_ATTESTATION_INCLUSION_DELAY): - helpers.next_slot(state) - - attestation.data.crosslink.end_epoch += 1 - - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state - - -def test_non_empty_custody_bitfield(state): - attestation = helpers.get_valid_attestation(state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - attestation.custody_bitfield = deepcopy(attestation.aggregation_bitfield) - - pre_state, post_state = run_attestation_processing(state, attestation, False) - - return pre_state, attestation, post_state - - -def test_empty_aggregation_bitfield(state): - attestation = helpers.get_valid_attestation(state) - state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - - attestation.aggregation_bitfield = b'\x00' * len(attestation.aggregation_bitfield) - - pre_state, post_state = run_attestation_processing(state, attestation) - - return pre_state, attestation, post_state diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_attester_slashing.py deleted file mode 100644 index 66617fd6dc..0000000000 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_attester_slashing.py +++ /dev/null @@ -1,107 +0,0 @@ -from copy import deepcopy -import pytest - - -# mark entire file as 'attester_slashing' -pytestmark = pytest.mark.attester_slashings - - -def run_attester_slashing_processing(state, attester_slashing, valid=True): - """ - Run ``spec.process_attester_slashing`` returning the pre and post state. - If ``valid == False``, run expecting ``AssertionError`` - """ - post_state = deepcopy(state) - - if not valid: - with pytest.raises(AssertionError): - spec.process_attester_slashing(post_state, attester_slashing) - return state, None - - spec.process_attester_slashing(post_state, attester_slashing) - - slashed_index = attester_slashing.attestation_1.custody_bit_0_indices[0] - slashed_validator = post_state.validator_registry[slashed_index] - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - # lost whistleblower reward - assert ( - helpers.get_balance(post_state, slashed_index) < - helpers.get_balance(state, slashed_index) - ) - proposer_index = spec.get_beacon_proposer_index(state) - # gained whistleblower reward - assert ( - helpers.get_balance(post_state, proposer_index) > - helpers.get_balance(state, proposer_index) - ) - - return state, post_state - - -def test_success_double(state): - attester_slashing = helpers.get_valid_attester_slashing(state) - - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing) - - return pre_state, attester_slashing, post_state - - -def test_success_surround(state): - helpers.next_epoch(state) - state.current_justified_epoch += 1 - attester_slashing = helpers.get_valid_attester_slashing(state) - - # set attestion1 to surround attestation 2 - attester_slashing.attestation_1.data.source_epoch = attester_slashing.attestation_2.data.source_epoch - 1 - attester_slashing.attestation_1.data.target_epoch = attester_slashing.attestation_2.data.target_epoch + 1 - - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing) - - return pre_state, attester_slashing, post_state - - -def test_same_data(state): - attester_slashing = helpers.get_valid_attester_slashing(state) - - attester_slashing.attestation_1.data = attester_slashing.attestation_2.data - - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) - - return pre_state, attester_slashing, post_state - - -def test_no_double_or_surround(state): - attester_slashing = helpers.get_valid_attester_slashing(state) - - attester_slashing.attestation_1.data.target_epoch += 1 - - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) - - return pre_state, attester_slashing, post_state - - -def test_participants_already_slashed(state): - attester_slashing = helpers.get_valid_attester_slashing(state) - - # set all indices to slashed - attestation_1 = attester_slashing.attestation_1 - validator_indices = attestation_1.custody_bit_0_indices + attestation_1.custody_bit_1_indices - for index in validator_indices: - state.validator_registry[index].slashed = True - - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) - - return pre_state, attester_slashing, post_state - - -def test_custody_bit_0_and_1(state): - attester_slashing = helpers.get_valid_attester_slashing(state) - - attester_slashing.attestation_1.custody_bit_1_indices = ( - attester_slashing.attestation_1.custody_bit_0_indices - ) - pre_state, post_state = run_attester_slashing_processing(state, attester_slashing, False) - - return pre_state, attester_slashing, post_state diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_block_header.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_block_header.py deleted file mode 100644 index f7032a2839..0000000000 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_block_header.py +++ /dev/null @@ -1,65 +0,0 @@ -from copy import deepcopy -import pytest - - -# mark entire file as 'header' -pytestmark = pytest.mark.header - - -def prepare_state_for_header_processing(state): - spec.process_slot(state) - helpers.advance_slot(state) - - -def run_block_header_processing(state, block, valid=True): - """ - Run ``spec.process_block_header`` returning the pre and post state. - If ``valid == False``, run expecting ``AssertionError`` - """ - prepare_state_for_header_processing(state) - post_state = deepcopy(state) - - if not valid: - with pytest.raises(AssertionError): - spec.process_block_header(post_state, block) - return state, None - - spec.process_block_header(post_state, block) - return state, post_state - - -def test_success(state): - block = helpers.build_empty_block_for_next_slot(state) - pre_state, post_state = run_block_header_processing(state, block) - return state, block, post_state - - -def test_invalid_slot(state): - block = helpers.build_empty_block_for_next_slot(state) - block.slot = state.slot + 2 # invalid slot - - pre_state, post_state = run_block_header_processing(state, block, valid=False) - return pre_state, block, None - - -def test_invalid_parent_block_root(state): - block = helpers.build_empty_block_for_next_slot(state) - block.parent_root = b'\12' * 32 # invalid prev root - - pre_state, post_state = run_block_header_processing(state, block, valid=False) - return pre_state, block, None - - -def test_proposer_slashed(state): - # use stub state to get proposer index of next slot - stub_state = deepcopy(state) - helpers.next_slot(stub_state) - proposer_index = spec.get_beacon_proposer_index(stub_state) - - # set proposer to slashed - state.validator_registry[proposer_index].slashed = True - - block = helpers.build_empty_block_for_next_slot(state) - - pre_state, post_state = run_block_header_processing(state, block, valid=False) - return pre_state, block, None diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_deposit.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_deposit.py deleted file mode 100644 index f3abab3544..0000000000 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_deposit.py +++ /dev/null @@ -1,128 +0,0 @@ -from copy import deepcopy -import pytest - - -# mark entire file as 'deposits' -pytestmark = pytest.mark.deposits - - -def test_success(state): - pre_state = deepcopy(state) - # fill previous deposits with zero-hash - deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) - - index = len(deposit_data_leaves) - pubkey = helpers.pubkeys[index] - privkey = helpers.privkeys[index] - deposit, root, deposit_data_leaves = helpers.build_deposit( - pre_state, - deposit_data_leaves, - pubkey, - privkey, - spec.MAX_EFFECTIVE_BALANCE, - ) - - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - - post_state = deepcopy(pre_state) - - spec.process_deposit(post_state, deposit) - - assert len(post_state.validator_registry) == len(state.validator_registry) + 1 - assert len(post_state.balances) == len(state.balances) + 1 - assert post_state.validator_registry[index].pubkey == helpers.pubkeys[index] - assert helpers.get_balance(post_state, index) == spec.MAX_EFFECTIVE_BALANCE - assert post_state.deposit_index == post_state.latest_eth1_data.deposit_count - - return pre_state, deposit, post_state - - -def test_success_top_up(state): - pre_state = deepcopy(state) - deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) - - validator_index = 0 - amount = spec.MAX_EFFECTIVE_BALANCE // 4 - pubkey = helpers.pubkeys[validator_index] - privkey = helpers.privkeys[validator_index] - deposit, root, deposit_data_leaves = helpers.build_deposit( - pre_state, - deposit_data_leaves, - pubkey, - privkey, - amount, - ) - - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - pre_balance = helpers.get_balance(pre_state, validator_index) - - post_state = deepcopy(pre_state) - - spec.process_deposit(post_state, deposit) - - assert len(post_state.validator_registry) == len(state.validator_registry) - assert len(post_state.balances) == len(state.balances) - assert post_state.deposit_index == post_state.latest_eth1_data.deposit_count - assert helpers.get_balance(post_state, validator_index) == pre_balance + amount - - return pre_state, deposit, post_state - - -def test_wrong_index(state): - pre_state = deepcopy(state) - deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) - - index = len(deposit_data_leaves) - pubkey = helpers.pubkeys[index] - privkey = helpers.privkeys[index] - deposit, root, deposit_data_leaves = helpers.build_deposit( - pre_state, - deposit_data_leaves, - pubkey, - privkey, - spec.MAX_EFFECTIVE_BALANCE, - ) - - # mess up deposit_index - deposit.index = pre_state.deposit_index + 1 - - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - - post_state = deepcopy(pre_state) - - with pytest.raises(AssertionError): - spec.process_deposit(post_state, deposit) - - return pre_state, deposit, None - - -def test_bad_merkle_proof(state): - pre_state = deepcopy(state) - deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) - - index = len(deposit_data_leaves) - pubkey = helpers.pubkeys[index] - privkey = helpers.privkeys[index] - deposit, root, deposit_data_leaves = helpers.build_deposit( - pre_state, - deposit_data_leaves, - pubkey, - privkey, - spec.MAX_EFFECTIVE_BALANCE, - ) - - # mess up merkle branch - deposit.proof[-1] = spec.ZERO_HASH - - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(deposit_data_leaves) - - post_state = deepcopy(pre_state) - - with pytest.raises(AssertionError): - spec.process_deposit(post_state, deposit) - - return pre_state, deposit, None diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_proposer_slashing.py deleted file mode 100644 index f97eb1584f..0000000000 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_proposer_slashing.py +++ /dev/null @@ -1,87 +0,0 @@ -from copy import deepcopy -import pytest - - -# mark entire file as 'proposer_slashings' -pytestmark = pytest.mark.proposer_slashings - - -def run_proposer_slashing_processing(state, proposer_slashing, valid=True): - """ - Run ``spec.process_proposer_slashing`` returning the pre and post state. - If ``valid == False``, run expecting ``AssertionError`` - """ - post_state = deepcopy(state) - - if not valid: - with pytest.raises(AssertionError): - spec.process_proposer_slashing(post_state, proposer_slashing) - return state, None - - spec.process_proposer_slashing(post_state, proposer_slashing) - - slashed_validator = post_state.validator_registry[proposer_slashing.proposer_index] - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - # lost whistleblower reward - assert ( - helpers.get_balance(post_state, proposer_slashing.proposer_index) < - helpers.get_balance(state, proposer_slashing.proposer_index) - ) - - return state, post_state - - -def test_success(state): - proposer_slashing = helpers.get_valid_proposer_slashing(state) - - pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing) - - return pre_state, proposer_slashing, post_state - - -def test_epochs_are_different(state): - proposer_slashing = helpers.get_valid_proposer_slashing(state) - - # set slots to be in different epochs - proposer_slashing.header_2.slot += spec.SLOTS_PER_EPOCH - - pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) - - return pre_state, proposer_slashing, post_state - - -def test_headers_are_same(state): - proposer_slashing = helpers.get_valid_proposer_slashing(state) - - # set headers to be the same - proposer_slashing.header_2 = proposer_slashing.header_1 - - pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) - - return pre_state, proposer_slashing, post_state - - -def test_proposer_is_slashed(state): - proposer_slashing = helpers.get_valid_proposer_slashing(state) - - # set proposer to slashed - state.validator_registry[proposer_slashing.proposer_index].slashed = True - - pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) - - return pre_state, proposer_slashing, post_state - - -def test_proposer_is_withdrawn(state): - proposer_slashing = helpers.get_valid_proposer_slashing(state) - - # set proposer withdrawable_epoch in past - current_epoch = spec.get_current_epoch(state) - proposer_index = proposer_slashing.proposer_index - state.validator_registry[proposer_index].withdrawable_epoch = current_epoch - 1 - - pre_state, post_state = run_proposer_slashing_processing(state, proposer_slashing, False) - - return pre_state, proposer_slashing, post_state diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py b/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py deleted file mode 100644 index 6c57c54f00..0000000000 --- a/test_libs/pyspec/tests/phase0/block_processing/test_process_transfer.py +++ /dev/null @@ -1,132 +0,0 @@ -from copy import deepcopy -import pytest - - -# mark entire file as 'transfers' -pytestmark = pytest.mark.transfers - - -def run_transfer_processing(state, transfer, valid=True): - """ - Run ``spec.process_transfer`` returning the pre and post state. - If ``valid == False``, run expecting ``AssertionError`` - """ - post_state = deepcopy(state) - - if not valid: - with pytest.raises(AssertionError): - spec.process_transfer(post_state, transfer) - return state, None - - spec.process_transfer(post_state, transfer) - - proposer_index = spec.get_beacon_proposer_index(state) - pre_transfer_sender_balance = state.balances[transfer.sender] - pre_transfer_recipient_balance = state.balances[transfer.recipient] - pre_transfer_proposer_balance = state.balances[proposer_index] - sender_balance = post_state.balances[transfer.sender] - recipient_balance = post_state.balances[transfer.recipient] - assert sender_balance == pre_transfer_sender_balance - transfer.amount - transfer.fee - assert recipient_balance == pre_transfer_recipient_balance + transfer.amount - assert post_state.balances[proposer_index] == pre_transfer_proposer_balance + transfer.fee - - return state, post_state - - -def test_success_non_activated(state): - transfer = helpers.get_valid_transfer(state) - # un-activate so validator can transfer - state.validator_registry[transfer.sender].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - pre_state, post_state = run_transfer_processing(state, transfer) - - return pre_state, transfer, post_state - - -def test_success_withdrawable(state): - helpers.next_epoch(state) - - transfer = helpers.get_valid_transfer(state) - - # withdrawable_epoch in past so can transfer - state.validator_registry[transfer.sender].withdrawable_epoch = spec.get_current_epoch(state) - 1 - - pre_state, post_state = run_transfer_processing(state, transfer) - - return pre_state, transfer, post_state - - -def test_success_active_above_max_effective(state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - amount = spec.MAX_EFFECTIVE_BALANCE // 32 - state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE + amount - transfer = helpers.get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) - - pre_state, post_state = run_transfer_processing(state, transfer) - - return pre_state, transfer, post_state - - -def test_active_but_transfer_past_effective_balance(state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - amount = spec.MAX_EFFECTIVE_BALANCE // 32 - state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE - transfer = helpers.get_valid_transfer(state, sender_index=sender_index, amount=amount, fee=0) - - pre_state, post_state = run_transfer_processing(state, transfer, False) - - return pre_state, transfer, post_state - - -def test_incorrect_slot(state): - transfer = helpers.get_valid_transfer(state, slot=state.slot + 1) - # un-activate so validator can transfer - state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - - pre_state, post_state = run_transfer_processing(state, transfer, False) - - return pre_state, transfer, post_state - - -def test_insufficient_balance(state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - amount = spec.MAX_EFFECTIVE_BALANCE - state.balances[sender_index] = spec.MAX_EFFECTIVE_BALANCE - transfer = helpers.get_valid_transfer(state, sender_index=sender_index, amount=amount + 1, fee=0) - - # un-activate so validator can transfer - state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - - pre_state, post_state = run_transfer_processing(state, transfer, False) - - return pre_state, transfer, post_state - - -def test_no_dust(state): - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - balance = state.balances[sender_index] - transfer = helpers.get_valid_transfer( - state, - sender_index=sender_index, - amount=balance - spec.MIN_DEPOSIT_AMOUNT + 1, - fee=0, - ) - - # un-activate so validator can transfer - state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - - pre_state, post_state = run_transfer_processing(state, transfer, False) - - return pre_state, transfer, post_state - - -def test_invalid_pubkey(state): - transfer = helpers.get_valid_transfer(state) - state.validator_registry[transfer.sender].withdrawal_credentials = spec.ZERO_HASH - - # un-activate so validator can transfer - state.validator_registry[transfer.sender].activation_epoch = spec.FAR_FUTURE_EPOCH - - pre_state, post_state = run_transfer_processing(state, transfer, False) - - return pre_state, transfer, post_state diff --git a/test_libs/pyspec/tests/phase0/block_processing/test_voluntary_exit.py b/test_libs/pyspec/tests/phase0/block_processing/test_voluntary_exit.py deleted file mode 100644 index a3e54f00e2..0000000000 --- a/test_libs/pyspec/tests/phase0/block_processing/test_voluntary_exit.py +++ /dev/null @@ -1,150 +0,0 @@ -from copy import deepcopy -import pytest - - -# mark entire file as 'voluntary_exits' -pytestmark = pytest.mark.voluntary_exits - - -def run_voluntary_exit_processing(state, voluntary_exit, valid=True): - """ - Run ``spec.process_voluntary_exit`` returning the pre and post state. - If ``valid == False``, run expecting ``AssertionError`` - """ - post_state = deepcopy(state) - - if not valid: - with pytest.raises(AssertionError): - spec.process_voluntary_exit(post_state, voluntary_exit) - return state, None - - spec.process_voluntary_exit(post_state, voluntary_exit) - - validator_index = voluntary_exit.validator_index - assert state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH - assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH - - return state, post_state - - -def test_success(state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - - current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch)[0] - privkey = helpers.pubkey_to_privkey[state.validator_registry[validator_index].pubkey] - - voluntary_exit = helpers.build_voluntary_exit( - state, - current_epoch, - validator_index, - privkey, - ) - - pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit) - return pre_state, voluntary_exit, post_state - - -def test_success_exit_queue(state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - - current_epoch = spec.get_current_epoch(state) - - # exit `MAX_EXITS_PER_EPOCH` - initial_indices = spec.get_active_validator_indices(state, current_epoch)[:spec.get_churn_limit(state)] - post_state = state - for index in initial_indices: - privkey = helpers.pubkey_to_privkey[state.validator_registry[index].pubkey] - voluntary_exit = helpers.build_voluntary_exit( - state, - current_epoch, - index, - privkey, - ) - - pre_state, post_state = run_voluntary_exit_processing(post_state, voluntary_exit) - - # exit an additional validator - validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] - privkey = helpers.pubkey_to_privkey[state.validator_registry[validator_index].pubkey] - voluntary_exit = helpers.build_voluntary_exit( - state, - current_epoch, - validator_index, - privkey, - ) - - pre_state, post_state = run_voluntary_exit_processing(post_state, voluntary_exit) - - assert ( - post_state.validator_registry[validator_index].exit_epoch == - post_state.validator_registry[initial_indices[0]].exit_epoch + 1 - ) - - return pre_state, voluntary_exit, post_state - - -def test_validator_not_active(state): - current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch)[0] - privkey = helpers.pubkey_to_privkey[state.validator_registry[validator_index].pubkey] - - state.validator_registry[validator_index].activation_epoch = spec.FAR_FUTURE_EPOCH - - # - # build and test voluntary exit - # - voluntary_exit = helpers.build_voluntary_exit( - state, - current_epoch, - validator_index, - privkey, - ) - - pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit, False) - return pre_state, voluntary_exit, post_state - - -def test_validator_already_exited(state): - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow validator able to exit - state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - - current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch)[0] - privkey = helpers.pubkey_to_privkey[state.validator_registry[validator_index].pubkey] - - # but validator already has exited - state.validator_registry[validator_index].exit_epoch = current_epoch + 2 - - voluntary_exit = helpers.build_voluntary_exit( - state, - current_epoch, - validator_index, - privkey, - ) - - pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit, False) - return pre_state, voluntary_exit, post_state - - -def test_validator_not_active_long_enough(state): - current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch)[0] - privkey = helpers.pubkey_to_privkey[state.validator_registry[validator_index].pubkey] - - voluntary_exit = helpers.build_voluntary_exit( - state, - current_epoch, - validator_index, - privkey, - ) - - assert ( - current_epoch - state.validator_registry[validator_index].activation_epoch < - spec.PERSISTENT_COMMITTEE_PERIOD - ) - - pre_state, post_state = run_voluntary_exit_processing(state, voluntary_exit, False) - return pre_state, voluntary_exit, post_state diff --git a/test_libs/pyspec/tests/phase0/conftest.py b/test_libs/pyspec/tests/phase0/conftest.py deleted file mode 100644 index 67dc34fa43..0000000000 --- a/test_libs/pyspec/tests/phase0/conftest.py +++ /dev/null @@ -1,37 +0,0 @@ -import pytest - -from eth2spec.phase0 import spec -from preset_loader import loader - -from tests.phase0 import helpers - - -def pytest_addoption(parser): - parser.addoption( - "--config", action="store", default="minimal", help="config: make the pyspec use the specified configuration" - ) - - -@pytest.fixture(autouse=True) -def config(request): - config_name = request.config.getoption("--config") - presets = loader.load_presets('../../configs/', config_name) - spec.apply_constants_preset(presets) - helpers.spec = spec - request.function.__globals__['spec'] = spec - request.function.__globals__['helpers'] = helpers - - -@pytest.fixture -def num_validators(config): - return spec.SLOTS_PER_EPOCH * 8 - - -@pytest.fixture -def deposit_data_leaves(): - return list() - - -@pytest.fixture -def state(num_validators, deposit_data_leaves): - return helpers.create_genesis_state(num_validators, deposit_data_leaves) diff --git a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py deleted file mode 100644 index 644bef415a..0000000000 --- a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py +++ /dev/null @@ -1,201 +0,0 @@ -from copy import deepcopy - -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py -======= -import eth2spec.phase0.spec as spec - -from eth2spec.phase0.spec import ( - process_slot, - get_crosslink_deltas, - process_crosslinks, - state_transition, -) -from eth2spec.test.context import spec_state_test -from eth2spec.test.helpers.state import ( - next_epoch, - next_slot -) -from eth2spec.test.helpers.block import apply_empty_block, sign_block -from eth2spec.test.helpers.attestations import ( - add_attestation_to_state, - build_empty_block_for_next_slot, - fill_aggregate_attestation, - get_crosslink_committee, - get_valid_attestation, - sign_attestation, -) - ->>>>>>> dev:test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py - -def run_process_crosslinks(state, valid=True): - """ - Run ``process_crosslinks``, yielding: - - pre-state ('pre') - - post-state ('post'). - If ``valid == False``, run expecting ``AssertionError`` - """ - # transition state to slot before state transition - slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = helpers.build_empty_block_for_next_slot(state) - block.slot = slot -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py - spec.state_transition(state, block) -======= - sign_block(state, block) - state_transition(state, block) ->>>>>>> dev:test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py - - # cache state before epoch transition - spec.process_slot(state) - -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py - post_state = deepcopy(state) - spec.process_crosslinks(post_state) - - return state, post_state -======= - yield 'pre', state - process_crosslinks(state) - yield 'post', state ->>>>>>> dev:test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py - - -@spec_state_test -def test_no_attestations(state): - yield from run_process_crosslinks(state) - - for shard in range(spec.SHARD_COUNT): - assert state.previous_crosslinks[shard] == state.current_crosslinks[shard] - - -@spec_state_test -def test_single_crosslink_update_from_current_epoch(state): - helpers.next_epoch(state) - -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py - attestation = helpers.get_valid_attestation(state) -======= - attestation = get_valid_attestation(state, signed=True) ->>>>>>> dev:test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py - - helpers.fill_aggregate_attestation(state, attestation) - helpers.add_attestation_to_state(state, attestation, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) - - assert len(state.current_epoch_attestations) == 1 - - shard = attestation.data.crosslink.shard - pre_crosslink = deepcopy(state.current_crosslinks[shard]) - - yield from run_process_crosslinks(state) - - assert state.previous_crosslinks[shard] != state.current_crosslinks[shard] - assert pre_crosslink != state.current_crosslinks[shard] - - -@spec_state_test -def test_single_crosslink_update_from_previous_epoch(state): - helpers.next_epoch(state) - -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py - attestation = helpers.get_valid_attestation(state) -======= - attestation = get_valid_attestation(state, signed=True) ->>>>>>> dev:test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py - - helpers.fill_aggregate_attestation(state, attestation) - helpers.add_attestation_to_state(state, attestation, state.slot + spec.SLOTS_PER_EPOCH) - - assert len(state.previous_epoch_attestations) == 1 - -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py - pre_state, post_state = run_process_crosslinks(state) - crosslink_deltas = spec.get_crosslink_deltas(state) -======= - shard = attestation.data.crosslink.shard - pre_crosslink = deepcopy(state.current_crosslinks[shard]) - - crosslink_deltas = get_crosslink_deltas(state) ->>>>>>> dev:test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py - - yield from run_process_crosslinks(state) - - assert state.previous_crosslinks[shard] != state.current_crosslinks[shard] - assert pre_crosslink != state.current_crosslinks[shard] - - # ensure rewarded - for index in spec.get_crosslink_committee(state, attestation.data.target_epoch, attestation.data.crosslink.shard): - assert crosslink_deltas[0][index] > 0 - assert crosslink_deltas[1][index] == 0 - - -@spec_state_test -def test_double_late_crosslink(state): -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py - helpers.next_epoch(state) - state.slot += 4 - - attestation_1 = helpers.get_valid_attestation(state) - helpers.fill_aggregate_attestation(state, attestation_1) - - # add attestation_1 in the next epoch - helpers.next_epoch(state) - helpers.add_attestation_to_state(state, attestation_1, state.slot + 1) -======= - if spec.get_epoch_committee_count(state, spec.get_current_epoch(state)) < spec.SHARD_COUNT: - print("warning: ignoring test, test-assumptions are incompatible with configuration") - return - - next_epoch(state) - state.slot += 4 - - attestation_1 = get_valid_attestation(state, signed=True) - fill_aggregate_attestation(state, attestation_1) - - # add attestation_1 to next epoch - next_epoch(state) - add_attestation_to_state(state, attestation_1, state.slot + 1) ->>>>>>> dev:test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py - - for slot in range(spec.SLOTS_PER_EPOCH): - attestation_2 = helpers.get_valid_attestation(state) - if attestation_2.data.crosslink.shard == attestation_1.data.crosslink.shard: - sign_attestation(state, attestation_2) - break -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py - helpers.next_slot(state) - helpers.fill_aggregate_attestation(state, attestation_2) -======= - next_slot(state) - apply_empty_block(state) - - fill_aggregate_attestation(state, attestation_2) ->>>>>>> dev:test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py - - # add attestation_2 in the next epoch after attestation_1 has - # already updated the relevant crosslink - helpers.next_epoch(state) - helpers.add_attestation_to_state(state, attestation_2, state.slot + 1) - - assert len(state.previous_epoch_attestations) == 1 - assert len(state.current_epoch_attestations) == 0 - -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/epoch_processing/test_process_crosslinks.py - pre_state, post_state = run_process_crosslinks(state) - crosslink_deltas = spec.get_crosslink_deltas(state) -======= - crosslink_deltas = get_crosslink_deltas(state) ->>>>>>> dev:test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py - - yield from run_process_crosslinks(state) - - shard = attestation_2.data.crosslink.shard - - # ensure that the current crosslinks were not updated by the second attestation - assert state.previous_crosslinks[shard] == state.current_crosslinks[shard] - # ensure no reward, only penalties for the failed crosslink - for index in spec.get_crosslink_committee( - state, - attestation_2.data.target_epoch, - attestation_2.data.crosslink.shard): - assert crosslink_deltas[0][index] == 0 - assert crosslink_deltas[1][index] > 0 diff --git a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py deleted file mode 100644 index 4841fa6161..0000000000 --- a/test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py +++ /dev/null @@ -1,104 +0,0 @@ -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py -from copy import deepcopy -import pytest - - -# mark entire file as 'state' -pytestmark = pytest.mark.state - - -======= -import eth2spec.phase0.spec as spec - -from eth2spec.phase0.spec import ( - get_current_epoch, - is_active_validator, - process_registry_updates -) -from eth2spec.phase0.spec import state_transition -from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block -from eth2spec.test.helpers.state import next_epoch -from eth2spec.test.context import spec_state_test - - -def run_process_registry_updates(state, valid=True): - """ - Run ``process_crosslinks``, yielding: - - pre-state ('pre') - - post-state ('post'). - If ``valid == False``, run expecting ``AssertionError`` - """ - # transition state to slot before state transition - slot = state.slot + (spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH) - 1 - block = build_empty_block_for_next_slot(state) - block.slot = slot - sign_block(state, block) - state_transition(state, block) - - # cache state before epoch transition - spec.process_slot(state) - - # process components of epoch transition before registry update - spec.process_justification_and_finalization(state) - spec.process_crosslinks(state) - spec.process_rewards_and_penalties(state) - - yield 'pre', state - process_registry_updates(state) - yield 'post', state - - -@spec_state_test ->>>>>>> dev:test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py -def test_activation(state): - index = 0 - assert spec.is_active_validator(state.validator_registry[index], spec.get_current_epoch(state)) - - # Mock a new deposit - state.validator_registry[index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - state.validator_registry[index].activation_epoch = spec.FAR_FUTURE_EPOCH - state.validator_registry[index].effective_balance = spec.MAX_EFFECTIVE_BALANCE - assert not spec.is_active_validator(state.validator_registry[index], spec.get_current_epoch(state)) - - for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py - block = helpers.next_epoch(state) - blocks.append(block) -======= - next_epoch(state) - - yield from run_process_registry_updates(state) ->>>>>>> dev:test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py - - assert state.validator_registry[index].activation_eligibility_epoch != spec.FAR_FUTURE_EPOCH - assert state.validator_registry[index].activation_epoch != spec.FAR_FUTURE_EPOCH - assert spec.is_active_validator( - state.validator_registry[index], - spec.get_current_epoch(state), - ) - - -@spec_state_test -def test_ejection(state): - index = 0 - assert spec.is_active_validator(state.validator_registry[index], spec.get_current_epoch(state)) - assert state.validator_registry[index].exit_epoch == spec.FAR_FUTURE_EPOCH - - # Mock an ejection - state.validator_registry[index].effective_balance = spec.EJECTION_BALANCE - - for _ in range(spec.ACTIVATION_EXIT_DELAY + 1): -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/epoch_processing/test_process_registry_updates.py - block = helpers.next_epoch(state) - blocks.append(block) -======= - next_epoch(state) - - yield from run_process_registry_updates(state) ->>>>>>> dev:test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py - - assert state.validator_registry[index].exit_epoch != spec.FAR_FUTURE_EPOCH - assert not spec.is_active_validator( - state.validator_registry[index], - spec.get_current_epoch(state), - ) diff --git a/test_libs/pyspec/tests/phase0/helpers.py b/test_libs/pyspec/tests/phase0/helpers.py deleted file mode 100644 index 140f74a53a..0000000000 --- a/test_libs/pyspec/tests/phase0/helpers.py +++ /dev/null @@ -1,400 +0,0 @@ -from copy import deepcopy - -from py_ecc import bls - -from eth2spec.utils.minimal_ssz import signing_root -from eth2spec.utils.merkle_minimal import ( - calc_merkle_tree_from_leaves, - get_merkle_proof, - get_merkle_root, -) - -privkeys = [i + 1 for i in range(1024)] -pubkeys = [bls.privtopub(privkey) for privkey in privkeys] -pubkey_to_privkey = {pubkey: privkey for privkey, pubkey in zip(privkeys, pubkeys)} - - -def advance_slot(state) -> None: - state.slot += 1 - - -def get_balance(state, index): - return state.balances[index] - - -def set_bitfield_bit(bitfield, i): - """ - Set the bit in ``bitfield`` at position ``i`` to ``1``. - """ - byte_index = i // 8 - bit_index = i % 8 - return ( - bitfield[:byte_index] + - bytes([bitfield[byte_index] | (1 << bit_index)]) + - bitfield[byte_index + 1:] - ) - - -def create_mock_genesis_validator_deposits(num_validators, deposit_data_leaves=None): - if not deposit_data_leaves: - deposit_data_leaves = [] - signature = b'\x33' * 96 - - deposit_data_list = [] - for i in range(num_validators): - pubkey = pubkeys[i] - deposit_data = spec.DepositData( - pubkey=pubkey, - # insecurely use pubkey as withdrawal key as well - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(pubkey)[1:], - amount=spec.MAX_EFFECTIVE_BALANCE, - signature=signature, - ) - item = deposit_data.hash_tree_root() - deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - root = get_merkle_root((tuple(deposit_data_leaves))) - proof = list(get_merkle_proof(tree, item_index=i)) - assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, i, root) - deposit_data_list.append(deposit_data) - - genesis_validator_deposits = [] - for i in range(num_validators): - genesis_validator_deposits.append(spec.Deposit( - proof=list(get_merkle_proof(tree, item_index=i)), - index=i, - data=deposit_data_list[i] - )) - return genesis_validator_deposits, root - - -def create_genesis_state(num_validators, deposit_data_leaves=None): - initial_deposits, deposit_root = create_mock_genesis_validator_deposits( - num_validators, - deposit_data_leaves, - ) - return spec.get_genesis_beacon_state( - initial_deposits, - genesis_time=0, - genesis_eth1_data=spec.Eth1Data( - deposit_root=deposit_root, - deposit_count=len(initial_deposits), - block_hash=spec.ZERO_HASH, - ), - ) - - -def build_empty_block_for_next_slot(state): - empty_block = spec.BeaconBlock() - empty_block.slot = state.slot + 1 - empty_block.body.eth1_data.deposit_count = state.deposit_index - previous_block_header = deepcopy(state.latest_block_header) - if previous_block_header.state_root == spec.ZERO_HASH: - previous_block_header.state_root = state.hash_tree_root() - empty_block.parent_root = signing_root(previous_block_header) - return empty_block - - -def build_deposit_data(state, pubkey, privkey, amount): - deposit_data = spec.DepositData( - pubkey=pubkey, - # insecurely use pubkey as withdrawal key as well - withdrawal_credentials=spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(pubkey)[1:], - amount=amount, - ) - signature = bls.sign( - message_hash=signing_root(deposit_data), - privkey=privkey, - domain=spec.bls_domain(spec.DOMAIN_DEPOSIT), - ) - deposit_data.signature = signature - return deposit_data - - -def build_attestation_data(state, slot, shard): - assert state.slot >= slot - - if slot == state.slot: - block_root = build_empty_block_for_next_slot(state).parent_root - else: - block_root = spec.get_block_root_at_slot(state, slot) - - current_epoch_start_slot = spec.get_epoch_start_slot(spec.get_current_epoch(state)) - if slot < current_epoch_start_slot: - epoch_boundary_root = spec.get_block_root(state, spec.get_previous_epoch(state)) - elif slot == current_epoch_start_slot: - epoch_boundary_root = block_root - else: - epoch_boundary_root = spec.get_block_root(state, spec.get_current_epoch(state)) - - if slot < current_epoch_start_slot: - justified_epoch = state.previous_justified_epoch - justified_block_root = state.previous_justified_root - else: - justified_epoch = state.current_justified_epoch - justified_block_root = state.current_justified_root - - crosslinks = ( - state.current_crosslinks if spec.slot_to_epoch(slot) == spec.get_current_epoch(state) - else state.previous_crosslinks - ) - parent_crosslink = crosslinks[shard] - return spec.AttestationData( - beacon_block_root=block_root, - source_epoch=justified_epoch, - source_root=justified_block_root, - target_epoch=spec.slot_to_epoch(slot), - target_root=epoch_boundary_root, - crosslink=spec.Crosslink( - shard=shard, - start_epoch=parent_crosslink.end_epoch, - end_epoch=min(spec.slot_to_epoch(slot), parent_crosslink.end_epoch + spec.MAX_EPOCHS_PER_CROSSLINK), - data_root=spec.ZERO_HASH, - parent_root=spec.hash_tree_root(parent_crosslink), - ), - ) - - -def build_voluntary_exit(state, epoch, validator_index, privkey): - voluntary_exit = spec.VoluntaryExit( - epoch=epoch, - validator_index=validator_index, - ) - voluntary_exit.signature = bls.sign( - message_hash=signing_root(voluntary_exit), - privkey=privkey, - domain=spec.get_domain( - state=state, - domain_type=spec.DOMAIN_VOLUNTARY_EXIT, - message_epoch=epoch, - ) - ) - - return voluntary_exit - - -def build_deposit(state, - deposit_data_leaves, - pubkey, - privkey, - amount): - deposit_data = build_deposit_data(state, pubkey, privkey, amount) - - item = deposit_data.hash_tree_root() - index = len(deposit_data_leaves) - deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(deposit_data_leaves)) - root = get_merkle_root((tuple(deposit_data_leaves))) - proof = list(get_merkle_proof(tree, item_index=index)) - assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) - - deposit = spec.Deposit( - proof=list(proof), - index=index, - data=deposit_data, - ) - - return deposit, root, deposit_data_leaves - - -def get_valid_proposer_slashing(state): - current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] - privkey = pubkey_to_privkey[state.validator_registry[validator_index].pubkey] - slot = state.slot - - header_1 = spec.BeaconBlockHeader( - slot=slot, - parent_root=spec.ZERO_HASH, - state_root=spec.ZERO_HASH, - body_root=spec.ZERO_HASH, - ) - header_2 = deepcopy(header_1) - header_2.parent_root = b'\x02' * 32 - header_2.slot = slot + 1 - - domain = spec.get_domain( - state=state, - domain_type=spec.DOMAIN_BEACON_PROPOSER, - ) - header_1.signature = bls.sign( - message_hash=signing_root(header_1), - privkey=privkey, - domain=domain, - ) - header_2.signature = bls.sign( - message_hash=signing_root(header_2), - privkey=privkey, - domain=domain, - ) - - return spec.ProposerSlashing( - proposer_index=validator_index, - header_1=header_1, - header_2=header_2, - ) - - -def get_valid_attester_slashing(state): - attestation_1 = get_valid_attestation(state) - attestation_2 = deepcopy(attestation_1) - attestation_2.data.target_root = b'\x01' * 32 - - return spec.AttesterSlashing( - attestation_1=spec.convert_to_indexed(state, attestation_1), - attestation_2=spec.convert_to_indexed(state, attestation_2), - ) - - -def get_valid_attestation(state, slot=None): - if slot is None: - slot = state.slot - - if spec.slot_to_epoch(slot) == spec.get_current_epoch(state): - shard = (state.latest_start_shard + slot) % spec.SLOTS_PER_EPOCH - else: - previous_shard_delta = spec.get_shard_delta(state, spec.get_previous_epoch(state)) - shard = (state.latest_start_shard - previous_shard_delta + slot) % spec.SHARD_COUNT - - attestation_data = build_attestation_data(state, slot, shard) - - crosslink_committee = spec.get_crosslink_committee( - state, - attestation_data.target_epoch, - attestation_data.crosslink.shard, - ) - - committee_size = len(crosslink_committee) - bitfield_length = (committee_size + 7) // 8 - aggregation_bitfield = b'\xC0' + b'\x00' * (bitfield_length - 1) - custody_bitfield = b'\x00' * bitfield_length - attestation = spec.Attestation( - aggregation_bitfield=aggregation_bitfield, - data=attestation_data, - custody_bitfield=custody_bitfield, - ) - participants = spec.get_attesting_indices( - state, - attestation.data, - attestation.aggregation_bitfield, - ) - assert len(participants) == 2 - - signatures = [] - for validator_index in participants: - privkey = privkeys[validator_index] - signatures.append( - get_attestation_signature( - state, - attestation.data, - privkey - ) - ) - - attestation.aggregation_signature = bls.aggregate_signatures(signatures) - return attestation - - -def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=None): - if slot is None: - slot = state.slot - current_epoch = spec.get_current_epoch(state) - if sender_index is None: - sender_index = spec.get_active_validator_indices(state, current_epoch)[-1] - recipient_index = spec.get_active_validator_indices(state, current_epoch)[0] - transfer_pubkey = pubkeys[-1] - transfer_privkey = privkeys[-1] - - if fee is None: - fee = get_balance(state, sender_index) // 32 - if amount is None: - amount = get_balance(state, sender_index) - fee - - transfer = spec.Transfer( - sender=sender_index, - recipient=recipient_index, - amount=amount, - fee=fee, - slot=slot, - pubkey=transfer_pubkey, - signature=spec.ZERO_HASH, - ) - transfer.signature = bls.sign( - message_hash=signing_root(transfer), - privkey=transfer_privkey, - domain=spec.get_domain( - state=state, - domain_type=spec.DOMAIN_TRANSFER, - message_epoch=spec.get_current_epoch(state), - ) - ) - - # ensure withdrawal_credentials reproducable - state.validator_registry[transfer.sender].withdrawal_credentials = ( - spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] - ) - - return transfer - - -def get_attestation_signature(state, attestation_data, privkey, custody_bit=0b0): - message_hash = spec.AttestationDataAndCustodyBit( - data=attestation_data, - custody_bit=custody_bit, - ).hash_tree_root() - - return bls.sign( - message_hash=message_hash, - privkey=privkey, - domain=spec.get_domain( - state=state, - domain_type=spec.DOMAIN_ATTESTATION, - message_epoch=attestation_data.target_epoch, - ) - ) - - -def fill_aggregate_attestation(state, attestation): - crosslink_committee = spec.get_crosslink_committee( - state, - attestation.data.target_epoch, - attestation.data.crosslink.shard, - ) - for i in range(len(crosslink_committee)): - attestation.aggregation_bitfield = set_bitfield_bit(attestation.aggregation_bitfield, i) - - -def add_attestation_to_state(state, attestation, slot): - block = build_empty_block_for_next_slot(state) - block.slot = slot - block.body.attestations.append(attestation) - spec.state_transition(state, block) - - -def next_slot(state): - """ - Transition to the next slot via an empty block. - Return the empty block that triggered the transition. - """ - block = build_empty_block_for_next_slot(state) - spec.state_transition(state, block) - return block - - -def next_epoch(state): - """ - Transition to the start slot of the next epoch via an empty block. - Return the empty block that triggered the transition. - """ - block = build_empty_block_for_next_slot(state) - block.slot += spec.SLOTS_PER_EPOCH - (state.slot % spec.SLOTS_PER_EPOCH) - spec.state_transition(state, block) - return block - - -def get_state_root(state, slot) -> bytes: - """ - Return the state root at a recent ``slot``. - """ - assert slot < state.slot <= slot + spec.SLOTS_PER_HISTORICAL_ROOT - return state.latest_state_roots[slot % spec.SLOTS_PER_HISTORICAL_ROOT] diff --git a/test_libs/pyspec/tests/phase0/test_finality.py b/test_libs/pyspec/tests/phase0/test_finality.py deleted file mode 100644 index 4ce7a0be4a..0000000000 --- a/test_libs/pyspec/tests/phase0/test_finality.py +++ /dev/null @@ -1,234 +0,0 @@ -from copy import deepcopy - -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/test_finality.py -import pytest - -# mark entire file as 'state' -pytestmark = pytest.mark.state -======= -import eth2spec.phase0.spec as spec -from eth2spec.phase0.spec import ( - get_current_epoch, - get_epoch_start_slot, -) -from .context import spec_state_test, never_bls -from .helpers.state import next_epoch -from .helpers.block import build_empty_block_for_next_slot, apply_empty_block -from .helpers.attestations import get_valid_attestation ->>>>>>> dev:test_libs/pyspec/eth2spec/test/test_finality.py - - -def check_finality(state, - prev_state, - current_justified_changed, - previous_justified_changed, - finalized_changed): - if current_justified_changed: - assert state.current_justified_epoch > prev_state.current_justified_epoch - assert state.current_justified_root != prev_state.current_justified_root - else: - assert state.current_justified_epoch == prev_state.current_justified_epoch - assert state.current_justified_root == prev_state.current_justified_root - - if previous_justified_changed: - assert state.previous_justified_epoch > prev_state.previous_justified_epoch - assert state.previous_justified_root != prev_state.previous_justified_root - else: - assert state.previous_justified_epoch == prev_state.previous_justified_epoch - assert state.previous_justified_root == prev_state.previous_justified_root - - if finalized_changed: - assert state.finalized_epoch > prev_state.finalized_epoch - assert state.finalized_root != prev_state.finalized_root - else: - assert state.finalized_epoch == prev_state.finalized_epoch - assert state.finalized_root == prev_state.finalized_root - - -def next_epoch_with_attestations(state, - fill_cur_epoch, - fill_prev_epoch): - post_state = deepcopy(state) - blocks = [] - for _ in range(spec.SLOTS_PER_EPOCH): - block = helpers.build_empty_block_for_next_slot(post_state) - if fill_cur_epoch: - slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/test_finality.py - if slot_to_attest >= spec.get_epoch_start_slot(spec.get_current_epoch(post_state)): - cur_attestation = helpers.get_valid_attestation(post_state, slot_to_attest) - helpers.fill_aggregate_attestation(post_state, cur_attestation) -======= - if slot_to_attest >= get_epoch_start_slot(get_current_epoch(post_state)): - cur_attestation = get_valid_attestation(post_state, slot_to_attest) ->>>>>>> dev:test_libs/pyspec/eth2spec/test/test_finality.py - block.body.attestations.append(cur_attestation) - - if fill_prev_epoch: - slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1 -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/test_finality.py - prev_attestation = helpers.get_valid_attestation(post_state, slot_to_attest) - helpers.fill_aggregate_attestation(post_state, prev_attestation) -======= - prev_attestation = get_valid_attestation(post_state, slot_to_attest) ->>>>>>> dev:test_libs/pyspec/eth2spec/test/test_finality.py - block.body.attestations.append(prev_attestation) - - spec.state_transition(post_state, block) - blocks.append(block) - - return state, blocks, post_state - - -@never_bls -@spec_state_test -def test_finality_rule_4(state): - yield 'pre', state - - blocks = [] - for epoch in range(4): - prev_state, new_blocks, state = next_epoch_with_attestations(state, True, False) - blocks += new_blocks - - # justification/finalization skipped at GENESIS_EPOCH - if epoch == 0: - check_finality(state, prev_state, False, False, False) - # justification/finalization skipped at GENESIS_EPOCH + 1 - elif epoch == 1: - check_finality(state, prev_state, False, False, False) - elif epoch == 2: - check_finality(state, prev_state, True, False, False) - elif epoch >= 3: - # rule 4 of finality - check_finality(state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.current_justified_epoch - assert state.finalized_root == prev_state.current_justified_root - - yield 'blocks', blocks, [spec.BeaconBlock] - yield 'post', state - - -@never_bls -@spec_state_test -def test_finality_rule_1(state): - # get past first two epochs that finality does not run on -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/test_finality.py - helpers.next_epoch(state) - helpers.next_epoch(state) -======= - next_epoch(state) - apply_empty_block(state) - next_epoch(state) - apply_empty_block(state) ->>>>>>> dev:test_libs/pyspec/eth2spec/test/test_finality.py - - yield 'pre', state - - blocks = [] - for epoch in range(3): - prev_state, new_blocks, state = next_epoch_with_attestations(state, False, True) - blocks += new_blocks - - if epoch == 0: - check_finality(state, prev_state, True, False, False) - elif epoch == 1: - check_finality(state, prev_state, True, True, False) - elif epoch == 2: - # finalized by rule 1 - check_finality(state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.previous_justified_epoch - assert state.finalized_root == prev_state.previous_justified_root - - yield 'blocks', blocks, [spec.BeaconBlock] - yield 'post', state - - -@never_bls -@spec_state_test -def test_finality_rule_2(state): - # get past first two epochs that finality does not run on -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/test_finality.py - helpers.next_epoch(state) - helpers.next_epoch(state) -======= - next_epoch(state) - apply_empty_block(state) - next_epoch(state) - apply_empty_block(state) ->>>>>>> dev:test_libs/pyspec/eth2spec/test/test_finality.py - - yield 'pre', state - - blocks = [] - for epoch in range(3): - if epoch == 0: - prev_state, new_blocks, state = next_epoch_with_attestations(state, True, False) - check_finality(state, prev_state, True, False, False) - elif epoch == 1: - prev_state, new_blocks, state = next_epoch_with_attestations(state, False, False) - check_finality(state, prev_state, False, True, False) - elif epoch == 2: - prev_state, new_blocks, state = next_epoch_with_attestations(state, False, True) - # finalized by rule 2 - check_finality(state, prev_state, True, False, True) - assert state.finalized_epoch == prev_state.previous_justified_epoch - assert state.finalized_root == prev_state.previous_justified_root - - blocks += new_blocks - - yield 'blocks', blocks, [spec.BeaconBlock] - yield 'post', state - - -@never_bls -@spec_state_test -def test_finality_rule_3(state): - """ - Test scenario described here - https://github.com/ethereum/eth2.0-specs/issues/611#issuecomment-463612892 - """ - # get past first two epochs that finality does not run on -<<<<<<< HEAD:test_libs/pyspec/tests/phase0/test_finality.py - helpers.next_epoch(state) - helpers.next_epoch(state) -======= - next_epoch(state) - apply_empty_block(state) - next_epoch(state) - apply_empty_block(state) ->>>>>>> dev:test_libs/pyspec/eth2spec/test/test_finality.py - - yield 'pre', state - - blocks = [] - prev_state, new_blocks, state = next_epoch_with_attestations(state, True, False) - blocks += new_blocks - check_finality(state, prev_state, True, False, False) - - # In epoch N, JE is set to N, prev JE is set to N-1 - prev_state, new_blocks, state = next_epoch_with_attestations(state, True, False) - blocks += new_blocks - check_finality(state, prev_state, True, True, True) - - # In epoch N+1, JE is N, prev JE is N-1, and not enough messages get in to do anything - prev_state, new_blocks, state = next_epoch_with_attestations(state, False, False) - blocks += new_blocks - check_finality(state, prev_state, False, True, False) - - # In epoch N+2, JE is N, prev JE is N, and enough messages from the previous epoch get in to justify N+1. - # N+1 now becomes the JE. Not enough messages from epoch N+2 itself get in to justify N+2 - prev_state, new_blocks, state = next_epoch_with_attestations(state, False, True) - blocks += new_blocks - # rule 2 - check_finality(state, prev_state, True, False, True) - - # In epoch N+3, LJE is N+1, prev LJE is N, and enough messages get in to justify epochs N+2 and N+3. - prev_state, new_blocks, state = next_epoch_with_attestations(state, True, True) - blocks += new_blocks - # rule 3 - check_finality(state, prev_state, True, True, True) - assert state.finalized_epoch == prev_state.current_justified_epoch - assert state.finalized_root == prev_state.current_justified_root - - yield 'blocks', blocks, [spec.BeaconBlock] - yield 'post', state diff --git a/test_libs/pyspec/tests/phase0/test_sanity.py b/test_libs/pyspec/tests/phase0/test_sanity.py deleted file mode 100644 index 47e4917ab0..0000000000 --- a/test_libs/pyspec/tests/phase0/test_sanity.py +++ /dev/null @@ -1,402 +0,0 @@ -from copy import deepcopy - -import pytest - -from py_ecc import bls - -from eth2spec.utils.minimal_ssz import signing_root - -from eth2spec.utils.merkle_minimal import ( - calc_merkle_tree_from_leaves, - get_merkle_proof, - get_merkle_root, -) - - -# mark entire file as 'sanity' -pytestmark = pytest.mark.sanity - - -def test_slot_transition(state): - test_state = deepcopy(state) - spec.process_slot(test_state) - helpers.advance_slot(test_state) - assert test_state.slot == state.slot + 1 - assert helpers.get_state_root(test_state, state.slot) == state.hash_tree_root() - return test_state - - -def test_empty_block_transition(state): - test_state = deepcopy(state) - - block = helpers.build_empty_block_for_next_slot(test_state) - spec.state_transition(test_state, block) - - assert len(test_state.eth1_data_votes) == len(state.eth1_data_votes) + 1 - assert spec.get_block_root_at_slot(test_state, state.slot) == block.parent_root - - return state, [block], test_state - - -def test_skipped_slots(state): - test_state = deepcopy(state) - block = helpers.build_empty_block_for_next_slot(test_state) - block.slot += 3 - - spec.state_transition(test_state, block) - - assert test_state.slot == block.slot - for slot in range(state.slot, test_state.slot): - assert spec.get_block_root_at_slot(test_state, slot) == block.parent_root - - return state, [block], test_state - - -def test_empty_epoch_transition(state): - test_state = deepcopy(state) - block = helpers.build_empty_block_for_next_slot(test_state) - block.slot += spec.SLOTS_PER_EPOCH - - spec.state_transition(test_state, block) - - assert test_state.slot == block.slot - for slot in range(state.slot, test_state.slot): - assert spec.get_block_root_at_slot(test_state, slot) == block.parent_root - - return state, [block], test_state - - -def test_empty_epoch_transition_not_finalizing(state): - test_state = deepcopy(state) - block = helpers.build_empty_block_for_next_slot(test_state) - block.slot += spec.SLOTS_PER_EPOCH * 5 - - spec.state_transition(test_state, block) - - assert test_state.slot == block.slot - assert test_state.finalized_epoch < spec.get_current_epoch(test_state) - 4 - for index in range(len(test_state.validator_registry)): - assert helpers.get_balance(test_state, index) < helpers.get_balance(state, index) - - return state, [block], test_state - - -def test_proposer_slashing(state): - test_state = deepcopy(state) - proposer_slashing = helpers.get_valid_proposer_slashing(state) - validator_index = proposer_slashing.proposer_index - - # - # Add to state via block transition - # - block = helpers.build_empty_block_for_next_slot(test_state) - block.body.proposer_slashings.append(proposer_slashing) - spec.state_transition(test_state, block) - - assert not state.validator_registry[validator_index].slashed - - slashed_validator = test_state.validator_registry[validator_index] - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - # lost whistleblower reward - assert helpers.get_balance(test_state, validator_index) < helpers.get_balance(state, validator_index) - - return state, [block], test_state - - -def test_attester_slashing(state): - test_state = deepcopy(state) - attester_slashing = helpers.get_valid_attester_slashing(state) - validator_index = attester_slashing.attestation_1.custody_bit_0_indices[0] - - # - # Add to state via block transition - # - block = helpers.build_empty_block_for_next_slot(test_state) - block.body.attester_slashings.append(attester_slashing) - spec.state_transition(test_state, block) - - assert not state.validator_registry[validator_index].slashed - - slashed_validator = test_state.validator_registry[validator_index] - assert slashed_validator.slashed - assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH - assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - # lost whistleblower reward - assert helpers.get_balance(test_state, validator_index) < helpers.get_balance(state, validator_index) - - proposer_index = spec.get_beacon_proposer_index(test_state) - # gained whistleblower reward - assert ( - helpers.get_balance(test_state, proposer_index) > - helpers.get_balance(state, proposer_index) - ) - - return state, [block], test_state - - -def test_deposit_in_block(state): - pre_state = deepcopy(state) - test_deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) - - index = len(test_deposit_data_leaves) - pubkey = helpers.pubkeys[index] - privkey = helpers.privkeys[index] - deposit_data = helpers.build_deposit_data(pre_state, pubkey, privkey, spec.MAX_EFFECTIVE_BALANCE) - - item = deposit_data.hash_tree_root() - test_deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(test_deposit_data_leaves)) - root = get_merkle_root((tuple(test_deposit_data_leaves))) - proof = list(get_merkle_proof(tree, item_index=index)) - assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, index, root) - - deposit = spec.Deposit( - proof=list(proof), - index=index, - data=deposit_data, - ) - - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(test_deposit_data_leaves) - post_state = deepcopy(pre_state) - block = helpers.build_empty_block_for_next_slot(post_state) - block.body.deposits.append(deposit) - - spec.state_transition(post_state, block) - assert len(post_state.validator_registry) == len(state.validator_registry) + 1 - assert len(post_state.balances) == len(state.balances) + 1 - assert helpers.get_balance(post_state, index) == spec.MAX_EFFECTIVE_BALANCE - assert post_state.validator_registry[index].pubkey == helpers.pubkeys[index] - - return pre_state, [block], post_state - - -def test_deposit_top_up(state): - pre_state = deepcopy(state) - test_deposit_data_leaves = [spec.ZERO_HASH] * len(pre_state.validator_registry) - - validator_index = 0 - amount = spec.MAX_EFFECTIVE_BALANCE // 4 - pubkey = helpers.pubkeys[validator_index] - privkey = helpers.privkeys[validator_index] - deposit_data = helpers.build_deposit_data(pre_state, pubkey, privkey, amount) - - merkle_index = len(test_deposit_data_leaves) - item = deposit_data.hash_tree_root() - test_deposit_data_leaves.append(item) - tree = calc_merkle_tree_from_leaves(tuple(test_deposit_data_leaves)) - root = get_merkle_root((tuple(test_deposit_data_leaves))) - proof = list(get_merkle_proof(tree, item_index=merkle_index)) - assert spec.verify_merkle_branch(item, proof, spec.DEPOSIT_CONTRACT_TREE_DEPTH, merkle_index, root) - - deposit = spec.Deposit( - proof=list(proof), - index=merkle_index, - data=deposit_data, - ) - - pre_state.latest_eth1_data.deposit_root = root - pre_state.latest_eth1_data.deposit_count = len(test_deposit_data_leaves) - block = helpers.build_empty_block_for_next_slot(pre_state) - block.body.deposits.append(deposit) - - pre_balance = helpers.get_balance(pre_state, validator_index) - post_state = deepcopy(pre_state) - spec.state_transition(post_state, block) - assert len(post_state.validator_registry) == len(pre_state.validator_registry) - assert len(post_state.balances) == len(pre_state.balances) - assert helpers.get_balance(post_state, validator_index) == pre_balance + amount - - return pre_state, [block], post_state - - -def test_attestation(state): - state.slot = spec.SLOTS_PER_EPOCH - test_state = deepcopy(state) - attestation = helpers.get_valid_attestation(state) - - # - # Add to state via block transition - # - attestation_block = helpers.build_empty_block_for_next_slot(test_state) - attestation_block.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY - attestation_block.body.attestations.append(attestation) - spec.state_transition(test_state, attestation_block) - - assert len(test_state.current_epoch_attestations) == len(state.current_epoch_attestations) + 1 - - # - # Epoch transition should move to previous_epoch_attestations - # - pre_current_epoch_attestations = deepcopy(test_state.current_epoch_attestations) - - epoch_block = helpers.build_empty_block_for_next_slot(test_state) - epoch_block.slot += spec.SLOTS_PER_EPOCH - spec.state_transition(test_state, epoch_block) - - assert len(test_state.current_epoch_attestations) == 0 - assert test_state.previous_epoch_attestations == pre_current_epoch_attestations - - return state, [attestation_block, epoch_block], test_state - - -def test_voluntary_exit(state): - pre_state = deepcopy(state) - validator_index = spec.get_active_validator_indices( - pre_state, - spec.get_current_epoch(pre_state) - )[-1] - - # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit - pre_state.slot += spec.PERSISTENT_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH - - post_state = deepcopy(pre_state) - - voluntary_exit = spec.VoluntaryExit( - epoch=spec.get_current_epoch(pre_state), - validator_index=validator_index, - ) - voluntary_exit.signature = bls.sign( - message_hash=signing_root(voluntary_exit), - privkey=helpers.privkeys[validator_index], - domain=spec.get_domain( - state=pre_state, - domain_type=spec.DOMAIN_VOLUNTARY_EXIT, - ) - ) - - # - # Add to state via block transition - # - initiate_exit_block = helpers.build_empty_block_for_next_slot(post_state) - initiate_exit_block.body.voluntary_exits.append(voluntary_exit) - spec.state_transition(post_state, initiate_exit_block) - - assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH - - # - # Process within epoch transition - # - exit_block = helpers.build_empty_block_for_next_slot(post_state) - exit_block.slot += spec.SLOTS_PER_EPOCH - spec.state_transition(post_state, exit_block) - - assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH - - return pre_state, [initiate_exit_block, exit_block], post_state - - -def test_transfer(state): - # overwrite default 0 to test - spec.apply_constants_preset({"MAX_TRANSFERS": 1}) - - pre_state = deepcopy(state) - current_epoch = spec.get_current_epoch(pre_state) - sender_index = spec.get_active_validator_indices(pre_state, current_epoch)[-1] - recipient_index = spec.get_active_validator_indices(pre_state, current_epoch)[0] - transfer_pubkey = helpers.pubkeys[-1] - transfer_privkey = helpers.privkeys[-1] - amount = helpers.get_balance(pre_state, sender_index) - pre_transfer_recipient_balance = helpers.get_balance(pre_state, recipient_index) - transfer = spec.Transfer( - sender=sender_index, - recipient=recipient_index, - amount=amount, - fee=0, - slot=pre_state.slot + 1, - pubkey=transfer_pubkey, - ) - transfer.signature = bls.sign( - message_hash=signing_root(transfer), - privkey=transfer_privkey, - domain=spec.get_domain( - state=pre_state, - domain_type=spec.DOMAIN_TRANSFER, - ) - ) - - # ensure withdrawal_credentials reproducable - pre_state.validator_registry[sender_index].withdrawal_credentials = ( - spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer_pubkey)[1:] - ) - # un-activate so validator can transfer - pre_state.validator_registry[sender_index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - - post_state = deepcopy(pre_state) - # - # Add to state via block transition - # - block = helpers.build_empty_block_for_next_slot(post_state) - block.body.transfers.append(transfer) - spec.state_transition(post_state, block) - - sender_balance = helpers.get_balance(post_state, sender_index) - recipient_balance = helpers.get_balance(post_state, recipient_index) - assert sender_balance == 0 - assert recipient_balance == pre_transfer_recipient_balance + amount - - return pre_state, [block], post_state - - -def test_balance_driven_status_transitions(state): - current_epoch = spec.get_current_epoch(state) - validator_index = spec.get_active_validator_indices(state, current_epoch)[-1] - - assert state.validator_registry[validator_index].exit_epoch == spec.FAR_FUTURE_EPOCH - - # set validator balance to below ejection threshold - state.validator_registry[validator_index].effective_balance = spec.EJECTION_BALANCE - - post_state = deepcopy(state) - # - # trigger epoch transition - # - block = helpers.build_empty_block_for_next_slot(post_state) - block.slot += spec.SLOTS_PER_EPOCH - spec.state_transition(post_state, block) - - assert post_state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH - - return state, [block], post_state - - -def test_historical_batch(state): - state.slot += spec.SLOTS_PER_HISTORICAL_ROOT - (state.slot % spec.SLOTS_PER_HISTORICAL_ROOT) - 1 - - post_state = deepcopy(state) - - block = helpers.build_empty_block_for_next_slot(post_state) - - spec.state_transition(post_state, block) - - assert post_state.slot == block.slot - assert spec.get_current_epoch(post_state) % (spec.SLOTS_PER_HISTORICAL_ROOT // spec.SLOTS_PER_EPOCH) == 0 - assert len(post_state.historical_roots) == len(state.historical_roots) + 1 - - return state, [block], post_state - - -def test_eth1_data_votes(state): - post_state = deepcopy(state) - - expected_votes = 0 - assert len(state.eth1_data_votes) == expected_votes - - blocks = [] - for _ in range(spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1): - block = helpers.build_empty_block_for_next_slot(post_state) - spec.state_transition(post_state, block) - expected_votes += 1 - assert len(post_state.eth1_data_votes) == expected_votes - blocks.append(block) - - block = helpers.build_empty_block_for_next_slot(post_state) - spec.state_transition(post_state, block) - blocks.append(block) - - assert post_state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 - assert len(post_state.eth1_data_votes) == 1 - - return state, blocks, post_state diff --git a/test_libs/pyspec/tests/phase1/__init__.py b/test_libs/pyspec/tests/phase1/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_phase0_block_processing.py b/test_libs/pyspec/tests/phase1/block_processing_phase1/test_phase0_block_processing.py deleted file mode 100644 index 246ba163c5..0000000000 --- a/test_libs/pyspec/tests/phase1/block_processing_phase1/test_phase0_block_processing.py +++ /dev/null @@ -1,7 +0,0 @@ -from tests.phase0.block_processing.test_process_attestation import * -from tests.phase0.block_processing.test_process_attester_slashing import * -from tests.phase0.block_processing.test_process_block_header import * -from tests.phase0.block_processing.test_process_deposit import * -from tests.phase0.block_processing.test_process_proposer_slashing import * -from tests.phase0.block_processing.test_process_transfer import * -from tests.phase0.block_processing.test_voluntary_exit import * diff --git a/test_libs/pyspec/tests/phase1/conftest.py b/test_libs/pyspec/tests/phase1/conftest.py deleted file mode 100644 index 4226241d77..0000000000 --- a/test_libs/pyspec/tests/phase1/conftest.py +++ /dev/null @@ -1,39 +0,0 @@ -import pytest - -from eth2spec.phase1 import spec -from preset_loader import loader - -from tests.phase0 import helpers as phase1_helpers -from tests.phase1 import helpers as helpers - - -def pytest_addoption(parser): - parser.addoption( - "--config", action="store", default="minimal", help="config: make the pyspec use the specified configuration" - ) - - -@pytest.fixture(autouse=True) -def config(request): - config_name = request.config.getoption("--config") - presets = loader.load_presets('../../configs/', config_name) - spec.apply_constants_preset(presets) - helpers.spec = spec - phase1_helpers.spec = spec - request.function.__globals__['spec'] = spec - request.function.__globals__['helpers'] = helpers - - -@pytest.fixture -def num_validators(config): - return spec.SLOTS_PER_EPOCH * 8 - - -@pytest.fixture -def deposit_data_leaves(): - return list() - - -@pytest.fixture -def state(num_validators, deposit_data_leaves): - return helpers.create_genesis_state(num_validators, deposit_data_leaves) diff --git a/test_libs/pyspec/tests/phase1/epoch_processing/test_phase0_epoch_processing.py b/test_libs/pyspec/tests/phase1/epoch_processing/test_phase0_epoch_processing.py deleted file mode 100644 index ec30125600..0000000000 --- a/test_libs/pyspec/tests/phase1/epoch_processing/test_phase0_epoch_processing.py +++ /dev/null @@ -1,2 +0,0 @@ -from tests.phase0.epoch_processing.test_process_crosslinks import * -from tests.phase0.epoch_processing.test_process_registry_updates import * diff --git a/test_libs/pyspec/tests/phase1/helpers.py b/test_libs/pyspec/tests/phase1/helpers.py deleted file mode 100644 index d7adbd8027..0000000000 --- a/test_libs/pyspec/tests/phase1/helpers.py +++ /dev/null @@ -1,39 +0,0 @@ -from py_ecc import bls - -from tests.phase0.helpers import * - - -def get_valid_early_derived_secret_reveal(state, epoch=None): - current_epoch = spec.get_current_epoch(state) - revealed_index = spec.get_active_validator_indices(state, current_epoch)[-1] - masker_index = spec.get_active_validator_indices(state, current_epoch)[0] - - if epoch is None: - epoch = current_epoch + spec.CUSTODY_PERIOD_TO_RANDAO_PADDING - - reveal = bls.sign( - message_hash=spec.hash_tree_root(epoch), - privkey=privkeys[revealed_index], - domain=spec.get_domain( - state=state, - domain_type=spec.DOMAIN_RANDAO, - message_epoch=epoch, - ), - ) - mask = bls.sign( - message_hash=spec.hash_tree_root(epoch), - privkey=privkeys[masker_index], - domain=spec.get_domain( - state=state, - domain_type=spec.DOMAIN_RANDAO, - message_epoch=epoch, - ), - ) - - return spec.EarlyDerivedSecretReveal( - revealed_index=revealed_index, - epoch=epoch, - reveal=reveal, - masker_index=masker_index, - mask=mask, - ) diff --git a/test_libs/pyspec/tests/phase1/test_finality.py b/test_libs/pyspec/tests/phase1/test_finality.py deleted file mode 100644 index 1e4e0374a2..0000000000 --- a/test_libs/pyspec/tests/phase1/test_finality.py +++ /dev/null @@ -1,4 +0,0 @@ -import pytest - - -from tests.phase0.test_finality import * diff --git a/test_libs/pyspec/tests/phase1/test_sanity.py b/test_libs/pyspec/tests/phase1/test_sanity.py deleted file mode 100644 index a9b4d71908..0000000000 --- a/test_libs/pyspec/tests/phase1/test_sanity.py +++ /dev/null @@ -1,4 +0,0 @@ -import pytest - - -from tests.phase0.test_sanity import * From 92916e5383ee2eb7139e52bed283bf979feb160d Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 31 May 2019 13:08:42 +0200 Subject: [PATCH 234/308] Tests run with both spec versions --- test_libs/pyspec/eth2spec/test/context.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index d0869ba686..670654a92f 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -110,4 +110,13 @@ def entry(*args, **kw): return entry -with_all_phases = with_phase0 +def with_all_phases(fn): + """ + Decorator to run everything with all availible spec phases + """ + def entry(*args, **kw): + kw['spec'] = spec_phase0 + fn(*args, **kw) + kw['spec'] = spec_phase1 + return fn(*args, **kw) + return entry From 28d98b5cf47ce2b85a618917a5a02b36b603ba93 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Fri, 31 May 2019 13:20:10 +0200 Subject: [PATCH 235/308] PEP8 cleanups --- Makefile | 3 +-- .../test_process_early_derived_secret_reveal.py | 5 +++-- test_libs/pyspec/eth2spec/test/helpers/deposits.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index af71f44fbd..33eb3ecec9 100644 --- a/Makefile +++ b/Makefile @@ -50,8 +50,7 @@ install_lint: lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ - flake8 --max-line-length=120 ./eth2spec; \ - flake8 --max-line-length=120 --ignore=F401,F403,F405,F821 ./tests; + flake8 --max-line-length=120 ./eth2spec # "make pyspec" to create the pyspec for all phases. pyspec: $(PY_SPEC_ALL_TARGETS) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py index 4b4608d4da..4b7de56c86 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py @@ -55,7 +55,7 @@ def test_reveal_from_current_epoch(spec, state): # @spec_state_test # def test_reveal_from_past_epoch(state): # randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state) - 1) -# +# # yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) @@ -99,7 +99,8 @@ def test_double_reveal(spec, state): intermediate_state, spec.get_current_epoch(pre_state) + spec.RANDAO_PENALTY_EPOCHS + 1, ) - post_state = dict(run_early_derived_secret_reveal_processing(spec, intermediate_state, randao_key_reveal2, False))['post'] + res = dict(run_early_derived_secret_reveal_processing(spec, intermediate_state, randao_key_reveal2, False)) + post_state = res['post'] yield 'randao_key_reveal', [randao_key_reveal1, randao_key_reveal2] yield 'post', post_state diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 9711f8b0fb..6d9b6e118b 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -70,7 +70,7 @@ def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_c withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(pubkey)[1:] deposit, root, deposit_data_leaves = build_deposit( - spec, + spec, state, deposit_data_leaves, pubkey, From 3cb43fcc2742f75cd203050e8ecff3207d297c17 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 01:49:52 +0200 Subject: [PATCH 236/308] fix ssz infer type decorator, one None argument too many --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index ed3e16f63e..47b4b5c9a6 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -420,10 +420,10 @@ def infer_input_type(fn): """ Decorator to run infer_type on the obj if typ argument is None """ - def infer_helper(obj, *args, typ=None, **kwargs): + def infer_helper(obj, typ=None, **kwargs): if typ is None: typ = infer_type(obj) - return fn(obj, *args, typ=typ, **kwargs) + return fn(obj, typ=typ, **kwargs) return infer_helper From 1c734d30a0d44849bb42687e4b3719ee5d3432f3 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 01:50:28 +0200 Subject: [PATCH 237/308] fix utils init --- test_libs/pyspec/eth2spec/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/__init__.py b/test_libs/pyspec/eth2spec/utils/__init__.py index 9f8b364287..4820447980 100644 --- a/test_libs/pyspec/eth2spec/utils/__init__.py +++ b/test_libs/pyspec/eth2spec/utils/__init__.py @@ -1,4 +1,4 @@ from .merkle_minimal import * from .hash_function import * -from .bls_stub import * +from .bls import * from .ssz import * From 33233c98ff4ef033098a0efcdef075f0e56bdedf Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 01:51:09 +0200 Subject: [PATCH 238/308] fix ssz imports --- test_libs/pyspec/eth2spec/test/helpers/attestations.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/block.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/block_header.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/deposits.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/transfers.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py | 2 +- test_libs/pyspec/eth2spec/test/sanity/test_blocks.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/attestations.py index 6ac0b994eb..45ee10c3ec 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/attestations.py @@ -16,7 +16,7 @@ from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import bls_sign, bls_aggregate_signatures -from eth2spec.utils.minimal_ssz import hash_tree_root +from eth2spec.utils.ssz.ssz_impl import hash_tree_root def build_attestation_data(state, slot, shard): diff --git a/test_libs/pyspec/eth2spec/test/helpers/block.py b/test_libs/pyspec/eth2spec/test/helpers/block.py index 715cf82dbe..28fd1229f7 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block.py @@ -8,7 +8,7 @@ ) from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import bls_sign, only_with_bls -from eth2spec.utils.minimal_ssz import signing_root, hash_tree_root +from eth2spec.utils.ssz.ssz_impl import signing_root, hash_tree_root # Fully ignore the function if BLS is off, beacon-proposer index calculation is slow. diff --git a/test_libs/pyspec/eth2spec/test/helpers/block_header.py b/test_libs/pyspec/eth2spec/test/helpers/block_header.py index 9aba62d37d..ad3fbc166f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/block_header.py +++ b/test_libs/pyspec/eth2spec/test/helpers/block_header.py @@ -3,7 +3,7 @@ from eth2spec.phase0.spec import get_domain from eth2spec.utils.bls import bls_sign -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root def sign_block_header(state, header, privkey): diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 2db3ae03c3..3080eba7f5 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -5,7 +5,7 @@ from eth2spec.test.helpers.keys import pubkeys, privkeys from eth2spec.utils.bls import bls_sign from eth2spec.utils.merkle_minimal import calc_merkle_tree_from_leaves, get_merkle_root, get_merkle_proof -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root def build_deposit_data(state, pubkey, privkey, amount, withdrawal_credentials, signed=False): diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index 01011cacd0..489791907f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -3,7 +3,7 @@ from eth2spec.phase0.spec import Eth1Data, ZERO_HASH, get_active_validator_indices from eth2spec.test.helpers.keys import pubkeys -from eth2spec.utils.minimal_ssz import hash_tree_root +from eth2spec.utils.ssz.ssz_impl import hash_tree_root def build_mock_validator(i: int, balance: int): diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index 2045f48ad6..9913ef88ad 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -5,7 +5,7 @@ from eth2spec.test.helpers.keys import pubkeys, privkeys from eth2spec.test.helpers.state import get_balance from eth2spec.utils.bls import bls_sign -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=None, signed=False): diff --git a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py index 54376d694b..e4a8158776 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/voluntary_exits.py @@ -3,7 +3,7 @@ from eth2spec.phase0.spec import VoluntaryExit, get_domain from eth2spec.utils.bls import bls_sign -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root def build_voluntary_exit(state, epoch, validator_index, privkey, signed=False): diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 654a41d62f..3d22347957 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -3,7 +3,7 @@ import eth2spec.phase0.spec as spec from eth2spec.utils.bls import bls_sign -from eth2spec.utils.minimal_ssz import signing_root +from eth2spec.utils.ssz.ssz_impl import signing_root from eth2spec.phase0.spec import ( # SSZ VoluntaryExit, From 77d4f0b407d05db6fae645d629878afeefe9aeef Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 01:53:50 +0200 Subject: [PATCH 239/308] update generator type hinting to use new type syntax --- .../eth2spec/test/sanity/test_blocks.py | 29 ++++++++++--------- .../pyspec/eth2spec/test/test_finality.py | 9 +++--- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 3d22347957..11ae4918e5 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -1,4 +1,5 @@ from copy import deepcopy +from typing import List import eth2spec.phase0.spec as spec from eth2spec.utils.bls import bls_sign @@ -36,7 +37,7 @@ def test_empty_block_transition(state): yield 'pre', state block = build_empty_block_for_next_slot(state, signed=True) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -54,7 +55,7 @@ def test_skipped_slots(state): block = build_empty_block_for_next_slot(state) block.slot += 3 sign_block(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -72,7 +73,7 @@ def test_empty_epoch_transition(state): block = build_empty_block_for_next_slot(state) block.slot += spec.SLOTS_PER_EPOCH sign_block(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -91,7 +92,7 @@ def test_empty_epoch_transition(state): # block = build_empty_block_for_next_slot(state) # block.slot += spec.SLOTS_PER_EPOCH * 5 # sign_block(state, block, proposer_index=0) -# yield 'blocks', [block], [spec.BeaconBlock] +# yield 'blocks', [block], List[spec.BeaconBlock] # # state_transition(state, block) # yield 'post', state @@ -119,7 +120,7 @@ def test_proposer_slashing(state): block = build_empty_block_for_next_slot(state) block.body.proposer_slashings.append(proposer_slashing) sign_block(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -152,7 +153,7 @@ def test_attester_slashing(state): block = build_empty_block_for_next_slot(state) block.body.attester_slashings.append(attester_slashing) sign_block(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -189,7 +190,7 @@ def test_deposit_in_block(state): block.body.deposits.append(deposit) sign_block(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -216,7 +217,7 @@ def test_deposit_top_up(state): block.body.deposits.append(deposit) sign_block(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -252,7 +253,7 @@ def test_attestation(state): sign_block(state, epoch_block) state_transition(state, epoch_block) - yield 'blocks', [attestation_block, epoch_block], [spec.BeaconBlock] + yield 'blocks', [attestation_block, epoch_block], List[spec.BeaconBlock] yield 'post', state assert len(state.current_epoch_attestations) == 0 @@ -298,7 +299,7 @@ def test_voluntary_exit(state): sign_block(state, exit_block) state_transition(state, exit_block) - yield 'blocks', [initiate_exit_block, exit_block], [spec.BeaconBlock] + yield 'blocks', [initiate_exit_block, exit_block], List[spec.BeaconBlock] yield 'post', state assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH @@ -326,7 +327,7 @@ def test_transfer(state): block.body.transfers.append(transfer) sign_block(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] state_transition(state, block) yield 'post', state @@ -355,7 +356,7 @@ def test_balance_driven_status_transitions(state): sign_block(state, block) state_transition(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] yield 'post', state assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH @@ -371,7 +372,7 @@ def test_historical_batch(state): block = build_empty_block_for_next_slot(state, signed=True) state_transition(state, block) - yield 'blocks', [block], [spec.BeaconBlock] + yield 'blocks', [block], List[spec.BeaconBlock] yield 'post', state assert state.slot == block.slot @@ -399,7 +400,7 @@ def test_historical_batch(state): # # state_transition(state, block) # -# yield 'blocks', [block], [spec.BeaconBlock] +# yield 'blocks', [block], List[spec.BeaconBlock] # yield 'post', state # # assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index cdd09bf239..b1948ad8dc 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -1,4 +1,5 @@ from copy import deepcopy +from typing import List import eth2spec.phase0.spec as spec from eth2spec.phase0.spec import ( @@ -86,7 +87,7 @@ def test_finality_rule_4(state): assert state.finalized_epoch == prev_state.current_justified_epoch assert state.finalized_root == prev_state.current_justified_root - yield 'blocks', blocks, [spec.BeaconBlock] + yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state @@ -116,7 +117,7 @@ def test_finality_rule_1(state): assert state.finalized_epoch == prev_state.previous_justified_epoch assert state.finalized_root == prev_state.previous_justified_root - yield 'blocks', blocks, [spec.BeaconBlock] + yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state @@ -148,7 +149,7 @@ def test_finality_rule_2(state): blocks += new_blocks - yield 'blocks', blocks, [spec.BeaconBlock] + yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state @@ -197,5 +198,5 @@ def test_finality_rule_3(state): assert state.finalized_epoch == prev_state.current_justified_epoch assert state.finalized_root == prev_state.current_justified_root - yield 'blocks', blocks, [spec.BeaconBlock] + yield 'blocks', blocks, List[spec.BeaconBlock] yield 'post', state From df25f22f012d606daaf2264489e75e2a9f040a54 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 01:55:20 +0200 Subject: [PATCH 240/308] get rid of scope-problem inducing init trick --- test_libs/pyspec/eth2spec/utils/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/__init__.py b/test_libs/pyspec/eth2spec/utils/__init__.py index 4820447980..e69de29bb2 100644 --- a/test_libs/pyspec/eth2spec/utils/__init__.py +++ b/test_libs/pyspec/eth2spec/utils/__init__.py @@ -1,4 +0,0 @@ -from .merkle_minimal import * -from .hash_function import * -from .bls import * -from .ssz import * From 7df788c7d5ba3e070b391cebf6713654d7999fc8 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 02:22:14 +0200 Subject: [PATCH 241/308] resolve linting problems, except ssz-partials --- scripts/phase0/build_spec.py | 4 ++-- scripts/phase0/function_puller.py | 2 +- test_libs/pyspec/eth2spec/debug/decode.py | 7 ++++++- test_libs/pyspec/eth2spec/debug/encode.py | 6 +++++- test_libs/pyspec/eth2spec/debug/random_value.py | 8 +++++++- test_libs/pyspec/eth2spec/utils/ssz/__init__.py | 2 -- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 10 ++++++++-- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 13 ++++++++----- 8 files changed, 37 insertions(+), 15 deletions(-) diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py index c7a377297c..0d5f640942 100644 --- a/scripts/phase0/build_spec.py +++ b/scripts/phase0/build_spec.py @@ -18,8 +18,8 @@ def build_phase0_spec(sourcefile, outfile): signing_root, ) from eth2spec.utils.ssz.ssz_typing import ( - uint8, uint16, uint32, uint64, uint128, uint256, - Container, Vector, BytesN + # unused: uint8, uint16, uint32, uint128, uint256, + uint64, Container, Vector, BytesN ) from eth2spec.utils.hash_function import hash from eth2spec.utils.bls import ( diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index d7765a4a44..ea8688e86f 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -68,7 +68,7 @@ def get_spec(file_name: str) -> List[str]: for type_line in ssz_type: if len(type_line) > 0: code_lines.append(' ' + type_line) - code_lines.append('\n') + code_lines.append('') for (ssz_type_name, _) in type_defs: code_lines.append(f' global_vars["{ssz_type_name}"] = {ssz_type_name}') code_lines.append(' global_vars["ssz_types"] = [') diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index c9657dc287..86716c5f4b 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -1,5 +1,10 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.utils.ssz.ssz_typing import * +from eth2spec.utils.ssz.ssz_typing import ( + is_uint_type, is_bool_type, is_list_type, + is_vector_type, is_bytes_type, is_bytesn_type, is_container_type, + read_vector_elem_type, read_list_elem_type, + Vector, BytesN +) def decode(data, typ): diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index a3c3c1189a..8be567f172 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -1,5 +1,9 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.utils.ssz.ssz_typing import * +from eth2spec.utils.ssz.ssz_typing import ( + is_uint_type, is_bool_type, is_list_type, is_vector_type, is_container_type, + read_elem_type, + uint +) def encode(value, typ, include_hash_tree_roots=False): diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index ab9c4c8859..43c3de3496 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -2,9 +2,15 @@ from typing import Any from enum import Enum -from eth2spec.utils.ssz.ssz_typing import * from eth2spec.utils.ssz.ssz_impl import is_basic_type +from eth2spec.utils.ssz.ssz_typing import ( + is_uint_type, is_bool_type, is_list_type, + is_vector_type, is_bytes_type, is_bytesn_type, is_container_type, + read_vector_elem_type, read_list_elem_type, + uint_byte_size +) + # in bytes UINT_SIZES = [1, 2, 4, 8, 16, 32] diff --git a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py b/test_libs/pyspec/eth2spec/utils/ssz/__init__.py index 6543bd9e51..e69de29bb2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/__init__.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/__init__.py @@ -1,2 +0,0 @@ -from .ssz_impl import * -from .ssz_partials import * diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 0ef89f97f6..fac8d80512 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -1,5 +1,11 @@ from ..merkle_minimal import merkleize_chunks, hash -from .ssz_typing import * +from eth2spec.utils.ssz.ssz_typing import ( + is_uint_type, is_bool_type, is_container_type, + is_list_kind, is_vector_kind, + read_vector_elem_type, read_elem_type, + uint_byte_size, + infer_input_type +) # SSZ Serialization # ----------------------------- @@ -22,6 +28,7 @@ def serialize_basic(value, typ): else: raise Exception("Type not supported: {}".format(typ)) + def deserialize_basic(value, typ): if is_uint_type(typ): return typ(int.from_bytes(value, 'little')) @@ -148,4 +155,3 @@ def signing_root(obj, typ): # ignore last field leaves = [hash_tree_root(field_value, typ=field_typ) for field_value, field_typ in obj.get_typed_values()[:-1]] return merkleize_chunks(chunkify(b''.join(leaves))) - diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 47b4b5c9a6..9424e9bd36 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -3,10 +3,6 @@ from typing import Union from typing_inspect import get_origin -T = TypeVar('T') -L = TypeVar('L') - - # SSZ integers # ----------------------------- @@ -21,6 +17,7 @@ def __new__(cls, value, *args, **kwargs): class uint8(uint): byte_len = 1 + def __new__(cls, value, *args, **kwargs): if value.bit_length() > 8: raise ValueError("value out of bounds for uint8") @@ -32,6 +29,7 @@ def __new__(cls, value, *args, **kwargs): class uint16(uint): byte_len = 2 + def __new__(cls, value, *args, **kwargs): if value.bit_length() > 16: raise ValueError("value out of bounds for uint16") @@ -40,6 +38,7 @@ def __new__(cls, value, *args, **kwargs): class uint32(uint): byte_len = 4 + def __new__(cls, value, *args, **kwargs): if value.bit_length() > 32: raise ValueError("value out of bounds for uint16") @@ -52,6 +51,7 @@ def __new__(cls, value, *args, **kwargs): class uint128(uint): byte_len = 16 + def __new__(cls, value, *args, **kwargs): if value.bit_length() > 128: raise ValueError("value out of bounds for uint128") @@ -60,6 +60,7 @@ def __new__(cls, value, *args, **kwargs): class uint256(uint): byte_len = 32 + def __new__(cls, value, *args, **kwargs): if value.bit_length() > 256: raise ValueError("value out of bounds for uint256") @@ -397,12 +398,14 @@ def get_zero_value(typ): elif issubclass(typ, Container): result = typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) else: - return Exception("Type not supported: {}".format(typ)) + return Exception("Type not supported: {}".format(typ)) return result + # Type helpers # ----------------------------- + def infer_type(obj): if is_uint_type(obj.__class__): return obj.__class__ From f3a517b6f62c1dc24291ece031addc7cfe6758fe Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 02:32:11 +0200 Subject: [PATCH 242/308] fix minor lint problems --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 9424e9bd36..7f20284a8f 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -6,6 +6,7 @@ # SSZ integers # ----------------------------- + class uint(int): byte_len = 0 @@ -23,6 +24,7 @@ def __new__(cls, value, *args, **kwargs): raise ValueError("value out of bounds for uint8") return super().__new__(cls, value) + # Alias for uint8 byte = NewType('byte', uint8) From b79f01e2fa3aaae32c57015df67159e3eefa7a06 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 02:39:41 +0200 Subject: [PATCH 243/308] update flake8 to support type annotation in linting, ignore operator format and default param equals without spaces warnings --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 401e4bdc9f..7d7c75052e 100644 --- a/Makefile +++ b/Makefile @@ -40,11 +40,11 @@ citest: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml . install_lint: - cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.5.0 + cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.7.7 lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ - flake8 --max-line-length=120 ./eth2spec; + flake8 --ignore=E252,W504 --max-line-length=120 ./eth2spec; # "make pyspec" to create the pyspec for all phases. pyspec: $(PY_SPEC_ALL_TARGETS) From fae1e9285d59767ee6749389a04bfc8cd085b359 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 02:40:29 +0200 Subject: [PATCH 244/308] resolve some new lint issues detected by newer version, and a few looked over ones --- .../eth2spec/test/block_processing/test_process_deposit.py | 4 ++-- test_libs/pyspec/eth2spec/test/helpers/bitfields.py | 6 +++--- test_libs/pyspec/eth2spec/test/helpers/transfers.py | 2 +- test_libs/pyspec/eth2spec/utils/hash_function.py | 3 ++- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py index 0430dd12fe..b12ec6bd92 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py @@ -138,7 +138,7 @@ def test_wrong_deposit_for_deposit_count(state): pubkey_1, privkey_1, spec.MAX_EFFECTIVE_BALANCE, - withdrawal_credentials=b'\x00'*32, + withdrawal_credentials=b'\x00' * 32, signed=True, ) deposit_count_1 = len(deposit_data_leaves) @@ -153,7 +153,7 @@ def test_wrong_deposit_for_deposit_count(state): pubkey_2, privkey_2, spec.MAX_EFFECTIVE_BALANCE, - withdrawal_credentials=b'\x00'*32, + withdrawal_credentials=b'\x00' * 32, signed=True, ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/bitfields.py b/test_libs/pyspec/eth2spec/test/helpers/bitfields.py index 7c25d073ab..50e5b6cbad 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/bitfields.py +++ b/test_libs/pyspec/eth2spec/test/helpers/bitfields.py @@ -5,7 +5,7 @@ def set_bitfield_bit(bitfield, i): byte_index = i // 8 bit_index = i % 8 return ( - bitfield[:byte_index] + - bytes([bitfield[byte_index] | (1 << bit_index)]) + - bitfield[byte_index + 1:] + bitfield[:byte_index] + + bytes([bitfield[byte_index] | (1 << bit_index)]) + + bitfield[byte_index + 1:] ) diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index 9913ef88ad..91f8ea60e2 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -36,7 +36,7 @@ def get_valid_transfer(state, slot=None, sender_index=None, amount=None, fee=Non # ensure withdrawal_credentials reproducible state.validator_registry[transfer.sender].withdrawal_credentials = ( - spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] + spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] ) return transfer diff --git a/test_libs/pyspec/eth2spec/utils/hash_function.py b/test_libs/pyspec/eth2spec/utils/hash_function.py index acd13edc40..f965827d06 100644 --- a/test_libs/pyspec/eth2spec/utils/hash_function.py +++ b/test_libs/pyspec/eth2spec/utils/hash_function.py @@ -1,4 +1,5 @@ from hashlib import sha256 -def hash(x): return sha256(x).digest() +def hash(x): + return sha256(x).digest() diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index fac8d80512..9c1fcdc611 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -57,7 +57,7 @@ def serialize(obj, typ=None): if is_basic_type(typ): return serialize_basic(obj, typ) elif is_list_kind(typ) or is_vector_kind(typ): - return encode_series(obj, [read_elem_type(typ)]*len(obj)) + return encode_series(obj, [read_elem_type(typ)] * len(obj)) elif is_container_type(typ): return encode_series(obj.get_field_values(), typ.get_field_types()) else: From 8e8ef2d0a40e1f37468b3c0622249f85e99d18cc Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 1 Jun 2019 03:19:29 +0200 Subject: [PATCH 245/308] Fix forgotten setup.py change, make generators using ssz work again --- test_libs/pyspec/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index 1a131a4171..e99b911eeb 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -9,5 +9,6 @@ "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.7.3", "py_ecc>=1.6.0", + "typing_inspect==0.4.0" ] ) From 6617f891462c0d0ed674585f8402d2e06778f56a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 1 Jun 2019 11:00:31 +0800 Subject: [PATCH 246/308] Rewrite `reqs_checksum` --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 164dc98a03..9cfc749118 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,13 +36,13 @@ commands: steps: - restore_cached_venv: venv_name: v1-pyspec-05 - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}--{{ checksum "deposit_contract/requirements-testing.txt" }}' + reqs_checksum: cache-v1-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_default_cached_venv: description: Save a venv into a cache with default keys" steps: - save_cached_venv: venv_name: v1-pyspec-05 - reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}--{{ checksum "deposit_contract/requirements-testing.txt" }}' + reqs_checksum: cache-v1-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./test_libs/pyspec/venv jobs: checkout_specs: From 2246f1b934d19f1e8c1e01a93300878499220625 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Sun, 2 Jun 2019 13:13:52 -0700 Subject: [PATCH 247/308] Update 0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index b5f208488d..046aea7b53 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1433,7 +1433,7 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[List[Gwei], List[Gwei]]: matching_head_attestations = get_matching_head_attestations(state, previous_epoch) for attestations in (matching_source_attestations, matching_target_attestations, matching_head_attestations): unslashed_attesting_indices = get_unslashed_attesting_indices(state, attestations) - attesting_balance = get_attesting_balance(state, attestations) + attesting_balance = get_total_balance(state, unslashed_attesting_indices) for index in eligible_validator_indices: if index in unslashed_attesting_indices: rewards[index] += get_base_reward(state, index) * attesting_balance // total_balance From e578c9583715531781674aa0f6acb7bffbb72cf5 Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Mon, 3 Jun 2019 14:34:00 +0900 Subject: [PATCH 248/308] Update link to Prysmatic Labs implementation of SSZ (#1138) * Update link to go SSZ implementation We have changed our SSZ implementation to it's own repo so this updates the link on the spec. * Update simple-serialize.md --- specs/simple-serialize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 8633c7ed15..81126a3123 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -161,7 +161,7 @@ Let `value` be a self-signed container object. The convention is that the signat | Rust | Shasper | ParityTech | [https://github.com/paritytech/shasper/tree/master/util/ssz](https://github.com/paritytech/shasper/tree/master/util/ssz) | | TypeScript | Lodestar | ChainSafe Systems | [https://github.com/ChainSafe/ssz-js](https://github.com/ChainSafe/ssz-js) | | Java | Cava | ConsenSys | [https://www.github.com/ConsenSys/cava/tree/master/ssz](https://www.github.com/ConsenSys/cava/tree/master/ssz) | -| Go | Prysm | Prysmatic Labs | [https://github.com/prysmaticlabs/prysm/tree/master/shared/ssz](https://github.com/prysmaticlabs/prysm/tree/master/shared/ssz) | +| Go | Prysm | Prysmatic Labs | [https://github.com/prysmaticlabs/go-ssz](https://github.com/prysmaticlabs/go-ssz) | | Swift | Yeeth | Dean Eigenmann | [https://github.com/yeeth/SimpleSerialize.swift](https://github.com/yeeth/SimpleSerialize.swift) | | C# | | Jordan Andrews | [https://github.com/codingupastorm/csharp-ssz](https://github.com/codingupastorm/csharp-ssz) | | C++ | | Jiyun Kim | [https://github.com/NAKsir-melody/cpp_ssz](https://github.com/NAKsir-melody/cpp_ssz) | From b8eddfafe77e3bcc81823afddf715534970ec540 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 3 Jun 2019 16:23:03 +0800 Subject: [PATCH 249/308] Remove version from `reqs_checksum` --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9cfc749118..d4c672ca26 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,13 +36,13 @@ commands: steps: - restore_cached_venv: venv_name: v1-pyspec-05 - reqs_checksum: cache-v1-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} + reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_default_cached_venv: description: Save a venv into a cache with default keys" steps: - save_cached_venv: venv_name: v1-pyspec-05 - reqs_checksum: cache-v1-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} + reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./test_libs/pyspec/venv jobs: checkout_specs: From a5576059f8e5504e065dffb428129695390bb4b1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 3 Jun 2019 17:14:29 +0800 Subject: [PATCH 250/308] Fix `ssz_partials.py` linter errors --- .../pyspec/eth2spec/utils/ssz/ssz_partials.py | 93 ++++++++++++++----- 1 file changed, 71 insertions(+), 22 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py index 6a243f4f35..6723674ebf 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py @@ -1,17 +1,44 @@ from ..merkle_minimal import hash, next_power_of_two -from .ssz_typing import * -from .ssz_impl import * +from .ssz_typing import ( + Container, + infer_input_type, + is_bool_type, + is_bytes_type, + is_bytesn_type, + is_container_type, + is_list_kind, + is_list_type, + is_uint_type, + is_vector_kind, + is_vector_type, + read_elem_type, + uint_byte_size, +) +from .ssz_impl import ( + chunkify, + deserialize_basic, + get_typed_values, + is_basic_type, + is_bottom_layer_kind, + pack, + serialize_basic, +) + ZERO_CHUNK = b'\x00' * 32 + def last_power_of_two(x): - return next_power_of_two(x+1) // 2 + return next_power_of_two(x + 1) // 2 + def concat_generalized_indices(x, y): return x * last_power_of_two(y) + y - last_power_of_two(y) + def rebase(objs, new_root): - return {concat_generalized_indices(new_root, k): v for k,v in objs.items()} + return {concat_generalized_indices(new_root, k): v for k, v in objs.items()} + def constrict_generalized_index(x, q): depth = last_power_of_two(x // q) @@ -20,32 +47,36 @@ def constrict_generalized_index(x, q): return None return o + def unrebase(objs, q): o = {} - for k,v in objs.items(): + for k, v in objs.items(): new_k = constrict_generalized_index(k, q) if new_k is not None: o[new_k] = v return o + def filler(starting_position, chunk_count): at, skip, end = chunk_count, 1, next_power_of_two(chunk_count) value = ZERO_CHUNK o = {} while at < end: - while at % (skip*2) == 0: + while at % (skip * 2) == 0: skip *= 2 value = hash(value + value) o[(starting_position + at) // skip] = value at += skip return o + def merkle_tree_of_chunks(chunks, root): starting_index = root * next_power_of_two(len(chunks)) - o = {starting_index+i: chunk for i,chunk in enumerate(chunks)} + o = {starting_index + i: chunk for i, chunk in enumerate(chunks)} o = {**o, **filler(starting_index, len(chunks))} return o + @infer_input_type def ssz_leaves(obj, typ=None, root=1): if is_list_kind(typ): @@ -57,33 +88,36 @@ def ssz_leaves(obj, typ=None, root=1): if is_bottom_layer_kind(typ): data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, read_elem_type(typ)) q = {**o, **merkle_tree_of_chunks(chunkify(data), base)} - #print(obj, root, typ, base, list(q.keys())) + # print(obj, root, typ, base, list(q.keys())) return(q) else: fields = get_typed_values(obj, typ=typ) sub_base = base * next_power_of_two(len(fields)) for i, (elem, elem_type) in enumerate(fields): - o = {**o, **ssz_leaves(elem, typ=elem_type, root=sub_base+i)} + o = {**o, **ssz_leaves(elem, typ=elem_type, root=sub_base + i)} q = {**o, **filler(sub_base, len(fields))} - #print(obj, root, typ, base, list(q.keys())) + # print(obj, root, typ, base, list(q.keys())) return(q) + def fill(objects): - objects = {k:v for k,v in objects.items()} + objects = {k: v for k, v in objects.items()} keys = sorted(objects.keys())[::-1] pos = 0 while pos < len(keys): k = keys[pos] - if k in objects and k^1 in objects and k//2 not in objects: - objects[k//2] = hash(objects[k&-2] + objects[k|1]) + if k in objects and k ^ 1 in objects and k // 2 not in objects: + objects[k // 2] = hash(objects[k & - 2] + objects[k | 1]) keys.append(k // 2) pos += 1 return objects + @infer_input_type def ssz_full(obj, typ=None): return fill(ssz_leaves(obj, typ=typ)) + def get_basic_type_size(typ): if is_uint_type(typ): return uint_byte_size(typ) @@ -92,6 +126,7 @@ def get_basic_type_size(typ): else: raise Exception("Type not basic: {}".format(typ)) + def get_bottom_layer_element_position(typ, base, length, index): """ Returns the generalized index and the byte range of the index'th value @@ -104,7 +139,8 @@ def get_bottom_layer_element_position(typ, base, length, index): chunk_count = (1 if is_basic_type(typ) else length) * elem_size // 32 generalized_index = base * next_power_of_two(chunk_count) + chunk_index start = elem_size * index % 32 - return generalized_index, start, start+elem_size + return generalized_index, start, start + elem_size + @infer_input_type def get_generalized_indices(obj, path, typ=None, root=1): @@ -132,14 +168,17 @@ def get_generalized_indices(obj, path, typ=None, root=1): root=base * next_power_of_two(field_count) + index ) + def get_branch_indices(tree_index): o = [tree_index, tree_index ^ 1] while o[-1] > 1: o.append((o[-1] // 2) ^ 1) return o[:-1] + def remove_redundant_indices(obj): - return {k:v for k,v in obj.items() if not (k*2 in obj and k*2+1 in obj)} + return {k: v for k, v in obj.items() if not (k * 2 in obj and k * 2 + 1 in obj)} + def merge(*args): o = {} @@ -147,14 +186,19 @@ def merge(*args): o = {**o, **arg} return fill(o) + @infer_input_type def get_nodes_along_path(obj, path, typ=None): indices = get_generalized_indices(obj, path, typ=typ) - return remove_redundant_indices(merge(*({i:obj.objects[i] for i in get_branch_indices(index)} for index in indices))) + return remove_redundant_indices(merge( + *({i: obj.objects[i] for i in get_branch_indices(index)} for index in indices) + )) + class OutOfRangeException(Exception): pass + class SSZPartial(): def __init__(self, typ, objects, root=1): assert not is_basic_type(typ) @@ -204,9 +248,9 @@ def __iter__(self): def __len__(self): if is_list_kind(self.typ): - if self.root*2+1 not in self.objects: - raise OutOfRangeException("Do not have required data: {}".format(self.root*2+1)) - return int.from_bytes(self.objects[self.root*2+1], 'little') + if self.root * 2 + 1 not in self.objects: + raise OutOfRangeException("Do not have required data: {}".format(self.root * 2 + 1)) + return int.from_bytes(self.objects[self.root * 2 + 1], 'little') elif is_vector_kind(self.typ): return self.typ.length elif is_container_type(self.typ): @@ -222,7 +266,9 @@ def full_value(self): elif is_vector_kind(self.typ): return self.typ(*(self[i] for i in range(len(self)))) elif is_container_type(self.typ): - full_value = lambda x: x.full_value() if hasattr(x, 'full_value') else x + def full_value(x): + return x.full_value() if hasattr(x, 'full_value') else x + return self.typ(**{field: full_value(self.getter(field)) for field in self.typ.get_field_names()}) elif is_basic_type(self.typ): return self.getter(0) @@ -235,8 +281,8 @@ def hash_tree_root(self): pos = 0 while pos < len(keys): k = keys[pos] - if k in o and k^1 in o and k//2 not in o: - o[k//2] = hash(o[k&-2] + o[k|1]) + if k in o and k ^ 1 in o and k // 2 not in o: + o[k // 2] = hash(o[k & - 2] + o[k | 1]) keys.append(k // 2) pos += 1 return o[self.root] @@ -244,13 +290,16 @@ def hash_tree_root(self): def __str__(self): return str(self.full_value()) + def ssz_partial(typ, objects, root=1): ssz_type = ( Container if is_container_type(typ) else typ if (is_vector_type(typ) or is_bytesn_type(typ)) else object ) + class Partial(SSZPartial, ssz_type): pass + if is_container_type(typ): Partial.__annotations__ = typ.__annotations__ o = Partial(typ, objects, root=root) From 6cd981128db1d4b4f215f4749a323ac26ce3433e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 3 Jun 2019 17:21:38 +0800 Subject: [PATCH 251/308] Delete extra newline --- scripts/phase0/function_puller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/phase0/function_puller.py b/scripts/phase0/function_puller.py index ea8688e86f..0f4b21c0e4 100644 --- a/scripts/phase0/function_puller.py +++ b/scripts/phase0/function_puller.py @@ -68,7 +68,6 @@ def get_spec(file_name: str) -> List[str]: for type_line in ssz_type: if len(type_line) > 0: code_lines.append(' ' + type_line) - code_lines.append('') for (ssz_type_name, _) in type_defs: code_lines.append(f' global_vars["{ssz_type_name}"] = {ssz_type_name}') code_lines.append(' global_vars["ssz_types"] = [') From 226adb35dfe63246509beb71baf6c2f9eb90465c Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 3 Jun 2019 14:22:03 +0200 Subject: [PATCH 252/308] Adds Docstrings and type-checking to spec builder. --- scripts/build_spec.py | 58 ++++++++++++++++++++++++++++---------- scripts/function_puller.py | 24 ++++++++++++---- 2 files changed, 62 insertions(+), 20 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index e4ad9dc5fe..2266af4efe 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -1,6 +1,13 @@ import re -import function_puller +from function_puller import ( + get_spec, + SpecObject, +) from argparse import ArgumentParser +from typing import ( + Dict, + Optional, +) PHASE0_IMPORTS = '''from typing import ( @@ -107,8 +114,16 @@ def apply_constants_preset(preset: Dict[str, Any]): ''' -def objects_to_spec(functions, constants, ssz_objects, inserts, imports): - new_type_definitions = '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in NEW_TYPES.items()]) +def objects_to_spec(functions: Dict[str, str], + constants: Dict[str, str], + ssz_objects: Dict[str, str], + inserts: Dict[str, str], + imports: Dict[str, str]) -> str: + """ + Given all the objects that constitute a spec, combine them into a single pyfile. + """ + new_type_definitions = \ + '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in NEW_TYPES.items()]) functions_spec = '\n\n'.join(functions.values()) constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]), constants)) ssz_objects_instantiation_spec = '\n'.join(map( @@ -120,7 +135,7 @@ def objects_to_spec(functions, constants, ssz_objects, inserts, imports): ssz_objects )) ssz_objects_reinitialization_spec = ( - 'def init_SSZ_types():\n global_vars = globals()\n' + 'def init_SSZ_types():\n global_vars = globals()\n' + ssz_objects_reinitialization_spec ) spec = ( @@ -139,19 +154,22 @@ def objects_to_spec(functions, constants, ssz_objects, inserts, imports): return spec -def combine_functions(old_funcitons, new_functions): +def combine_functions(old_funcitons: Dict[str, str], new_functions: Dict[str, str]) -> Dict[str, str]: for key, value in new_functions.items(): old_funcitons[key] = value return old_funcitons -def combine_constants(old_constants, new_constants): +def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, str]) -> Dict[str, str]: for key, value in new_constants.items(): old_constants[key] = value return old_constants -def dependency_order_ssz_objects(objects): +def dependency_order_ssz_objects(objects: Dict[str, str]) -> Dict[str, str]: + """ + Determines which SSZ Object is depenedent on which other and orders them appropriately + """ items = list(objects.items()) for key, value in items: dependencies = re.findall(r'(: [\[]*[A-Z][a-z][\w]+)', value) @@ -164,7 +182,11 @@ def dependency_order_ssz_objects(objects): objects[item] = objects.pop(item) -def combine_ssz_objects(old_objects, new_objects): +def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str]) -> Dict[str, str]: + """ + Thakes in old spec and new spec ssz objects, combines them, + and returns the newer versions of the objects in dependency order. + """ for key, value in new_objects.items(): # remove leading "{" and trailing "\n}" old_objects[key] = old_objects.get(key, '')[1:-3] @@ -179,7 +201,10 @@ def combine_ssz_objects(old_objects, new_objects): combine_inserts = combine_functions -def combine_spec_objects(spec0, spec1): +def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: + """ + Takes in two spec variants (as tuples of their objects) and combines them using the appropriate combiner function. + """ functions0, constants0, ssz_objects0, inserts0 = spec0 functions1, constants1, ssz_objects1, inserts1 = spec1 functions = combine_functions(functions0, functions1) @@ -189,8 +214,8 @@ def combine_spec_objects(spec0, spec1): return functions, constants, ssz_objects, inserts -def build_phase0_spec(sourcefile, outfile=None): - functions, constants, ssz_objects, inserts = function_puller.get_spec(sourcefile) +def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]: + functions, constants, ssz_objects, inserts = get_spec(sourcefile) spec = objects_to_spec(functions, constants, ssz_objects, inserts, PHASE0_IMPORTS) if outfile is not None: with open(outfile, 'w') as out: @@ -199,10 +224,13 @@ def build_phase0_spec(sourcefile, outfile=None): return spec -def build_phase1_spec(phase0_sourcefile, phase1_custody_sourcefile, phase1_shard_sourcefile, outfile=None): - phase0_spec = function_puller.get_spec(phase0_sourcefile) - phase1_custody = function_puller.get_spec(phase1_custody_sourcefile) - phase1_shard_data = function_puller.get_spec(phase1_shard_sourcefile) +def build_phase1_spec(phase0_sourcefile: str, + phase1_custody_sourcefile: str, + phase1_shard_sourcefile: str, + outfile: str=None) -> Optional[str]: + phase0_spec = get_spec(phase0_sourcefile) + phase1_custody = get_spec(phase1_custody_sourcefile) + phase1_shard_data = get_spec(phase1_shard_sourcefile) spec_objects = phase0_spec for value in [phase1_custody, phase1_shard_data]: spec_objects = combine_spec_objects(spec_objects, value) diff --git a/scripts/function_puller.py b/scripts/function_puller.py index 72cb0365c4..7a10140cdb 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -1,13 +1,25 @@ import re -from typing import Dict, Tuple +from typing import Dict, Tuple, NewType FUNCTION_REGEX = r'^def [\w_]*' BEGIN_INSERT_REGEX = r'# begin insert ' END_INSERT_REGEX = r'# end insert' +SpecObject = NewType('SpecObjects', Tuple[Dict[str, str], Dict[str, str], Dict[str, str], Dict[str, str]]) -def get_spec(file_name: str) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str], Dict[str, str]]: + +def get_spec(file_name: str) -> SpecObject: + """ + Takes in the file name of a spec.md file, opens it and returns the following objects: + functions = {function_name: function_code} + constants= {constant_name: constant_code} + ssz_objects= {object_name: object} + inserts= {insert_tag: code to be inserted} + + Note: This function makes heavy use of the inherent ordering of dicts, + if this is not supported by your python version, it will not work. + """ pulling_from = None # line number of start of latest object current_name = None # most recent section title insert_name = None # stores the label of the current insert object @@ -27,21 +39,23 @@ def get_spec(file_name: str) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, elif line[:3] == '```': pulling_from = None elif inserts_matcher.match(line) is not None: + # Find @insert names insert_name = re.search(r'@[\w]*', line).group(0) elif insert_name is not None: + # In insert mode, either the next line is more code, or the end of the insert if re.match(END_INSERT_REGEX, line) is not None: insert_name = None else: inserts[insert_name] = inserts.get(insert_name, '') + line + '\n' else: - # Handle function definitions + # Handle function definitions & ssz_objects if pulling_from is not None: func_match = function_matcher.match(line) if func_match is not None: current_name = func_match.group(0) - if function_matcher.match(current_name) is None: + if function_matcher.match(current_name) is None: # The current line is an SSZ Object ssz_objects[current_name] = ssz_objects.get(current_name, '') + line + '\n' - else: + else: # The current line is code functions[current_name] = functions.get(current_name, '') + line + '\n' # Handle constant table entries elif pulling_from is None and len(line) > 0 and line[0] == '|': From 381fcc321510a675cc63370d0db331dccf3201e7 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 3 Jun 2019 15:14:20 +0200 Subject: [PATCH 253/308] Adds label explainer --- scripts/README.md | 32 ++++++++++++++++++++++++++++++++ scripts/build_spec.py | 8 ++++---- 2 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 scripts/README.md diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000000..d6ce4c1bdf --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,32 @@ +# Building pyspecs from specs.md + +The benefits of the particular spec design is that a given `spec.md` file can be converted to a `spec.py` file for the purposes of testing and linting. The result of this is that bugs are discovered and patched more quickly. + +Specs can bue built from either a single markdown document or multiple files that must be combined in a given order. Given 2 spec objects, `build_spec.combine_spec_objects` will combine them into a single spec object which, subsequently, can be converted into a `specs.py`. + +## Usage + +For usage of the spec builder run `python3 -m build_spec --help`. + +## `@Labels` and inserts + +The functioning of the spec combiner is largely automatic in that given `spec0.md` and `spec1.md`, in that SSZ Objects will be extended and old functions will be overwritten. Extra functionality is provided for more granular control over how files are combined. In the event that only a small portion of code is to be added to an existing function, insert functionality is provided. This saves having to completely redefine the old function from `spec0.md` in `spec1.md`. This is done by marking where the change is to occur in the old file and marking which code is to be inserted in the new file. This is done as follows: + +* In the old file, a label is added as a python comment marking where the code is to be inserted. This would appear as follows in `spec0.md`: + +```python +def foo(x): + x << 1 + # @YourLabelHere + return x +``` + +* In spec1, the new code could then be inserted by having a code-block that looked as follows: + +```python +#begin insert @YourLabelHere + x += x +#end insert @YourLabelHere +``` + +**Note** that the code to be inserted has the **same level of indentation** as the surrounding code of its destination insert point. diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 2266af4efe..4ba9b616ab 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -154,10 +154,10 @@ def objects_to_spec(functions: Dict[str, str], return spec -def combine_functions(old_funcitons: Dict[str, str], new_functions: Dict[str, str]) -> Dict[str, str]: +def combine_functions(old_functions: Dict[str, str], new_functions: Dict[str, str]) -> Dict[str, str]: for key, value in new_functions.items(): - old_funcitons[key] = value - return old_funcitons + old_functions[key] = value + return old_functions def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, str]) -> Dict[str, str]: @@ -184,7 +184,7 @@ def dependency_order_ssz_objects(objects: Dict[str, str]) -> Dict[str, str]: def combine_ssz_objects(old_objects: Dict[str, str], new_objects: Dict[str, str]) -> Dict[str, str]: """ - Thakes in old spec and new spec ssz objects, combines them, + Takes in old spec and new spec ssz objects, combines them, and returns the newer versions of the objects in dependency order. """ for key, value in new_objects.items(): From fb584bc067e4aa474e4db04f27435cdeb36e15b8 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 3 Jun 2019 17:23:30 +0200 Subject: [PATCH 254/308] fix linting dependency + caching issue --- .circleci/config.yml | 10 +++++----- Makefile | 3 --- test_libs/pyspec/requirements-testing.txt | 1 + 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f3c5f6a81c..244d20c55e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,13 +60,13 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-03 + venv_name: v2-pyspec reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Install pyspec requirements - command: make install_test && make install_lint + command: make install_test - save_cached_venv: - venv_name: v1-pyspec-03 + venv_name: v2-pyspec reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' venv_path: ./test_libs/pyspec/venv test: @@ -77,7 +77,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-03 + venv_name: v2-pyspec reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run py-tests @@ -92,7 +92,7 @@ jobs: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - restore_cached_venv: - venv_name: v1-pyspec-03 + venv_name: v2-pyspec reqs_checksum: '{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}' - run: name: Run linter diff --git a/Makefile b/Makefile index 7d7c75052e..51cb1aaa3a 100644 --- a/Makefile +++ b/Makefile @@ -39,9 +39,6 @@ test: $(PY_SPEC_ALL_TARGETS) citest: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; python -m pytest --junitxml=test-reports/eth2spec/test_results.xml . -install_lint: - cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install flake8==3.7.7 - lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ flake8 --ignore=E252,W504 --max-line-length=120 ./eth2spec; diff --git a/test_libs/pyspec/requirements-testing.txt b/test_libs/pyspec/requirements-testing.txt index 388a878a90..331d0fa284 100644 --- a/test_libs/pyspec/requirements-testing.txt +++ b/test_libs/pyspec/requirements-testing.txt @@ -1,3 +1,4 @@ -r requirements.txt pytest>=3.6,<3.7 ../config_helpers +flake8==3.7.7 From 1b5370f8e242c27a02ccd28666217d3912cebaff Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Tue, 4 Jun 2019 18:20:16 +0900 Subject: [PATCH 255/308] Update 0beacon-chain-validator.md --- specs/validator/0_beacon-chain-validator.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 1bb89f8e74..b16e09feb0 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -115,12 +115,11 @@ Once a validator has been processed and added to the beacon state's `validator_r In normal operation, the validator is quickly activated at which point the validator is added to the shuffling and begins validation after an additional `ACTIVATION_EXIT_DELAY` epochs (25.6 minutes). -The function [`is_active_validator`](../core/0_beacon-chain.md#is_active_validator) can be used to check if a validator is active during a given shuffling epoch. Note that the `BeaconState` contains a field `current_shuffling_epoch` which dictates from which epoch the current active validators are taken. Usage is as follows: +The function [`is_active_validator`](../core/0_beacon-chain.md#is_active_validator) can be used to check if a validator is active during a given epoch. Usage is as follows: ```python -shuffling_epoch = state.current_shuffling_epoch validator = state.validator_registry[validator_index] -is_active = is_active_validator(validator, shuffling_epoch) +is_active = is_active_validator(validator, get_current_epoch(state)) ``` Once a validator is activated, the validator is assigned [responsibilities](#beacon-chain-responsibilities) until exited. From a7ee6f108ef5dbde7efc05d7b807ee067946f986 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 4 Jun 2019 17:42:21 +0800 Subject: [PATCH 256/308] Refactor --- test_libs/pyspec/eth2spec/debug/decode.py | 3 +- test_libs/pyspec/eth2spec/debug/encode.py | 3 +- .../pyspec/eth2spec/debug/random_value.py | 44 ++++----- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 89 +++++++++++-------- 4 files changed, 80 insertions(+), 59 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/decode.py b/test_libs/pyspec/eth2spec/debug/decode.py index 86716c5f4b..5ce1160258 100644 --- a/test_libs/pyspec/eth2spec/debug/decode.py +++ b/test_libs/pyspec/eth2spec/debug/decode.py @@ -36,5 +36,4 @@ def decode(data, typ): hash_tree_root(ret, typ).hex()) return ret else: - print(data, typ) - raise Exception("Type not recognized") + raise Exception(f"Type not recognized: data={data}, typ={typ}") diff --git a/test_libs/pyspec/eth2spec/debug/encode.py b/test_libs/pyspec/eth2spec/debug/encode.py index 8be567f172..61dd87928b 100644 --- a/test_libs/pyspec/eth2spec/debug/encode.py +++ b/test_libs/pyspec/eth2spec/debug/encode.py @@ -33,5 +33,4 @@ def encode(value, typ, include_hash_tree_roots=False): ret["hash_tree_root"] = '0x' + hash_tree_root(value, typ).hex() return ret else: - print(value, typ) - raise Exception("Type not recognized") + raise Exception(f"Type not recognized: value={value}, typ={typ}") diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 43c3de3496..189c1268a0 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -56,65 +56,69 @@ def get_random_ssz_object(rng: Random, """ if chaos: mode = rng.choice(list(RandomizationMode)) - # Bytes array if is_bytes_type(typ): + # Bytes array if mode == RandomizationMode.mode_nil_count: return b'' - if mode == RandomizationMode.mode_max_count: + elif mode == RandomizationMode.mode_max_count: return get_random_bytes_list(rng, max_bytes_length) - if mode == RandomizationMode.mode_one_count: + elif mode == RandomizationMode.mode_one_count: return get_random_bytes_list(rng, 1) - if mode == RandomizationMode.mode_zero: + elif mode == RandomizationMode.mode_zero: return b'\x00' - if mode == RandomizationMode.mode_max: + elif mode == RandomizationMode.mode_max: return b'\xff' - return get_random_bytes_list(rng, rng.randint(0, max_bytes_length)) + else: + return get_random_bytes_list(rng, rng.randint(0, max_bytes_length)) elif is_bytesn_type(typ): + # BytesN length = typ.length # Sanity, don't generate absurdly big random values # If a client is aiming to performance-test, they should create a benchmark suite. assert length <= max_bytes_length if mode == RandomizationMode.mode_zero: return b'\x00' * length - if mode == RandomizationMode.mode_max: + elif mode == RandomizationMode.mode_max: return b'\xff' * length - return get_random_bytes_list(rng, length) - # Basic types + else: + return get_random_bytes_list(rng, length) elif is_basic_type(typ): + # Basic types if mode == RandomizationMode.mode_zero: return get_min_basic_value(typ) - if mode == RandomizationMode.mode_max: + elif mode == RandomizationMode.mode_max: return get_max_basic_value(typ) - return get_random_basic_value(rng, typ) - # Vector: + else: + return get_random_basic_value(rng, typ) elif is_vector_type(typ): + # Vector elem_typ = read_vector_elem_type(typ) return [ get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) for _ in range(typ.length) ] - # List: elif is_list_type(typ): + # List elem_typ = read_list_elem_type(typ) length = rng.randint(0, max_list_length) if mode == RandomizationMode.mode_one_count: length = 1 - if mode == RandomizationMode.mode_max_count: + elif mode == RandomizationMode.mode_max_count: length = max_list_length + return [ get_random_ssz_object(rng, elem_typ, max_bytes_length, max_list_length, mode, chaos) for _ in range(length) ] - # Container: elif is_container_type(typ): + # Container return typ(**{ field: get_random_ssz_object(rng, subtype, max_bytes_length, max_list_length, mode, chaos) for field, subtype in typ.get_fields() }) else: - print(typ) - raise Exception("Type not recognized") + raise Exception(f"Type not recognized: typ={typ}") def get_random_bytes_list(rng: Random, length: int) -> bytes: @@ -129,7 +133,7 @@ def get_random_basic_value(rng: Random, typ) -> Any: assert size in UINT_SIZES return rng.randint(0, 256**size - 1) else: - raise ValueError("Not a basic type") + raise ValueError(f"Not a basic type: typ={typ}") def get_min_basic_value(typ) -> Any: @@ -140,7 +144,7 @@ def get_min_basic_value(typ) -> Any: assert size in UINT_SIZES return 0 else: - raise ValueError("Not a basic type") + raise ValueError(f"Not a basic type: typ={typ}") def get_max_basic_value(typ) -> Any: @@ -151,4 +155,4 @@ def get_max_basic_value(typ) -> Any: assert size in UINT_SIZES return 256**size - 1 else: - raise ValueError("Not a basic type") + raise ValueError(f"Not a basic type: typ={typ}") diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 7f20284a8f..fcc3fab82b 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -83,13 +83,15 @@ def is_uint_type(typ): def uint_byte_size(typ): if hasattr(typ, '__supertype__'): typ = typ.__supertype__ + if isinstance(typ, type): if issubclass(typ, uint): return typ.byte_len elif issubclass(typ, int): # Default to uint64 return 8 - raise TypeError("Type %s is not an uint (or int-default uint64) type" % typ) + else: + raise TypeError("Type %s is not an uint (or int-default uint64) type" % typ) # SSZ Container base class @@ -167,32 +169,34 @@ def _is_vector_instance_of(a, b): # Other must not be a BytesN if issubclass(b, bytes): return False - if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): + elif not hasattr(b, 'elem_type') or not hasattr(b, 'length'): # Vector (b) is not an instance of Vector[X, Y] (a) return False - if not hasattr(a, 'elem_type') or not hasattr(a, 'length'): + elif not hasattr(a, 'elem_type') or not hasattr(a, 'length'): # Vector[X, Y] (b) is an instance of Vector (a) return True - - # Vector[X, Y] (a) is an instance of Vector[X, Y] (b) - return a.elem_type == b.elem_type and a.length == b.length + else: + # Vector[X, Y] (a) is an instance of Vector[X, Y] (b) + return a.elem_type == b.elem_type and a.length == b.length def _is_equal_vector_type(a, b): # Other must not be a BytesN if issubclass(b, bytes): return False - if not hasattr(a, 'elem_type') or not hasattr(a, 'length'): + elif not hasattr(a, 'elem_type') or not hasattr(a, 'length'): if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): # Vector == Vector return True - # Vector != Vector[X, Y] - return False - if not hasattr(b, 'elem_type') or not hasattr(b, 'length'): + else: + # Vector != Vector[X, Y] + return False + elif not hasattr(b, 'elem_type') or not hasattr(b, 'length'): # Vector[X, Y] != Vector return False - # Vector[X, Y] == Vector[X, Y] - return a.elem_type == b.elem_type and a.length == b.length + else: + # Vector[X, Y] == Vector[X, Y] + return a.elem_type == b.elem_type and a.length == b.length class VectorMeta(type): @@ -233,7 +237,7 @@ def __init__(self, *args: Iterable): cls = self.__class__ if not hasattr(cls, 'elem_type'): raise TypeError("Type Vector without elem_type data cannot be instantiated") - if not hasattr(cls, 'length'): + elif not hasattr(cls, 'length'): raise TypeError("Type Vector without length data cannot be instantiated") if len(args) != cls.length: @@ -282,32 +286,34 @@ def _is_bytes_n_instance_of(a, b): # Other has to be a Bytes derivative class to be a BytesN if not issubclass(b, bytes): return False - if not hasattr(b, 'length'): + elif not hasattr(b, 'length'): # BytesN (b) is not an instance of BytesN[X] (a) return False - if not hasattr(a, 'length'): + elif not hasattr(a, 'length'): # BytesN[X] (b) is an instance of BytesN (a) return True - - # BytesN[X] (a) is an instance of BytesN[X] (b) - return a.length == b.length + else: + # BytesN[X] (a) is an instance of BytesN[X] (b) + return a.length == b.length def _is_equal_bytes_n_type(a, b): # Other has to be a Bytes derivative class to be a BytesN if not issubclass(b, bytes): return False - if not hasattr(a, 'length'): + elif not hasattr(a, 'length'): if not hasattr(b, 'length'): # BytesN == BytesN return True - # BytesN != BytesN[X] - return False - if not hasattr(b, 'length'): + else: + # BytesN != BytesN[X] + return False + elif not hasattr(b, 'length'): # BytesN[X] != BytesN return False - # BytesN[X] == BytesN[X] - return a.length == b.length + else: + # BytesN[X] == BytesN[X] + return a.length == b.length class BytesNMeta(type): @@ -341,14 +347,15 @@ def __hash__(self): def parse_bytes(val): if val is None: return None - if isinstance(val, str): + elif isinstance(val, str): # TODO: import from eth-utils instead, and do: hexstr_if_str(to_bytes, val) return None - if isinstance(val, bytes): + elif isinstance(val, bytes): return val - if isinstance(val, int): + elif isinstance(val, int): return bytes([val]) - return None + else: + return None class BytesN(bytes, metaclass=BytesNMeta): @@ -433,6 +440,9 @@ def infer_helper(obj, typ=None, **kwargs): def is_bool_type(typ): + """ + Checks if the given type is a bool. + """ if hasattr(typ, '__supertype__'): typ = typ.__supertype__ return isinstance(typ, type) and issubclass(typ, bool) @@ -446,36 +456,45 @@ def is_list_type(typ): def is_bytes_type(typ): + """ + Check if the given type is a ``bytes``. + """ # Do not accept subclasses of bytes here, to avoid confusion with BytesN return typ == bytes +def is_bytesn_type(typ): + """ + Check if the given type is a BytesN. + """ + return isinstance(typ, type) and issubclass(typ, BytesN) + + def is_list_kind(typ): """ - Checks if the given type is a kind of list. Can be bytes. + Check if the given type is a kind of list. Can be bytes. """ return is_list_type(typ) or is_bytes_type(typ) def is_vector_type(typ): """ - Checks if the given type is a vector. + Check if the given type is a vector. """ return isinstance(typ, type) and issubclass(typ, Vector) -def is_bytesn_type(typ): - return isinstance(typ, type) and issubclass(typ, BytesN) - - def is_vector_kind(typ): """ - Checks if the given type is a kind of vector. Can be BytesN. + Check if the given type is a kind of vector. Can be BytesN. """ return is_vector_type(typ) or is_bytesn_type(typ) def is_container_type(typ): + """ + Check if the given type is a container. + """ return isinstance(typ, type) and issubclass(typ, Container) From e2eab66a9e1fb0a1cfa8afbc9ff9aaf0a06489d5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 4 Jun 2019 17:55:05 +0800 Subject: [PATCH 257/308] Refactor --- test_libs/pyspec/eth2spec/debug/random_value.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test_libs/pyspec/eth2spec/debug/random_value.py b/test_libs/pyspec/eth2spec/debug/random_value.py index 189c1268a0..3edcc88084 100644 --- a/test_libs/pyspec/eth2spec/debug/random_value.py +++ b/test_libs/pyspec/eth2spec/debug/random_value.py @@ -12,9 +12,9 @@ ) # in bytes -UINT_SIZES = [1, 2, 4, 8, 16, 32] +UINT_SIZES = (1, 2, 4, 8, 16, 32) -random_mode_names = ["random", "zero", "max", "nil", "one", "lengthy"] +random_mode_names = ("random", "zero", "max", "nil", "one", "lengthy") class RandomizationMode(Enum): @@ -128,7 +128,7 @@ def get_random_bytes_list(rng: Random, length: int) -> bytes: def get_random_basic_value(rng: Random, typ) -> Any: if is_bool_type(typ): return rng.choice((True, False)) - if is_uint_type(typ): + elif is_uint_type(typ): size = uint_byte_size(typ) assert size in UINT_SIZES return rng.randint(0, 256**size - 1) @@ -139,7 +139,7 @@ def get_random_basic_value(rng: Random, typ) -> Any: def get_min_basic_value(typ) -> Any: if is_bool_type(typ): return False - if is_uint_type(typ): + elif is_uint_type(typ): size = uint_byte_size(typ) assert size in UINT_SIZES return 0 @@ -150,7 +150,7 @@ def get_min_basic_value(typ) -> Any: def get_max_basic_value(typ) -> Any: if is_bool_type(typ): return True - if is_uint_type(typ): + elif is_uint_type(typ): size = uint_byte_size(typ) assert size in UINT_SIZES return 256**size - 1 From 6d55ba9c8c6e2b497722f6388f4037fea0f127eb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 4 Jun 2019 18:59:52 +0800 Subject: [PATCH 258/308] minor refactor --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index fcc3fab82b..5bd6c0a240 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -233,7 +233,6 @@ def __hash__(self): class Vector(metaclass=VectorMeta): def __init__(self, *args: Iterable): - cls = self.__class__ if not hasattr(cls, 'elem_type'): raise TypeError("Type Vector without elem_type data cannot be instantiated") @@ -282,6 +281,10 @@ def __eq__(self, other): return self.hash_tree_root() == other.hash_tree_root() +# SSZ BytesN +# ----------------------------- + + def _is_bytes_n_instance_of(a, b): # Other has to be a Bytes derivative class to be a BytesN if not issubclass(b, bytes): @@ -441,7 +444,7 @@ def infer_helper(obj, typ=None, **kwargs): def is_bool_type(typ): """ - Checks if the given type is a bool. + Check if the given type is a bool. """ if hasattr(typ, '__supertype__'): typ = typ.__supertype__ @@ -450,7 +453,7 @@ def is_bool_type(typ): def is_list_type(typ): """ - Checks if the given type is a list. + Check if the given type is a list. """ return get_origin(typ) is List or get_origin(typ) is list From 8631cad251e0ce84e400a737c937a48c914228fd Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Tue, 4 Jun 2019 15:22:34 +0200 Subject: [PATCH 259/308] Apply suggestions from code review Co-Authored-By: Danny Ryan --- test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py index 9c1fcdc611..ea4313c67d 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_impl.py @@ -109,6 +109,7 @@ def pack(values, subtype): def chunkify(bytez): + # pad `bytez` to nearest 32-byte multiple bytez += b'\x00' * (-len(bytez) % 32) return [bytez[i:i + 32] for i in range(0, len(bytez), 32)] From fe9c708d8345611b495d61552f6095e168d88952 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Tue, 4 Jun 2019 15:31:20 +0200 Subject: [PATCH 260/308] Fix whitespace Co-Authored-By: Danny Ryan --- specs/core/0_beacon-chain.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index bc0b79f47b..06c1c302d0 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -532,15 +532,12 @@ class BeaconState(Container): slot: uint64 genesis_time: uint64 fork: Fork # For versioning hard forks - # Validator registry validator_registry: List[Validator] balances: List[uint64] - # Randomness and committees latest_randao_mixes: Vector[Bytes32, LATEST_RANDAO_MIXES_LENGTH] latest_start_shard: uint64 - # Finality previous_epoch_attestations: List[PendingAttestation] current_epoch_attestations: List[PendingAttestation] @@ -551,7 +548,6 @@ class BeaconState(Container): justification_bitfield: uint64 finalized_epoch: uint64 finalized_root: Bytes32 - # Recent state current_crosslinks: Vector[Crosslink, SHARD_COUNT] previous_crosslinks: Vector[Crosslink, SHARD_COUNT] @@ -561,7 +557,6 @@ class BeaconState(Container): latest_slashed_balances: Vector[uint64, LATEST_SLASHED_EXIT_LENGTH] latest_block_header: BeaconBlockHeader historical_roots: List[Bytes32] - # Ethereum 1.0 chain data latest_eth1_data: Eth1Data eth1_data_votes: List[Eth1Data] From c06a60c95a89359d02dbc0979c75d8a9c9db88ae Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 4 Jun 2019 15:37:08 +0200 Subject: [PATCH 261/308] Fix descriptions Co-Authored-By: Hsiao-Wei Wang --- scripts/README.md | 4 ++-- test_libs/pyspec/eth2spec/test/context.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index d6ce4c1bdf..b47c238cef 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,8 +1,8 @@ # Building pyspecs from specs.md -The benefits of the particular spec design is that a given `spec.md` file can be converted to a `spec.py` file for the purposes of testing and linting. The result of this is that bugs are discovered and patched more quickly. +The benefit of the particular spec design is that the given markdown files can be converted to a `spec.py` file for the purposes of testing and linting. The result of this is that bugs are discovered and patched more quickly. -Specs can bue built from either a single markdown document or multiple files that must be combined in a given order. Given 2 spec objects, `build_spec.combine_spec_objects` will combine them into a single spec object which, subsequently, can be converted into a `specs.py`. +Specs can be built from either a single markdown document or multiple files that must be combined in a given order. Given 2 spec objects, `build_spec.combine_spec_objects` will combine them into a single spec object which, subsequently, can be converted into a `specs.py`. ## Usage diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 670654a92f..046a71caab 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -92,7 +92,7 @@ def entry(*args, **kw): def with_phase0(fn): """ - Decorator to use phase 0's spec and helpers + Decorator to use phase 0's spec. """ def entry(*args, **kw): kw['spec'] = spec_phase0 @@ -102,7 +102,7 @@ def entry(*args, **kw): def with_phase1(fn): """ - Decorator to use phase 1's spec and helpers + Decorator to use phase 1's spec """ def entry(*args, **kw): kw['spec'] = spec_phase1 From 59137fd5a6cc629733be6ae0b59e962418fd124c Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 4 Jun 2019 15:40:15 +0200 Subject: [PATCH 262/308] fix get_zero_value exception raise + fix up type checks --- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 5bd6c0a240..1e05ccd5d3 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -394,24 +394,22 @@ def hash_tree_root(self): # SSZ Defaults # ----------------------------- def get_zero_value(typ): - result = None if is_uint_type(typ): - result = 0 + return 0 elif is_list_type(typ): - result = [] - elif issubclass(typ, bool): - result = False - elif issubclass(typ, Vector): - result = typ() - elif issubclass(typ, BytesN): - result = typ() - elif issubclass(typ, bytes): - result = b'' - elif issubclass(typ, Container): - result = typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) + return [] + elif is_bool_type(typ): + return False + elif is_vector_type(typ): + return typ() + elif is_bytesn_type(typ): + return typ() + elif is_bytes_type(typ): + return b'' + elif is_container_type(typ): + return typ(**{f: get_zero_value(t) for f, t in typ.get_fields()}) else: - return Exception("Type not supported: {}".format(typ)) - return result + raise Exception("Type not supported: {}".format(typ)) # Type helpers @@ -522,7 +520,7 @@ def read_elem_type(typ): return read_list_elem_type(typ) elif is_vector_type(typ): return read_vector_elem_type(typ) - elif issubclass(typ, bytes): + elif issubclass(typ, bytes): # bytes or bytesN return byte else: raise TypeError("Unexpected type: {}".format(typ)) From 9d00a76493a386b79d20f1aca8fcf1a0c39b4372 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 4 Jun 2019 16:14:14 +0200 Subject: [PATCH 263/308] Skips tests properly --- ...est_process_early_derived_secret_reveal.py | 16 ++-- .../eth2spec/test/sanity/test_blocks.py | 88 ++++++++++--------- 2 files changed, 55 insertions(+), 49 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py index 4b7de56c86..c28c2e859e 100644 --- a/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py @@ -1,3 +1,5 @@ +import pytest + from eth2spec.test.helpers.custody import get_valid_early_derived_secret_reveal from eth2spec.test.context import with_phase1, spec_state_test, expect_assertion_error @@ -51,12 +53,14 @@ def test_reveal_from_current_epoch(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -# @with_phase1 -# @spec_state_test -# def test_reveal_from_past_epoch(state): -# randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state) - 1) -# -# yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) +@with_phase1 +@spec_state_test +def test_reveal_from_past_epoch(spec, state): + if spec.get_current_epoch(state) < 1: + pytest.skip('testing of previous epoch requires epoch of at least 1') + randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state) - 1) + + yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) @with_phase1 diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 0544586321..63da6aa37e 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -72,24 +72,25 @@ def test_empty_epoch_transition(spec, state): assert spec.get_block_root_at_slot(state, slot) == block.parent_root -# @spec_state_test -# def test_empty_epoch_transition_not_finalizing(spec, state): -# # copy for later balance lookups. -# pre_state = deepcopy(state) -# yield 'pre', state -# -# block = build_empty_block_for_next_slot(spec, state) -# block.slot += spec.SLOTS_PER_EPOCH * 5 -# sign_block(spec, state, block, proposer_index=0) -# yield 'blocks', [block], [spec.BeaconBlock] -# -# spec.state_transition(state, block) -# yield 'post', state -# -# assert state.slot == block.slot -# assert state.finalized_epoch < spec.get_current_epoch(state) - 4 -# for index in range(len(state.validator_registry)): -# assert get_balance(state, index) < get_balance(pre_state, index) +@with_all_phases +@spec_state_test +def test_empty_epoch_transition_not_finalizing(spec, state): + # copy for later balance lookups. + pre_state = deepcopy(state) + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + block.slot += spec.SLOTS_PER_EPOCH * 5 + sign_block(spec, state, block, proposer_index=0) + yield 'blocks', [block], [spec.BeaconBlock] + + spec.state_transition(state, block) + yield 'post', state + + assert state.slot == block.slot + assert state.finalized_epoch < spec.get_current_epoch(state) - 4 + for index in range(len(state.validator_registry)): + assert get_balance(state, index) < get_balance(pre_state, index) @with_all_phases @@ -378,28 +379,29 @@ def test_historical_batch(spec, state): assert len(state.historical_roots) == pre_historical_roots_len + 1 -# @spec_state_test -# def test_eth1_data_votes(spec, state): -# yield 'pre', state -# -# expected_votes = 0 -# assert len(state.eth1_data_votes) == expected_votes -# -# blocks = [] -# for _ in range(spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1): -# block = build_empty_block_for_next_slot(spec, state) -# spec.state_transition(state, block) -# expected_votes += 1 -# assert len(state.eth1_data_votes) == expected_votes -# blocks.append(block) -# -# block = build_empty_block_for_next_slot(spec, state) -# blocks.append(block) -# -# spec.state_transition(state, block) -# -# yield 'blocks', [block], [spec.BeaconBlock] -# yield 'post', state -# -# assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 -# assert len(state.eth1_data_votes) == 1 +@with_all_phases +@spec_state_test +def test_eth1_data_votes(spec, state): + yield 'pre', state + + expected_votes = 0 + assert len(state.eth1_data_votes) == expected_votes + + blocks = [] + for _ in range(spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1): + block = build_empty_block_for_next_slot(spec, state) + spec.state_transition(state, block) + expected_votes += 1 + assert len(state.eth1_data_votes) == expected_votes + blocks.append(block) + + block = build_empty_block_for_next_slot(spec, state) + blocks.append(block) + + spec.state_transition(state, block) + + yield 'blocks', [block], [spec.BeaconBlock] + yield 'post', state + + assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 + assert len(state.eth1_data_votes) == 1 From f6bb47c402b5ec9f9db3324e7387b243dcf7e834 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 4 Jun 2019 17:08:18 +0200 Subject: [PATCH 264/308] Apply @dankrad's suggestions from code review --- scripts/README.md | 2 +- scripts/build_spec.py | 2 +- test_libs/pyspec/eth2spec/utils/minimal_ssz.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index b47c238cef..25b46decfe 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -10,7 +10,7 @@ For usage of the spec builder run `python3 -m build_spec --help`. ## `@Labels` and inserts -The functioning of the spec combiner is largely automatic in that given `spec0.md` and `spec1.md`, in that SSZ Objects will be extended and old functions will be overwritten. Extra functionality is provided for more granular control over how files are combined. In the event that only a small portion of code is to be added to an existing function, insert functionality is provided. This saves having to completely redefine the old function from `spec0.md` in `spec1.md`. This is done by marking where the change is to occur in the old file and marking which code is to be inserted in the new file. This is done as follows: +The functioning of the spec combiner is largely automatic in that given `spec0.md` and `spec1.md`, SSZ Objects will be extended and old functions will be overwritten. Extra functionality is provided for more granular control over how files are combined. In the event that only a small portion of code is to be added to an existing function, insert functionality is provided. This saves having to completely redefine the old function from `spec0.md` in `spec1.md`. This is done by marking where the change is to occur in the old file and marking which code is to be inserted in the new file. This is done as follows: * In the old file, a label is added as a python comment marking where the code is to be inserted. This would appear as follows in `spec0.md`: diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 4ba9b616ab..c6bb55292f 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -166,7 +166,7 @@ def combine_constants(old_constants: Dict[str, str], new_constants: Dict[str, st return old_constants -def dependency_order_ssz_objects(objects: Dict[str, str]) -> Dict[str, str]: +def dependency_order_ssz_objects(objects: Dict[str, str]) -> None: """ Determines which SSZ Object is depenedent on which other and orders them appropriately """ diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py index f46c0acef0..9a555af66d 100644 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py @@ -56,8 +56,7 @@ def __len__(self): return self.length -def type_of(obj): - return obj.__class__ +type_of = type def empty(obj): From 578328bb54f0ccd410e763d29a9a45f4294b2c2a Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 4 Jun 2019 18:11:50 +0200 Subject: [PATCH 265/308] call comments: remove partials from ssz rework PR, review that seperately --- .../pyspec/eth2spec/utils/ssz/ssz_partials.py | 306 ------------------ 1 file changed, 306 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py deleted file mode 100644 index 6723674ebf..0000000000 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_partials.py +++ /dev/null @@ -1,306 +0,0 @@ -from ..merkle_minimal import hash, next_power_of_two -from .ssz_typing import ( - Container, - infer_input_type, - is_bool_type, - is_bytes_type, - is_bytesn_type, - is_container_type, - is_list_kind, - is_list_type, - is_uint_type, - is_vector_kind, - is_vector_type, - read_elem_type, - uint_byte_size, -) -from .ssz_impl import ( - chunkify, - deserialize_basic, - get_typed_values, - is_basic_type, - is_bottom_layer_kind, - pack, - serialize_basic, -) - - -ZERO_CHUNK = b'\x00' * 32 - - -def last_power_of_two(x): - return next_power_of_two(x + 1) // 2 - - -def concat_generalized_indices(x, y): - return x * last_power_of_two(y) + y - last_power_of_two(y) - - -def rebase(objs, new_root): - return {concat_generalized_indices(new_root, k): v for k, v in objs.items()} - - -def constrict_generalized_index(x, q): - depth = last_power_of_two(x // q) - o = depth + x - q * depth - if concat_generalized_indices(q, o) != x: - return None - return o - - -def unrebase(objs, q): - o = {} - for k, v in objs.items(): - new_k = constrict_generalized_index(k, q) - if new_k is not None: - o[new_k] = v - return o - - -def filler(starting_position, chunk_count): - at, skip, end = chunk_count, 1, next_power_of_two(chunk_count) - value = ZERO_CHUNK - o = {} - while at < end: - while at % (skip * 2) == 0: - skip *= 2 - value = hash(value + value) - o[(starting_position + at) // skip] = value - at += skip - return o - - -def merkle_tree_of_chunks(chunks, root): - starting_index = root * next_power_of_two(len(chunks)) - o = {starting_index + i: chunk for i, chunk in enumerate(chunks)} - o = {**o, **filler(starting_index, len(chunks))} - return o - - -@infer_input_type -def ssz_leaves(obj, typ=None, root=1): - if is_list_kind(typ): - o = {root * 2 + 1: len(obj).to_bytes(32, 'little')} - base = root * 2 - else: - o = {} - base = root - if is_bottom_layer_kind(typ): - data = serialize_basic(obj, typ) if is_basic_type(typ) else pack(obj, read_elem_type(typ)) - q = {**o, **merkle_tree_of_chunks(chunkify(data), base)} - # print(obj, root, typ, base, list(q.keys())) - return(q) - else: - fields = get_typed_values(obj, typ=typ) - sub_base = base * next_power_of_two(len(fields)) - for i, (elem, elem_type) in enumerate(fields): - o = {**o, **ssz_leaves(elem, typ=elem_type, root=sub_base + i)} - q = {**o, **filler(sub_base, len(fields))} - # print(obj, root, typ, base, list(q.keys())) - return(q) - - -def fill(objects): - objects = {k: v for k, v in objects.items()} - keys = sorted(objects.keys())[::-1] - pos = 0 - while pos < len(keys): - k = keys[pos] - if k in objects and k ^ 1 in objects and k // 2 not in objects: - objects[k // 2] = hash(objects[k & - 2] + objects[k | 1]) - keys.append(k // 2) - pos += 1 - return objects - - -@infer_input_type -def ssz_full(obj, typ=None): - return fill(ssz_leaves(obj, typ=typ)) - - -def get_basic_type_size(typ): - if is_uint_type(typ): - return uint_byte_size(typ) - elif is_bool_type(typ): - return 1 - else: - raise Exception("Type not basic: {}".format(typ)) - - -def get_bottom_layer_element_position(typ, base, length, index): - """ - Returns the generalized index and the byte range of the index'th value - in the list with the given base generalized index and given length - """ - assert index < (1 if is_basic_type(typ) else length) - elem_typ = typ if is_basic_type(typ) else read_elem_type(typ) - elem_size = get_basic_type_size(elem_typ) - chunk_index = index * elem_size // 32 - chunk_count = (1 if is_basic_type(typ) else length) * elem_size // 32 - generalized_index = base * next_power_of_two(chunk_count) + chunk_index - start = elem_size * index % 32 - return generalized_index, start, start + elem_size - - -@infer_input_type -def get_generalized_indices(obj, path, typ=None, root=1): - if len(path) == 0: - return [root] if is_basic_type(typ) else list(ssz_leaves(obj, typ=typ, root=root).keys()) - if path[0] == '__len__': - return [root * 2 + 1] if is_list_type(typ) else [] - base = root * 2 if is_list_kind(typ) else root - if is_bottom_layer_kind(typ): - length = 1 if is_basic_type(typ) else len(obj) - index, _, _ = get_bottom_layer_element_position(typ, base, length, path[0]) - return [index] - else: - if is_container_type(typ): - fields = typ.get_field_names() - field_count, index = len(fields), fields.index(path[0]) - elem_type = typ.get_field_types()[index] - child = obj.get_field_values()[index] - else: - field_count, index, elem_type, child = len(obj), path[0], read_elem_type(typ), obj[path[0]] - return get_generalized_indices( - child, - path[1:], - typ=elem_type, - root=base * next_power_of_two(field_count) + index - ) - - -def get_branch_indices(tree_index): - o = [tree_index, tree_index ^ 1] - while o[-1] > 1: - o.append((o[-1] // 2) ^ 1) - return o[:-1] - - -def remove_redundant_indices(obj): - return {k: v for k, v in obj.items() if not (k * 2 in obj and k * 2 + 1 in obj)} - - -def merge(*args): - o = {} - for arg in args: - o = {**o, **arg} - return fill(o) - - -@infer_input_type -def get_nodes_along_path(obj, path, typ=None): - indices = get_generalized_indices(obj, path, typ=typ) - return remove_redundant_indices(merge( - *({i: obj.objects[i] for i in get_branch_indices(index)} for index in indices) - )) - - -class OutOfRangeException(Exception): - pass - - -class SSZPartial(): - def __init__(self, typ, objects, root=1): - assert not is_basic_type(typ) - self.objects = objects - self.typ = typ - self.root = root - if is_container_type(self.typ): - for field in self.typ.get_field_names(): - try: - setattr(self, field, self.getter(field)) - except OutOfRangeException: - pass - - def getter(self, index): - base = self.root * 2 if is_list_kind(self.typ) else self.root - if is_bottom_layer_kind(self.typ): - tree_index, start, end = get_bottom_layer_element_position( - self.typ, base, len(self), index - ) - if tree_index not in self.objects: - raise OutOfRangeException("Do not have required data") - else: - return deserialize_basic( - self.objects[tree_index][start:end], - self.typ if is_basic_type(self.typ) else read_elem_type(self.typ) - ) - else: - if is_container_type(self.typ): - fields = self.typ.get_field_names() - field_count, index = len(fields), fields.index(index) - elem_type = self.typ.get_field_types()[index] - else: - field_count, index, elem_type = len(self), index, read_elem_type(self.typ) - tree_index = base * next_power_of_two(field_count) + index - if tree_index not in self.objects: - raise OutOfRangeException("Do not have required data") - if is_basic_type(elem_type): - return deserialize_basic(self.objects[tree_index][:get_basic_type_size(elem_type)], elem_type) - else: - return ssz_partial(elem_type, self.objects, root=tree_index) - - def __getitem__(self, index): - return self.getter(index) - - def __iter__(self): - return (self[i] for i in range(len(self))) - - def __len__(self): - if is_list_kind(self.typ): - if self.root * 2 + 1 not in self.objects: - raise OutOfRangeException("Do not have required data: {}".format(self.root * 2 + 1)) - return int.from_bytes(self.objects[self.root * 2 + 1], 'little') - elif is_vector_kind(self.typ): - return self.typ.length - elif is_container_type(self.typ): - return len(self.typ.get_fields()) - else: - raise Exception("Unsupported type: {}".format(self.typ)) - - def full_value(self): - if is_bytes_type(self.typ) or is_bytesn_type(self.typ): - return bytes([self.getter(i) for i in range(len(self))]) - elif is_list_kind(self.typ): - return [self[i] for i in range(len(self))] - elif is_vector_kind(self.typ): - return self.typ(*(self[i] for i in range(len(self)))) - elif is_container_type(self.typ): - def full_value(x): - return x.full_value() if hasattr(x, 'full_value') else x - - return self.typ(**{field: full_value(self.getter(field)) for field in self.typ.get_field_names()}) - elif is_basic_type(self.typ): - return self.getter(0) - else: - raise Exception("Unsupported type: {}".format(self.typ)) - - def hash_tree_root(self): - o = {**self.objects} - keys = sorted(o.keys())[::-1] - pos = 0 - while pos < len(keys): - k = keys[pos] - if k in o and k ^ 1 in o and k // 2 not in o: - o[k // 2] = hash(o[k & - 2] + o[k | 1]) - keys.append(k // 2) - pos += 1 - return o[self.root] - - def __str__(self): - return str(self.full_value()) - - -def ssz_partial(typ, objects, root=1): - ssz_type = ( - Container if is_container_type(typ) else - typ if (is_vector_type(typ) or is_bytesn_type(typ)) else object - ) - - class Partial(SSZPartial, ssz_type): - pass - - if is_container_type(typ): - Partial.__annotations__ = typ.__annotations__ - o = Partial(typ, objects, root=root) - return o From 6168a90a20dcd567f27642fcb54af4b5485e1f1b Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 4 Jun 2019 18:12:23 +0200 Subject: [PATCH 266/308] speed and simplicity improvement for next_power_of_two function --- test_libs/pyspec/eth2spec/utils/merkle_minimal.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index 420f0b5f1f..3f1f130a4c 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -41,16 +41,7 @@ def next_power_of_two(v: int) -> int: """ if v == 0: return 1 - # effectively fill the bitstring (1 less, do not want to with ones, then increment for next power of 2. - v -= 1 - v |= v >> (1 << 0) - v |= v >> (1 << 1) - v |= v >> (1 << 2) - v |= v >> (1 << 3) - v |= v >> (1 << 4) - v |= v >> (1 << 5) - v += 1 - return v + return 1 << (v-1).bit_length() def merkleize_chunks(chunks): From 4bf3a26afc3e35910cb530c33914e8140773b7af Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 4 Jun 2019 18:18:18 +0200 Subject: [PATCH 267/308] fix formatting --- test_libs/pyspec/eth2spec/utils/merkle_minimal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py index 3f1f130a4c..c508f0df29 100644 --- a/test_libs/pyspec/eth2spec/utils/merkle_minimal.py +++ b/test_libs/pyspec/eth2spec/utils/merkle_minimal.py @@ -41,7 +41,7 @@ def next_power_of_two(v: int) -> int: """ if v == 0: return 1 - return 1 << (v-1).bit_length() + return 1 << (v - 1).bit_length() def merkleize_chunks(chunks): From 9bdb18245e23ea6af565c3cde6bf12b826adb144 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 4 Jun 2019 18:22:42 +0200 Subject: [PATCH 268/308] remove tautological type definition --- specs/core/0_beacon-chain.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 06c1c302d0..4940d35f4f 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -574,7 +574,6 @@ We define the following Python custom types for type hinting and readability: | `Shard` | `uint64` | a shard number | | `ValidatorIndex` | `uint64` | a validator registry index | | `Gwei` | `uint64` | an amount in Gwei | -| `Bytes32` | `Bytes32` | 32 bytes of binary data | | `BLSPubkey` | `Bytes48` | a BLS12-381 public key | | `BLSSignature` | `Bytes96` | a BLS12-381 signature | From e83500cef8ec9a936dd566f797af2f746358032b Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Wed, 5 Jun 2019 14:52:09 +0900 Subject: [PATCH 269/308] Reorganize data structures to mirror beacon state order --- specs/core/0_beacon-chain.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 4940d35f4f..6d18d18e62 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -23,16 +23,15 @@ - [Data structures](#data-structures) - [Misc dependencies](#misc-dependencies) - [`Fork`](#fork) - - [`Crosslink`](#crosslink) - - [`Eth1Data`](#eth1data) + - [`Validator`](#validator) - [`AttestationData`](#attestationdata) - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) - [`IndexedAttestation`](#indexedattestation) - - [`DepositData`](#depositdata) - - [`BeaconBlockHeader`](#beaconblockheader) - - [`Validator`](#validator) - [`PendingAttestation`](#pendingattestation) + - [`Crosslink`](#crosslink) + - [`Eth1Data`](#eth1data) - [`HistoricalBatch`](#historicalbatch) + - [`DepositData`](#depositdata) - [Beacon operations](#beacon-operations) - [`ProposerSlashing`](#proposerslashing) - [`AttesterSlashing`](#attesterslashing) From 65d2a502191f4a70edc3d0906f9cb4d5ae6c953f Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Wed, 5 Jun 2019 14:57:54 +0900 Subject: [PATCH 270/308] Change data structure to match beacon state order --- specs/core/0_beacon-chain.md | 118 +++++++++++++++++------------------ 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 6d18d18e62..e24176c499 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -274,31 +274,26 @@ class Fork(Container): epoch: uint64 ``` -#### `Crosslink` - -```python -class Crosslink(Container): - # Shard number - shard: uint64 - # Crosslinking data from epochs [start....end-1] - start_epoch: uint64 - end_epoch: uint64 - # Root of the previous crosslink - parent_root: Bytes32 - # Root of the crosslinked shard data since the previous crosslink - data_root: Bytes32 -``` - -#### `Eth1Data` +#### `Validator` ```python -class Eth1Data(Container): - # Root of the deposit tree - deposit_root: Bytes32 - # Total number of deposits - deposit_count: uint64 - # Block hash - block_hash: Bytes32 +class Validator(Container): + # BLS public key + pubkey: Bytes48 + # Withdrawal credentials + withdrawal_credentials: Bytes32 + # Epoch when became eligible for activation + activation_eligibility_epoch: uint64 + # Epoch when validator activated + activation_epoch: uint64 + # Epoch when validator exited + exit_epoch: uint64 + # Epoch when validator is eligible to withdraw + withdrawable_epoch: uint64 + # Was the validator slashed + slashed: bool + # Effective balance + effective_balance: uint64 ``` #### `AttestationData` @@ -341,20 +336,6 @@ class IndexedAttestation(Container): signature: Bytes96 ``` -#### `DepositData` - -```python -class DepositData(Container): - # BLS pubkey - pubkey: Bytes48 - # Withdrawal credentials - withdrawal_credentials: Bytes32 - # Amount in Gwei - amount: uint64 - # Container self-signature - signature: Bytes96 -``` - #### `BeaconBlockHeader` ```python @@ -366,28 +347,6 @@ class BeaconBlockHeader(Container): signature: Bytes96 ``` -#### `Validator` - -```python -class Validator(Container): - # BLS public key - pubkey: Bytes48 - # Withdrawal credentials - withdrawal_credentials: Bytes32 - # Epoch when became eligible for activation - activation_eligibility_epoch: uint64 - # Epoch when validator activated - activation_epoch: uint64 - # Epoch when validator exited - exit_epoch: uint64 - # Epoch when validator is eligible to withdraw - withdrawable_epoch: uint64 - # Was the validator slashed - slashed: bool - # Effective balance - effective_balance: uint64 -``` - #### `PendingAttestation` ```python @@ -402,6 +361,33 @@ class PendingAttestation(Container): proposer_index: uint64 ``` +#### `Crosslink` + +```python +class Crosslink(Container): + # Shard number + shard: uint64 + # Crosslinking data from epochs [start....end-1] + start_epoch: uint64 + end_epoch: uint64 + # Root of the previous crosslink + parent_root: Bytes32 + # Root of the crosslinked shard data since the previous crosslink + data_root: Bytes32 +``` + +#### `Eth1Data` + +```python +class Eth1Data(Container): + # Root of the deposit tree + deposit_root: Bytes32 + # Total number of deposits + deposit_count: uint64 + # Block hash + block_hash: Bytes32 +``` + #### `HistoricalBatch` ```python @@ -412,6 +398,20 @@ class HistoricalBatch(Container): state_roots: Vector[Bytes32, SLOTS_PER_HISTORICAL_ROOT] ``` +#### `DepositData` + +```python +class DepositData(Container): + # BLS pubkey + pubkey: Bytes48 + # Withdrawal credentials + withdrawal_credentials: Bytes32 + # Amount in Gwei + amount: uint64 + # Container self-signature + signature: Bytes96 +``` + ### Beacon operations #### `ProposerSlashing` From c250296d8ae2176ddf6297a47a2d86fffc62ba38 Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Wed, 5 Jun 2019 15:07:50 +0900 Subject: [PATCH 271/308] Move crosslink above attestation data --- specs/core/0_beacon-chain.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index e24176c499..58bfc8bf47 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -24,11 +24,11 @@ - [Misc dependencies](#misc-dependencies) - [`Fork`](#fork) - [`Validator`](#validator) + - [`Crosslink`](#crosslink) - [`AttestationData`](#attestationdata) - [`AttestationDataAndCustodyBit`](#attestationdataandcustodybit) - [`IndexedAttestation`](#indexedattestation) - [`PendingAttestation`](#pendingattestation) - - [`Crosslink`](#crosslink) - [`Eth1Data`](#eth1data) - [`HistoricalBatch`](#historicalbatch) - [`DepositData`](#depositdata) @@ -296,6 +296,21 @@ class Validator(Container): effective_balance: uint64 ``` +#### `Crosslink` + +```python +class Crosslink(Container): + # Shard number + shard: uint64 + # Crosslinking data from epochs [start....end-1] + start_epoch: uint64 + end_epoch: uint64 + # Root of the previous crosslink + parent_root: Bytes32 + # Root of the crosslinked shard data since the previous crosslink + data_root: Bytes32 +``` + #### `AttestationData` ```python @@ -361,21 +376,6 @@ class PendingAttestation(Container): proposer_index: uint64 ``` -#### `Crosslink` - -```python -class Crosslink(Container): - # Shard number - shard: uint64 - # Crosslinking data from epochs [start....end-1] - start_epoch: uint64 - end_epoch: uint64 - # Root of the previous crosslink - parent_root: Bytes32 - # Root of the crosslinked shard data since the previous crosslink - data_root: Bytes32 -``` - #### `Eth1Data` ```python From e5fb91c4a2ba2eaa24db1840bf4e85a079918a23 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Wed, 5 Jun 2019 16:23:44 +0200 Subject: [PATCH 272/308] Make test generators work with phase 1 execution --- test_generators/epoch_processing/main.py | 6 +- test_generators/operations/main.py | 6 +- test_generators/sanity/main.py | 6 +- test_generators/shuffling/main.py | 9 ++- .../eth2spec/test/sanity/test_blocks.py | 72 +++++++++---------- 5 files changed, 54 insertions(+), 45 deletions(-) diff --git a/test_generators/epoch_processing/main.py b/test_generators/epoch_processing/main.py index 8f067e4a35..2ce895fc5b 100644 --- a/test_generators/epoch_processing/main.py +++ b/test_generators/epoch_processing/main.py @@ -1,6 +1,7 @@ from typing import Callable, Iterable -from eth2spec.phase0 import spec +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.phase1 import spec as spec_phase1 from eth2spec.test.epoch_processing import ( test_process_crosslinks, test_process_registry_updates @@ -14,7 +15,8 @@ def create_suite(transition_name: str, config_name: str, get_cases: Callable[[], -> Callable[[str], gen_typing.TestSuiteOutput]: def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: presets = loader.load_presets(configs_path, config_name) - spec.apply_constants_preset(presets) + spec_phase0.apply_constants_preset(presets) + spec_phase1.apply_constants_preset(presets) return ("%s_%s" % (transition_name, config_name), transition_name, gen_suite.render_suite( title="%s epoch processing" % transition_name, diff --git a/test_generators/operations/main.py b/test_generators/operations/main.py index 0a487ae74f..82e05b3075 100644 --- a/test_generators/operations/main.py +++ b/test_generators/operations/main.py @@ -13,14 +13,16 @@ from gen_base import gen_runner, gen_suite, gen_typing from gen_from_tests.gen import generate_from_tests from preset_loader import loader -from eth2spec.phase0 import spec +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.phase1 import spec as spec_phase1 def create_suite(operation_name: str, config_name: str, get_cases: Callable[[], Iterable[gen_typing.TestCase]]) \ -> Callable[[str], gen_typing.TestSuiteOutput]: def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: presets = loader.load_presets(configs_path, config_name) - spec.apply_constants_preset(presets) + spec_phase0.apply_constants_preset(presets) + spec_phase1.apply_constants_preset(presets) return ("%s_%s" % (operation_name, config_name), operation_name, gen_suite.render_suite( title="%s operation" % operation_name, diff --git a/test_generators/sanity/main.py b/test_generators/sanity/main.py index bba6ed03df..a9c0fe1603 100644 --- a/test_generators/sanity/main.py +++ b/test_generators/sanity/main.py @@ -5,14 +5,16 @@ from gen_base import gen_runner, gen_suite, gen_typing from gen_from_tests.gen import generate_from_tests from preset_loader import loader -from eth2spec.phase0 import spec +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.phase1 import spec as spec_phase1 def create_suite(handler_name: str, config_name: str, get_cases: Callable[[], Iterable[gen_typing.TestCase]]) \ -> Callable[[str], gen_typing.TestSuiteOutput]: def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: presets = loader.load_presets(configs_path, config_name) - spec.apply_constants_preset(presets) + spec_phase0.apply_constants_preset(presets) + spec_phase1.apply_constants_preset(presets) return ("%sanity_s_%s" % (handler_name, config_name), handler_name, gen_suite.render_suite( title="sanity testing", diff --git a/test_generators/shuffling/main.py b/test_generators/shuffling/main.py index 7115971896..862c4d9107 100644 --- a/test_generators/shuffling/main.py +++ b/test_generators/shuffling/main.py @@ -1,4 +1,5 @@ -from eth2spec.phase0 import spec +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.phase1 import spec as spec_phase1 from eth_utils import ( to_dict, to_tuple ) @@ -22,7 +23,8 @@ def shuffling_test_cases(): def mini_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput: presets = loader.load_presets(configs_path, 'minimal') - spec.apply_constants_preset(presets) + spec_phase0.apply_constants_preset(presets) + spec_phase1.apply_constants_preset(presets) return ("shuffling_minimal", "core", gen_suite.render_suite( title="Swap-or-Not Shuffling tests with minimal config", @@ -37,7 +39,8 @@ def mini_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput: def full_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput: presets = loader.load_presets(configs_path, 'mainnet') - spec.apply_constants_preset(presets) + spec_phase0.apply_constants_preset(presets) + spec_phase1.apply_constants_preset(presets) return ("shuffling_full", "core", gen_suite.render_suite( title="Swap-or-Not Shuffling tests with mainnet config", diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index ece5f3ef50..587c377429 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -74,25 +74,25 @@ def test_empty_epoch_transition(spec, state): assert spec.get_block_root_at_slot(state, slot) == block.parent_root -@with_all_phases -@spec_state_test -def test_empty_epoch_transition_not_finalizing(spec, state): - # copy for later balance lookups. - pre_state = deepcopy(state) - yield 'pre', state +# @with_all_phases +# @spec_state_test +# def test_empty_epoch_transition_not_finalizing(spec, state): +# # copy for later balance lookups. +# pre_state = deepcopy(state) +# yield 'pre', state - block = build_empty_block_for_next_slot(spec, state) - block.slot += spec.SLOTS_PER_EPOCH * 5 - sign_block(spec, state, block, proposer_index=0) - yield 'blocks', [block], List[spec.BeaconBlock] +# block = build_empty_block_for_next_slot(spec, state) +# block.slot += spec.SLOTS_PER_EPOCH * 5 +# sign_block(spec, state, block, proposer_index=0) +# yield 'blocks', [block], List[spec.BeaconBlock] - spec.state_transition(state, block) - yield 'post', state +# spec.state_transition(state, block) +# yield 'post', state - assert state.slot == block.slot - assert state.finalized_epoch < spec.get_current_epoch(state) - 4 - for index in range(len(state.validator_registry)): - assert get_balance(state, index) < get_balance(pre_state, index) +# assert state.slot == block.slot +# assert state.finalized_epoch < spec.get_current_epoch(state) - 4 +# for index in range(len(state.validator_registry)): +# assert get_balance(state, index) < get_balance(pre_state, index) @with_all_phases @@ -381,29 +381,29 @@ def test_historical_batch(spec, state): assert len(state.historical_roots) == pre_historical_roots_len + 1 -@with_all_phases -@spec_state_test -def test_eth1_data_votes(spec, state): - yield 'pre', state +# @with_all_phases +# @spec_state_test +# def test_eth1_data_votes(spec, state): +# yield 'pre', state - expected_votes = 0 - assert len(state.eth1_data_votes) == expected_votes +# expected_votes = 0 +# assert len(state.eth1_data_votes) == expected_votes - blocks = [] - for _ in range(spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1): - block = build_empty_block_for_next_slot(spec, state) - spec.state_transition(state, block) - expected_votes += 1 - assert len(state.eth1_data_votes) == expected_votes - blocks.append(block) +# blocks = [] +# for _ in range(spec.SLOTS_PER_ETH1_VOTING_PERIOD - 1): +# block = build_empty_block_for_next_slot(spec, state) +# spec.state_transition(state, block) +# expected_votes += 1 +# assert len(state.eth1_data_votes) == expected_votes +# blocks.append(block) - block = build_empty_block_for_next_slot(spec, state) - blocks.append(block) +# block = build_empty_block_for_next_slot(spec, state) +# blocks.append(block) - spec.state_transition(state, block) +# spec.state_transition(state, block) - yield 'blocks', [block], List[spec.BeaconBlock] - yield 'post', state +# yield 'blocks', [block], List[spec.BeaconBlock] +# yield 'post', state - assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 - assert len(state.eth1_data_votes) == 1 +# assert state.slot % spec.SLOTS_PER_ETH1_VOTING_PERIOD == 0 +# assert len(state.eth1_data_votes) == 1 From 853c34eb60a707ede5339597af666f380f1218e4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Wed, 5 Jun 2019 09:50:15 -0600 Subject: [PATCH 273/308] add beaconblockheader back to toc --- specs/core/0_beacon-chain.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 58bfc8bf47..f5fc0c98e3 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -32,6 +32,7 @@ - [`Eth1Data`](#eth1data) - [`HistoricalBatch`](#historicalbatch) - [`DepositData`](#depositdata) + - [`BeaconBlockHeader`](#beaconblockheader) - [Beacon operations](#beacon-operations) - [`ProposerSlashing`](#proposerslashing) - [`AttesterSlashing`](#attesterslashing) @@ -351,17 +352,6 @@ class IndexedAttestation(Container): signature: Bytes96 ``` -#### `BeaconBlockHeader` - -```python -class BeaconBlockHeader(Container): - slot: uint64 - parent_root: Bytes32 - state_root: Bytes32 - body_root: Bytes32 - signature: Bytes96 -``` - #### `PendingAttestation` ```python @@ -412,6 +402,17 @@ class DepositData(Container): signature: Bytes96 ``` +#### `BeaconBlockHeader` + +```python +class BeaconBlockHeader(Container): + slot: uint64 + parent_root: Bytes32 + state_root: Bytes32 + body_root: Bytes32 + signature: Bytes96 +``` + ### Beacon operations #### `ProposerSlashing` From a7554d503c10d1bb423731486ae45c3d1258168a Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 Jun 2019 18:02:39 +0200 Subject: [PATCH 274/308] fix for typing check of vector elements with non-type element type (annotation) --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 1e05ccd5d3..8c2fc359c2 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -1,4 +1,3 @@ -from inspect import isclass from typing import List, Iterable, TypeVar, Type, NewType from typing import Union from typing_inspect import get_origin @@ -247,8 +246,8 @@ def __init__(self, *args: Iterable): self.items = list(args) - # cannot check non-class objects - if isclass(cls.elem_type): + # cannot check non-type objects + if isinstance(cls.elem_type, type): for i, item in enumerate(self.items): if not isinstance(item, cls.elem_type): raise TypeError("Typed vector cannot hold differently typed value" From 1cc7c7309d25bd9c26c09824ec9898e8ee22e5dd Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 Jun 2019 18:28:09 +0200 Subject: [PATCH 275/308] change to issubclass, hope parametrized external type is ok --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 8c2fc359c2..4944fb600c 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -249,7 +249,7 @@ def __init__(self, *args: Iterable): # cannot check non-type objects if isinstance(cls.elem_type, type): for i, item in enumerate(self.items): - if not isinstance(item, cls.elem_type): + if not issubclass(type(item), cls.elem_type): raise TypeError("Typed vector cannot hold differently typed value" " at index %d. Got type: %s, expected type: %s" % (i, type(item), cls.elem_type)) From b9abc5f2cf349cf0ec211458df36b86b92b42edf Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 5 Jun 2019 18:53:34 +0200 Subject: [PATCH 276/308] List[uint64] is not like a type but just for annotation, same for other Generics with __args__, Vector/BytesN work, because their metaclasses produce non-parametrized types, so don't check vector values when type contains args --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 4944fb600c..368041f90a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -246,8 +246,8 @@ def __init__(self, *args: Iterable): self.items = list(args) - # cannot check non-type objects - if isinstance(cls.elem_type, type): + # cannot check non-type objects, or parametrized types + if isinstance(cls.elem_type, type) and not hasattr(cls.elem_type, '__args__'): for i, item in enumerate(self.items): if not issubclass(type(item), cls.elem_type): raise TypeError("Typed vector cannot hold differently typed value" From eefd3062539e058519e19cc1c50fa97284665d77 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Wed, 5 Jun 2019 13:07:10 -0400 Subject: [PATCH 277/308] Update 0_beacon-chain-validator.md --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index b16e09feb0..2f5aa4264b 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -277,7 +277,7 @@ Up to `MAX_VOLUNTARY_EXITS` [`VoluntaryExit`](../core/0_beacon-chain.md#voluntar ### Attestations -A validator is expected to create, sign, and broadcast an attestation during each epoch. The committee, assigned shard, and assigned slot for which the validator performs this role during an epoch is defined by `get_committee_assignment(state, epoch, validator_index)`. +A validator is expected to create, sign, and broadcast an attestation during each epoch. The `committee`, assigned `shard`, and assigned `slot` for which the validator performs this role during an epoch is defined by `get_committee_assignment(state, epoch, validator_index)`. A validator should create and broadcast the attestation halfway through the `slot` during which the validator is assigned ― that is, `SECONDS_PER_SLOT * 0.5` seconds after the start of `slot`. From e498ff7e94ee47ea69d390860a20c8d2a7061ca3 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Wed, 5 Jun 2019 21:28:30 +0200 Subject: [PATCH 278/308] Separates tests into phases --- .../eth2spec/test/{block_processing => phase_0}/__init__.py | 0 .../{epoch_processing => phase_0/block_processing}/__init__.py | 0 .../{ => phase_0}/block_processing/test_process_attestation.py | 0 .../block_processing/test_process_attester_slashing.py | 0 .../{ => phase_0}/block_processing/test_process_block_header.py | 0 .../test/{ => phase_0}/block_processing/test_process_deposit.py | 0 .../block_processing/test_process_proposer_slashing.py | 0 .../test/{ => phase_0}/block_processing/test_process_transfer.py | 0 .../{ => phase_0}/block_processing/test_process_voluntary_exit.py | 0 .../pyspec/eth2spec/test/phase_0/epoch_processing/__init__.py | 0 .../{ => phase_0}/epoch_processing/test_process_crosslinks.py | 0 .../epoch_processing/test_process_registry_updates.py | 0 test_libs/pyspec/eth2spec/test/phase_1/__init__.py | 0 .../pyspec/eth2spec/test/phase_1/block_processing/__init__.py | 0 .../block_processing/test_process_early_derived_secret_reveal.py | 0 15 files changed, 0 insertions(+), 0 deletions(-) rename test_libs/pyspec/eth2spec/test/{block_processing => phase_0}/__init__.py (100%) rename test_libs/pyspec/eth2spec/test/{epoch_processing => phase_0/block_processing}/__init__.py (100%) rename test_libs/pyspec/eth2spec/test/{ => phase_0}/block_processing/test_process_attestation.py (100%) rename test_libs/pyspec/eth2spec/test/{ => phase_0}/block_processing/test_process_attester_slashing.py (100%) rename test_libs/pyspec/eth2spec/test/{ => phase_0}/block_processing/test_process_block_header.py (100%) rename test_libs/pyspec/eth2spec/test/{ => phase_0}/block_processing/test_process_deposit.py (100%) rename test_libs/pyspec/eth2spec/test/{ => phase_0}/block_processing/test_process_proposer_slashing.py (100%) rename test_libs/pyspec/eth2spec/test/{ => phase_0}/block_processing/test_process_transfer.py (100%) rename test_libs/pyspec/eth2spec/test/{ => phase_0}/block_processing/test_process_voluntary_exit.py (100%) create mode 100644 test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/__init__.py rename test_libs/pyspec/eth2spec/test/{ => phase_0}/epoch_processing/test_process_crosslinks.py (100%) rename test_libs/pyspec/eth2spec/test/{ => phase_0}/epoch_processing/test_process_registry_updates.py (100%) create mode 100644 test_libs/pyspec/eth2spec/test/phase_1/__init__.py create mode 100644 test_libs/pyspec/eth2spec/test/phase_1/block_processing/__init__.py rename test_libs/pyspec/eth2spec/test/{ => phase_1}/block_processing/test_process_early_derived_secret_reveal.py (100%) diff --git a/test_libs/pyspec/eth2spec/test/block_processing/__init__.py b/test_libs/pyspec/eth2spec/test/phase_0/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/block_processing/__init__.py rename to test_libs/pyspec/eth2spec/test/phase_0/__init__.py diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/__init__.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/__init__.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/epoch_processing/__init__.py rename to test_libs/pyspec/eth2spec/test/phase_0/block_processing/__init__.py diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/block_processing/test_process_attestation.py rename to test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/block_processing/test_process_attester_slashing.py rename to test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/block_processing/test_process_block_header.py rename to test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/block_processing/test_process_deposit.py rename to test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/block_processing/test_process_proposer_slashing.py rename to test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/block_processing/test_process_transfer.py rename to test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/block_processing/test_process_voluntary_exit.py rename to test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/__init__.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/epoch_processing/test_process_crosslinks.py rename to test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py diff --git a/test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/epoch_processing/test_process_registry_updates.py rename to test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py diff --git a/test_libs/pyspec/eth2spec/test/phase_1/__init__.py b/test_libs/pyspec/eth2spec/test/phase_1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/__init__.py b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py similarity index 100% rename from test_libs/pyspec/eth2spec/test/block_processing/test_process_early_derived_secret_reveal.py rename to test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py From 60d9dc68c4d95f28b50a1d144985b3a6cfdfd726 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Wed, 5 Jun 2019 21:42:55 +0200 Subject: [PATCH 279/308] Apply suggestions from @djrtwo's code review --- scripts/build_spec.py | 25 +- specs/core/0_beacon-chain.md | 2 + test_libs/pyspec/eth2spec/test/context.py | 4 +- .../test_process_attestation.py | 6 +- ...est_process_early_derived_secret_reveal.py | 8 +- .../pyspec/eth2spec/utils/minimal_ssz.py | 340 ------------------ 6 files changed, 26 insertions(+), 359 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/utils/minimal_ssz.py diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 50d5955eeb..7a51970e38 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -6,6 +6,7 @@ from argparse import ArgumentParser from typing import ( Dict, + List, Optional, ) @@ -122,13 +123,16 @@ def objects_to_spec(functions: Dict[str, str], constants: Dict[str, str], ssz_objects: Dict[str, str], inserts: Dict[str, str], - imports: Dict[str, str]) -> str: + imports: Dict[str, str], + new_types: Dict[str, str], + byte_types: List[int], + ) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. """ new_type_definitions = \ - '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in NEW_TYPES.items()]) - new_type_definitions += '\n' + '\n'.join(['Bytes%s = BytesN[%s]' % (n, n) for n in BYTE_TYPES]) + '\n'.join(['''%s = NewType('%s', %s)''' % (key, key, value) for key, value in new_types.items()]) + new_type_definitions += '\n' + '\n'.join(['Bytes%s = BytesN[%s]' % (n, n) for n in byte_types]) functions_spec = '\n\n'.join(functions.values()) constants_spec = '\n'.join(map(lambda x: '%s = %s' % (x, constants[x]), constants)) ssz_objects_instantiation_spec = '\n\n'.join(ssz_objects.values()) @@ -217,12 +221,11 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject: def build_phase0_spec(sourcefile: str, outfile: str=None) -> Optional[str]: functions, constants, ssz_objects, inserts = get_spec(sourcefile) - spec = objects_to_spec(functions, constants, ssz_objects, inserts, PHASE0_IMPORTS) + spec = objects_to_spec(functions, constants, ssz_objects, inserts, PHASE0_IMPORTS, NEW_TYPES, BYTE_TYPES) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) - else: - return spec + return spec def build_phase1_spec(phase0_sourcefile: str, @@ -235,12 +238,11 @@ def build_phase1_spec(phase0_sourcefile: str, spec_objects = phase0_spec for value in [phase1_custody, phase1_shard_data]: spec_objects = combine_spec_objects(spec_objects, value) - spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS) + spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS, NEW_TYPES, BYTE_TYPES) if outfile is not None: with open(outfile, 'w') as out: out.write(spec) - else: - return spec + return spec if __name__ == '__main__': @@ -262,7 +264,10 @@ def build_phase1_spec(phase0_sourcefile: str, args = parser.parse_args() if args.phase == 0: - build_phase0_spec(*args.files) + if len(args.files) == 2: + build_phase0_spec(*args.files) + else: + print(" Phase 0 requires an output as well as an input file.") elif args.phase == 1: if len(args.files) == 4: build_phase1_spec(*args.files) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index c58327d8d9..a6d9d23c58 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1246,6 +1246,8 @@ def process_slot(state: BeaconState) -> None: ### Epoch processing +Note: the `# @LabelHere` lines below are placeholders to show that code will be inserted here in a future phase. + ```python def process_epoch(state: BeaconState) -> None: process_justification_and_finalization(state) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 046a71caab..8bfce82436 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -112,11 +112,11 @@ def entry(*args, **kw): def with_all_phases(fn): """ - Decorator to run everything with all availible spec phases + Decorator to run everything with all available spec phases """ def entry(*args, **kw): kw['spec'] = spec_phase0 fn(*args, **kw) kw['spec'] = spec_phase1 - return fn(*args, **kw) + fn(*args, **kw) return entry diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 45cd604c5c..8e214fe7d0 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -1,6 +1,6 @@ from copy import deepcopy -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phase0 from eth2spec.test.helpers.attestations import ( get_valid_attestation, sign_attestation, @@ -214,7 +214,7 @@ def test_bad_source_root(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases +@with_phase0 @spec_state_test def test_non_zero_crosslink_data_root(spec, state): attestation = get_valid_attestation(spec, state) @@ -288,7 +288,7 @@ def test_inconsistent_bitfields(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases +@with_phase0 @spec_state_test def test_non_empty_custody_bitfield(spec, state): attestation = get_valid_attestation(spec, state) diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py index c28c2e859e..64440ef60a 100644 --- a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py @@ -1,6 +1,6 @@ -import pytest - from eth2spec.test.helpers.custody import get_valid_early_derived_secret_reveal +from eth2spec.test.helpers.block import apply_empty_block +from eth2spec.test.helpers.state import next_epoch from eth2spec.test.context import with_phase1, spec_state_test, expect_assertion_error @@ -56,8 +56,8 @@ def test_reveal_from_current_epoch(spec, state): @with_phase1 @spec_state_test def test_reveal_from_past_epoch(spec, state): - if spec.get_current_epoch(state) < 1: - pytest.skip('testing of previous epoch requires epoch of at least 1') + next_epoch(spec, state) + apply_empty_block(spec, state) randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state) - 1) yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) diff --git a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py b/test_libs/pyspec/eth2spec/utils/minimal_ssz.py deleted file mode 100644 index 9a555af66d..0000000000 --- a/test_libs/pyspec/eth2spec/utils/minimal_ssz.py +++ /dev/null @@ -1,340 +0,0 @@ -from typing import Any - -from .hash_function import hash - -BYTES_PER_CHUNK = 32 -BYTES_PER_LENGTH_OFFSET = 4 -ZERO_CHUNK = b'\x00' * BYTES_PER_CHUNK - - -def SSZType(fields): - class SSZObject(): - def __init__(self, **kwargs): - for f, t in fields.items(): - if f not in kwargs: - setattr(self, f, get_zero_value(t)) - else: - setattr(self, f, kwargs[f]) - - def __eq__(self, other): - return self.fields == other.fields and self.serialize() == other.serialize() - - def __hash__(self): - return int.from_bytes(self.hash_tree_root(), byteorder="little") - - def __str__(self): - output = [] - for field in self.fields: - output.append(f'{field}: {getattr(self, field)}') - return "\n".join(output) - - def serialize(self): - return serialize_value(self, self.__class__) - - def hash_tree_root(self): - return hash_tree_root(self, self.__class__) - - SSZObject.fields = fields - return SSZObject - - -class Vector(): - def __init__(self, items): - self.items = items - self.length = len(items) - - def __getitem__(self, key): - return self.items[key] - - def __setitem__(self, key, value): - self.items[key] = value - - def __iter__(self): - return iter(self.items) - - def __len__(self): - return self.length - - -type_of = type - - -def empty(obj): - for field in obj.fields: - field = get_zero_value(field) - return obj - - -def is_basic(typ): - # if not a string, it is a complex, and cannot be basic - if not isinstance(typ, str): - return False - # "uintN": N-bit unsigned integer (where N in [8, 16, 32, 64, 128, 256]) - elif typ[:4] == 'uint' and typ[4:] in ['8', '16', '32', '64', '128', '256']: - return True - # "bool": True or False - elif typ == 'bool': - return True - # alias: "byte" -> "uint8" - elif typ == 'byte': - return True - # default - else: - return False - - -def is_constant_sized(typ): - # basic objects are fixed size by definition - if is_basic(typ): - return True - # dynamic size array type, "list": [elem_type]. - # Not constant size by definition. - elif isinstance(typ, list) and len(typ) == 1: - return False - # fixed size array type, "vector": [elem_type, length] - # Constant size, but only if the elements are. - elif isinstance(typ, list) and len(typ) == 2: - return is_constant_sized(typ[0]) - # bytes array (fixed or dynamic size) - elif isinstance(typ, str) and typ[:5] == 'bytes': - # if no length suffix, it has a dynamic size - return typ != 'bytes' - # containers are only constant-size if all of the fields are constant size. - elif hasattr(typ, 'fields'): - for subtype in typ.fields.values(): - if not is_constant_sized(subtype): - return False - return True - else: - raise Exception("Type not recognized") - - -def coerce_to_bytes(x): - if isinstance(x, str): - o = x.encode('utf-8') - assert len(o) == len(x) - return o - elif isinstance(x, bytes): - return x - else: - raise Exception("Expecting bytes") - - -def encode_series(values, types): - # Recursively serialize - parts = [(is_constant_sized(types[i]), serialize_value(values[i], types[i])) for i in range(len(values))] - - # Compute and check lengths - fixed_lengths = [len(serialized) if constant_size else BYTES_PER_LENGTH_OFFSET - for (constant_size, serialized) in parts] - variable_lengths = [len(serialized) if not constant_size else 0 - for (constant_size, serialized) in parts] - - # Check if integer is not out of bounds (Python) - assert sum(fixed_lengths + variable_lengths) < 2 ** (BYTES_PER_LENGTH_OFFSET * 8) - - # Interleave offsets of variable-size parts with fixed-size parts. - # Avoid quadratic complexity in calculation of offsets. - offset = sum(fixed_lengths) - variable_parts = [] - fixed_parts = [] - for (constant_size, serialized) in parts: - if constant_size: - fixed_parts.append(serialized) - else: - fixed_parts.append(offset.to_bytes(BYTES_PER_LENGTH_OFFSET, 'little')) - variable_parts.append(serialized) - offset += len(serialized) - - # Return the concatenation of the fixed-size parts (offsets interleaved) with the variable-size parts - return b"".join(fixed_parts + variable_parts) - - -def serialize_value(value, typ=None): - if typ is None: - typ = infer_type(value) - # "uintN" - if isinstance(typ, str) and typ[:4] == 'uint': - length = int(typ[4:]) - assert length in (8, 16, 32, 64, 128, 256) - return value.to_bytes(length // 8, 'little') - # "bool" - elif isinstance(typ, str) and typ == 'bool': - assert value in (True, False) - return b'\x01' if value is True else b'\x00' - # Vector - elif isinstance(typ, list) and len(typ) == 2: - # (regardless of element type, sanity-check if the length reported in the vector type matches the value length) - assert len(value) == typ[1] - return encode_series(value, [typ[0]] * len(value)) - # List - elif isinstance(typ, list) and len(typ) == 1: - return encode_series(value, [typ[0]] * len(value)) - # "bytes" (variable size) - elif isinstance(typ, str) and typ == 'bytes': - return coerce_to_bytes(value) - # "bytesN" (fixed size) - elif isinstance(typ, str) and len(typ) > 5 and typ[:5] == 'bytes': - assert len(value) == int(typ[5:]), (value, int(typ[5:])) - return coerce_to_bytes(value) - # containers - elif hasattr(typ, 'fields'): - values = [getattr(value, field) for field in typ.fields.keys()] - types = list(typ.fields.values()) - return encode_series(values, types) - else: - print(value, typ) - raise Exception("Type not recognized") - - -def get_zero_value(typ: Any) -> Any: - if isinstance(typ, str): - # Bytes array - if typ == 'bytes': - return b'' - # bytesN - elif typ[:5] == 'bytes' and len(typ) > 5: - length = int(typ[5:]) - return b'\x00' * length - # Basic types - elif typ == 'bool': - return False - elif typ[:4] == 'uint': - return 0 - elif typ == 'byte': - return 0x00 - else: - raise ValueError("Type not recognized") - # Vector: - elif isinstance(typ, list) and len(typ) == 2: - return [get_zero_value(typ[0]) for _ in range(typ[1])] - # List: - elif isinstance(typ, list) and len(typ) == 1: - return [] - # Container: - elif hasattr(typ, 'fields'): - return typ(**{field: get_zero_value(subtype) for field, subtype in typ.fields.items()}) - else: - print(typ) - raise Exception("Type not recognized") - - -def chunkify(bytez): - bytez += b'\x00' * (-len(bytez) % BYTES_PER_CHUNK) - return [bytez[i:i + 32] for i in range(0, len(bytez), 32)] - - -def pack(values, subtype): - return chunkify(b''.join([serialize_value(value, subtype) for value in values])) - - -def is_power_of_two(x): - return x > 0 and x & (x - 1) == 0 - - -def merkleize(chunks): - tree = chunks[::] - while not is_power_of_two(len(tree)): - tree.append(ZERO_CHUNK) - tree = [ZERO_CHUNK] * len(tree) + tree - for i in range(len(tree) // 2 - 1, 0, -1): - tree[i] = hash(tree[i * 2] + tree[i * 2 + 1]) - return tree[1] - - -def mix_in_length(root, length): - return hash(root + length.to_bytes(32, 'little')) - - -def infer_type(value): - """ - Note: defaults to uint64 for integer type inference due to lack of information. - Other integer sizes are still supported, see spec. - :param value: The value to infer a SSZ type for. - :return: The SSZ type. - """ - if hasattr(value.__class__, 'fields'): - return value.__class__ - elif isinstance(value, Vector): - if len(value) > 0: - return [infer_type(value[0]), len(value)] - else: - # Element type does not matter too much, - # assumed to be a basic type for size-encoding purposes, vector is empty. - return ['uint64'] - elif isinstance(value, list): - if len(value) > 0: - return [infer_type(value[0])] - else: - # Element type does not matter, list-content size will be encoded regardless, list is empty. - return ['uint64'] - elif isinstance(value, (bytes, str)): - return 'bytes' - elif isinstance(value, int): - return 'uint64' - else: - raise Exception("Failed to infer type") - - -def hash_tree_root(value, typ=None): - if typ is None: - typ = infer_type(value) - # ------------------------------------- - # merkleize(pack(value)) - # basic object: merkleize packed version (merkleization pads it to 32 bytes if it is not already) - if is_basic(typ): - return merkleize(pack([value], typ)) - # or a vector of basic objects - elif isinstance(typ, list) and len(typ) == 2 and is_basic(typ[0]): - assert len(value) == typ[1] - return merkleize(pack(value, typ[0])) - # ------------------------------------- - # mix_in_length(merkleize(pack(value)), len(value)) - # if value is a list of basic objects - elif isinstance(typ, list) and len(typ) == 1 and is_basic(typ[0]): - return mix_in_length(merkleize(pack(value, typ[0])), len(value)) - # (needs some extra work for non-fixed-sized bytes array) - elif typ == 'bytes': - return mix_in_length(merkleize(chunkify(coerce_to_bytes(value))), len(value)) - # ------------------------------------- - # merkleize([hash_tree_root(element) for element in value]) - # if value is a vector of composite objects - elif isinstance(typ, list) and len(typ) == 2 and not is_basic(typ[0]): - return merkleize([hash_tree_root(element, typ[0]) for element in value]) - # (needs some extra work for fixed-sized bytes array) - elif isinstance(typ, str) and typ[:5] == 'bytes' and len(typ) > 5: - assert len(value) == int(typ[5:]) - return merkleize(chunkify(coerce_to_bytes(value))) - # or a container - elif hasattr(typ, 'fields'): - return merkleize([hash_tree_root(getattr(value, field), subtype) for field, subtype in typ.fields.items()]) - # ------------------------------------- - # mix_in_length(merkleize([hash_tree_root(element) for element in value]), len(value)) - # if value is a list of composite objects - elif isinstance(typ, list) and len(typ) == 1 and not is_basic(typ[0]): - return mix_in_length(merkleize([hash_tree_root(element, typ[0]) for element in value]), len(value)) - # ------------------------------------- - else: - raise Exception("Type not recognized") - - -def truncate(container): - field_keys = list(container.fields.keys()) - truncated_fields = { - key: container.fields[key] - for key in field_keys[:-1] - } - truncated_class = SSZType(truncated_fields) - kwargs = { - field: getattr(container, field) - for field in field_keys[:-1] - } - return truncated_class(**kwargs) - - -def signing_root(container): - return hash_tree_root(truncate(container)) - - -def serialize(ssz_object): - return getattr(ssz_object, 'serialize')() From 4c1b9ef6d6fc3b844eeb9f7c8bff8c7a614ba48d Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 6 Jun 2019 11:04:55 +0200 Subject: [PATCH 280/308] Fixes custody key reveal test bug --- .../test_process_early_derived_secret_reveal.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py index 64440ef60a..6c5785b2fb 100644 --- a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py @@ -1,6 +1,6 @@ from eth2spec.test.helpers.custody import get_valid_early_derived_secret_reveal from eth2spec.test.helpers.block import apply_empty_block -from eth2spec.test.helpers.state import next_epoch +from eth2spec.test.helpers.state import next_epoch, get_balance from eth2spec.test.context import with_phase1, spec_state_test, expect_assertion_error @@ -19,6 +19,8 @@ def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, v expect_assertion_error(lambda: spec.process_early_derived_secret_reveal(state, randao_key_reveal)) yield 'post', None return + + pre_slashed_balance = get_balance(state, randao_key_reveal.revealed_index) spec.process_early_derived_secret_reveal(state, randao_key_reveal) @@ -28,12 +30,8 @@ def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, v assert slashed_validator.slashed assert slashed_validator.exit_epoch < spec.FAR_FUTURE_EPOCH assert slashed_validator.withdrawable_epoch < spec.FAR_FUTURE_EPOCH - # lost whistleblower reward - # FIXME: Currently broken because get_base_reward in genesis epoch is 0 - # assert ( - # state.balances[randao_key_reveal.revealed_index] < - # state.balances[randao_key_reveal.revealed_index] - # ) + + assert get_balance(state, randao_key_reveal.revealed_index) < pre_slashed_balance yield 'post', state From 35c03c5f3e7235630357dde8baf9f29224ee8500 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 6 Jun 2019 11:39:22 +0200 Subject: [PATCH 281/308] Adds review suggestions I missed --- configs/fork_timelines/mainnet.yaml | 2 +- specs/core/1_custody-game.md | 23 ++++++++++--------- specs/core/1_shard-data-chains.md | 13 ++++------- ...est_process_early_derived_secret_reveal.py | 2 +- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/configs/fork_timelines/mainnet.yaml b/configs/fork_timelines/mainnet.yaml index 8d51d6582a..0bb3c9db11 100644 --- a/configs/fork_timelines/mainnet.yaml +++ b/configs/fork_timelines/mainnet.yaml @@ -7,6 +7,6 @@ phase0: 67108864 # phase0_funny_fork_name: 67116000 # Example 2: -# Should be equal to PHASE_1_GENESIS_EPOCH +# Should be equal to PHASE_1_FORK_EPOCH # (placeholder in example value here) # phase1: 67163000 diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md index 954505c917..6c89ef8531 100644 --- a/specs/core/1_custody-game.md +++ b/specs/core/1_custody-game.md @@ -372,7 +372,7 @@ def process_custody_key_reveal(state: BeaconState, # Reward Block Preposer proposer_index = get_beacon_proposer_index(state) - increase_balance(state, proposer_index, get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT) + increase_balance(state, proposer_index, get_base_reward(state, reveal.revealer_index) // MINOR_REWARD_QUOTIENT) ``` #### Early derived secret reveals @@ -462,7 +462,7 @@ For each `challenge` in `block.body.custody_chunk_challenges`, run the following def process_chunk_challenge(state: BeaconState, challenge: CustodyChunkChallenge) -> None: # Verify the attestation - assert validate_indexed_attestation(state, convert_to_indexed(state, challenge.attestation)) + validate_indexed_attestation(state, convert_to_indexed(state, challenge.attestation)) # Verify it is not too late to challenge assert slot_to_epoch(challenge.attestation.data.slot) >= get_current_epoch(state) - MAX_CHUNK_CHALLENGE_DELAY responder = state.validator_registry[challenge.responder_index] @@ -517,14 +517,15 @@ def process_bit_challenge(state: BeaconState, assert is_slashable_validator(challenger, get_current_epoch(state)) # Verify the attestation - assert validate_indexed_attestation(state, convert_to_indexed(state, challenge.attestation)) + attestation = challenge.attestation + validate_indexed_attestation(state, convert_to_indexed(state, attestation)) # Verify the attestation is eligible for challenging responder = state.validator_registry[challenge.responder_index] - assert (slot_to_epoch(challenge.attestation.data.slot) + responder.max_reveal_lateness <= + assert (slot_to_epoch(attestation.data.slot) + responder.max_reveal_lateness <= get_validators_custody_reveal_period(state, challenge.responder_index)) # Verify the responder participated in the attestation - attesters = get_attesting_indices(state, challenge.attestation.data, challenge.attestation.aggregation_bitfield) + attesters = get_attesting_indices(state, attestation.data, attestation.aggregation_bitfield) assert challenge.responder_index in attesters # A validator can be the challenger for at most one challenge at a time @@ -536,7 +537,7 @@ def process_bit_challenge(state: BeaconState, get_validators_custody_reveal_period( state=state, index=challenge.responder_index, - epoch=slot_to_epoch(challenge.attestation.data.slot)), + epoch=slot_to_epoch(attestation.data.slot)), challenge.responder_index ) assert bls_verify( @@ -551,10 +552,10 @@ def process_bit_challenge(state: BeaconState, ) # Verify the chunk count - chunk_count = get_custody_chunk_count(challenge.attestation.data.crosslink) + chunk_count = get_custody_chunk_count(attestation.data.crosslink) assert verify_bitfield(challenge.chunk_bits, chunk_count) # Verify the first bit of the hash of the chunk bits does not equal the custody bit - custody_bit = get_bitfield_bit(challenge.attestation.custody_bitfield, attesters.index(challenge.responder_index)) + custody_bit = get_bitfield_bit(attestation.custody_bitfield, attesters.index(challenge.responder_index)) assert custody_bit != get_bitfield_bit(get_chunk_bits_root(challenge.chunk_bits), 0) # Add new bit challenge record new_record = CustodyBitChallengeRecord( @@ -562,7 +563,7 @@ def process_bit_challenge(state: BeaconState, challenger_index=challenge.challenger_index, responder_index=challenge.responder_index, inclusion_epoch=get_current_epoch(state), - data_root=challenge.attestation.data.crosslink.data_root, + data_root=attestation.data.crosslink.data_root, chunk_count=chunk_count, chunk_bits_merkle_root=hash_tree_root(challenge.chunk_bits), responder_key=challenge.responder_key, @@ -669,8 +670,8 @@ Run `process_reveal_deadlines(state)` immediately after `process_registry_update # end insert @process_reveal_deadlines def process_reveal_deadlines(state: BeaconState) -> None: for index, validator in enumerate(state.validator_registry): - if (validator.next_custody_reveal_period + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) - < get_validators_custody_reveal_period(state, index)): + deadline = validator.next_custody_reveal_period + (CUSTODY_RESPONSE_DEADLINE // EPOCHS_PER_CUSTODY_PERIOD) + if get_validators_custody_reveal_period(state, index) > deadline: slash_validator(state, index) ``` diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 044868209a..21e08e7c96 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -46,8 +46,8 @@ This document describes the shard data layer and the shard fork choice rule in P | - | - | | `BYTES_PER_SHARD_BLOCK_BODY` | `2**14` (= 16,384) | | `MAX_SHARD_ATTESTIONS` | `2**4` (= 16) | -| `PHASE_1_GENESIS_EPOCH` | **TBD** | -| `PHASE_1_GENESIS_SLOT` | **TBD** | +| `PHASE_1_FORK_EPOCH` | **TBD** | +| `PHASE_1_FORK_SLOT` | **TBD** | | `GENESIS_SHARD_SLOT` | 0 | ### Time parameters @@ -274,14 +274,12 @@ Let: * `beacon_blocks` be the `BeaconBlock` list such that `beacon_blocks[slot]` is the canonical `BeaconBlock` at slot `slot` * `beacon_state` be the canonical `BeaconState` after processing `beacon_blocks[-1]` * `valid_shard_blocks` be the list of valid `ShardBlock`, recursively defined -* `unix_time` be the current unix time * `candidate` be a candidate `ShardBlock` for which validity is to be determined by running `is_valid_shard_block` ```python def is_valid_shard_block(beacon_blocks: List[BeaconBlock], beacon_state: BeaconState, valid_shard_blocks: List[ShardBlock], - unix_time: int, candidate: ShardBlock) -> bool: # Check if block is already determined valid for _, block in enumerate(valid_shard_blocks): @@ -289,8 +287,7 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], return True # Check slot number - assert candidate.slot >= PHASE_1_GENESIS_SLOT - assert unix_time >= beacon_state.genesis_time + (block.slot - GENESIS_SLOT) * SECONDS_PER_SLOT + assert candidate.slot >= PHASE_1_FORK_SLOT # Check shard number assert candidate.shard <= SHARD_COUNT @@ -304,7 +301,7 @@ def is_valid_shard_block(beacon_blocks: List[BeaconBlock], assert candidate.state_root == ZERO_HASH # [to be removed in phase 2] # Check parent block - if candidate.slot == PHASE_1_GENESIS_SLOT: + if candidate.slot == PHASE_1_FORK_SLOT: assert candidate.parent_root == ZERO_HASH else: parent_block = next( @@ -386,7 +383,7 @@ def is_valid_beacon_attestation(shard: Shard, return True # Check previous attestation - if candidate.data.previous_crosslink.epoch <= PHASE_1_GENESIS_EPOCH: + if candidate.data.previous_crosslink.epoch <= PHASE_1_FORK_EPOCH: assert candidate.data.previous_crosslink.data_root == ZERO_HASH else: previous_attestation = next( diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py index 6c5785b2fb..30f19e031e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py @@ -19,7 +19,7 @@ def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, v expect_assertion_error(lambda: spec.process_early_derived_secret_reveal(state, randao_key_reveal)) yield 'post', None return - + pre_slashed_balance = get_balance(state, randao_key_reveal.revealed_index) spec.process_early_derived_secret_reveal(state, randao_key_reveal) From 84ce28a71da5f4795007640d45bf637643dae4c7 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Thu, 6 Jun 2019 09:45:20 -0400 Subject: [PATCH 282/308] Update simple-serialize.md --- specs/simple-serialize.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md index 81126a3123..1cd2750a78 100644 --- a/specs/simple-serialize.md +++ b/specs/simple-serialize.md @@ -13,9 +13,11 @@ - [Composite types](#composite-types) - [Aliases](#aliases) - [Default values](#default-values) + - [Illegal types](#illegal-types) - [Serialization](#serialization) - [`"uintN"`](#uintn) - [`"bool"`](#bool) + - [`"null`](#null) - [Vectors, containers, lists, unions](#vectors-containers-lists-unions) - [Deserialization](#deserialization) - [Merkleization](#merkleization) From 956c476d81157195ac4fb313e4872a56f0a64508 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 6 Jun 2019 16:23:30 +0200 Subject: [PATCH 283/308] Move over to parameterised decorators for phases --- test_libs/pyspec/eth2spec/test/context.py | 42 +++++++------------ .../test_process_attestation.py | 40 +++++++++--------- .../test_process_attester_slashing.py | 20 ++++----- .../test_process_block_header.py | 12 +++--- .../block_processing/test_process_deposit.py | 18 ++++---- .../test_process_proposer_slashing.py | 22 +++++----- .../block_processing/test_process_transfer.py | 26 ++++++------ .../test_process_voluntary_exit.py | 18 ++++---- .../test_process_crosslinks.py | 10 ++--- .../test_process_registry_updates.py | 6 +-- ...est_process_early_derived_secret_reveal.py | 18 ++++---- .../eth2spec/test/sanity/test_blocks.py | 30 ++++++------- .../pyspec/eth2spec/test/sanity/test_slots.py | 12 +++--- .../pyspec/eth2spec/test/test_finality.py | 10 ++--- 14 files changed, 135 insertions(+), 149 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 8bfce82436..5dac3cefcb 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -90,33 +90,19 @@ def entry(*args, **kw): return entry -def with_phase0(fn): +def with_phases(phases): """ - Decorator to use phase 0's spec. + Decorator factory that returns a decorator that runs a test for the appropriate phases """ - def entry(*args, **kw): - kw['spec'] = spec_phase0 - return fn(*args, **kw) - return entry - - -def with_phase1(fn): - """ - Decorator to use phase 1's spec - """ - def entry(*args, **kw): - kw['spec'] = spec_phase1 - return fn(*args, **kw) - return entry - - -def with_all_phases(fn): - """ - Decorator to run everything with all available spec phases - """ - def entry(*args, **kw): - kw['spec'] = spec_phase0 - fn(*args, **kw) - kw['spec'] = spec_phase1 - fn(*args, **kw) - return entry + def decorator(fn): + def run_with_spec_version(spec, *args, **kw): + kw['spec'] = spec + fn(*args, **kw) + + def wrapper(*args, **kw): + if 'phase0' in phases: + run_with_spec_version(spec_phase0, *args, **kw) + if 'phase1' in phases: + run_with_spec_version(spec_phase1, *args, **kw) + return wrapper + return decorator diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 8e214fe7d0..7840345b6d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -1,6 +1,6 @@ from copy import deepcopy -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phase0 +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_phases from eth2spec.test.helpers.attestations import ( get_valid_attestation, sign_attestation, @@ -47,7 +47,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): yield 'post', state -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_success(spec, state): attestation = get_valid_attestation(spec, state, signed=True) @@ -56,7 +56,7 @@ def test_success(spec, state): yield from run_attestation_processing(spec, state, attestation) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_success_previous_epoch(spec, state): attestation = get_valid_attestation(spec, state, signed=True) @@ -66,7 +66,7 @@ def test_success_previous_epoch(spec, state): yield from run_attestation_processing(spec, state, attestation) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_success_since_max_epochs_per_crosslink(spec, state): for _ in range(spec.MAX_EPOCHS_PER_CROSSLINK + 2): @@ -85,7 +85,7 @@ def test_success_since_max_epochs_per_crosslink(spec, state): yield from run_attestation_processing(spec, state, attestation) -@with_all_phases +@with_phases(['phase0', 'phase1']) @always_bls @spec_state_test def test_invalid_attestation_signature(spec, state): @@ -95,7 +95,7 @@ def test_invalid_attestation_signature(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_before_inclusion_delay(spec, state): attestation = get_valid_attestation(spec, state, signed=True) @@ -104,7 +104,7 @@ def test_before_inclusion_delay(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_after_epoch_slots(spec, state): attestation = get_valid_attestation(spec, state, signed=True) @@ -115,7 +115,7 @@ def test_after_epoch_slots(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_old_source_epoch(spec, state): state.slot = spec.SLOTS_PER_EPOCH * 5 @@ -135,7 +135,7 @@ def test_old_source_epoch(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_wrong_shard(spec, state): attestation = get_valid_attestation(spec, state) @@ -148,7 +148,7 @@ def test_wrong_shard(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_new_source_epoch(spec, state): attestation = get_valid_attestation(spec, state) @@ -161,7 +161,7 @@ def test_new_source_epoch(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_source_root_is_target_root(spec, state): attestation = get_valid_attestation(spec, state) @@ -174,7 +174,7 @@ def test_source_root_is_target_root(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_invalid_current_source_root(spec, state): state.slot = spec.SLOTS_PER_EPOCH * 5 @@ -201,7 +201,7 @@ def test_invalid_current_source_root(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_bad_source_root(spec, state): attestation = get_valid_attestation(spec, state) @@ -214,7 +214,7 @@ def test_bad_source_root(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_phase0 +@with_phases(['phase0']) @spec_state_test def test_non_zero_crosslink_data_root(spec, state): attestation = get_valid_attestation(spec, state) @@ -227,7 +227,7 @@ def test_non_zero_crosslink_data_root(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_bad_parent_crosslink(spec, state): next_epoch(spec, state) @@ -243,7 +243,7 @@ def test_bad_parent_crosslink(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_bad_crosslink_start_epoch(spec, state): next_epoch(spec, state) @@ -259,7 +259,7 @@ def test_bad_crosslink_start_epoch(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_bad_crosslink_end_epoch(spec, state): next_epoch(spec, state) @@ -275,7 +275,7 @@ def test_bad_crosslink_end_epoch(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_inconsistent_bitfields(spec, state): attestation = get_valid_attestation(spec, state) @@ -288,7 +288,7 @@ def test_inconsistent_bitfields(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_phase0 +@with_phases(['phase0']) @spec_state_test def test_non_empty_custody_bitfield(spec, state): attestation = get_valid_attestation(spec, state) @@ -301,7 +301,7 @@ def test_non_empty_custody_bitfield(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_empty_aggregation_bitfield(spec, state): attestation = get_valid_attestation(spec, state) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index 6c7637d599..aad68cc4b2 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_phases from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing from eth2spec.test.helpers.block import apply_empty_block @@ -54,7 +54,7 @@ def run_attester_slashing_processing(spec, state, attester_slashing, valid=True) yield 'post', state -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_success_double(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) @@ -62,7 +62,7 @@ def test_success_double(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_success_surround(spec, state): next_epoch(spec, state) @@ -80,7 +80,7 @@ def test_success_surround(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing) -@with_all_phases +@with_phases(['phase0', 'phase1']) @always_bls @spec_state_test def test_invalid_sig_1(spec, state): @@ -88,7 +88,7 @@ def test_invalid_sig_1(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @always_bls @spec_state_test def test_invalid_sig_2(spec, state): @@ -96,7 +96,7 @@ def test_invalid_sig_2(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @always_bls @spec_state_test def test_invalid_sig_1_and_2(spec, state): @@ -104,7 +104,7 @@ def test_invalid_sig_1_and_2(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_same_data(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) @@ -115,7 +115,7 @@ def test_same_data(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_no_double_or_surround(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) @@ -126,7 +126,7 @@ def test_no_double_or_surround(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_participants_already_slashed(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) @@ -140,7 +140,7 @@ def test_participants_already_slashed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_custody_bit_0_and_1(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py index f3c017982d..b2d22232fb 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py @@ -1,6 +1,6 @@ from copy import deepcopy -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_phases from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, sign_block @@ -34,14 +34,14 @@ def run_block_header_processing(spec, state, block, valid=True): yield 'post', state -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_success_block_header(spec, state): block = build_empty_block_for_next_slot(spec, state, signed=True) yield from run_block_header_processing(spec, state, block) -@with_all_phases +@with_phases(['phase0', 'phase1']) @always_bls @spec_state_test def test_invalid_sig_block_header(spec, state): @@ -49,7 +49,7 @@ def test_invalid_sig_block_header(spec, state): yield from run_block_header_processing(spec, state, block, valid=False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_invalid_slot_block_header(spec, state): block = build_empty_block_for_next_slot(spec, state) @@ -59,7 +59,7 @@ def test_invalid_slot_block_header(spec, state): yield from run_block_header_processing(spec, state, block, valid=False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_invalid_parent_root(spec, state): block = build_empty_block_for_next_slot(spec, state) @@ -69,7 +69,7 @@ def test_invalid_parent_root(spec, state): yield from run_block_header_processing(spec, state, block, valid=False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_proposer_slashed(spec, state): # use stub state to get proposer index of next slot diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index c50b11f2ef..302a3d37f4 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_phases from eth2spec.test.helpers.deposits import ( build_deposit, prepare_state_and_deposit, @@ -52,7 +52,7 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef assert state.deposit_index == state.latest_eth1_data.deposit_count -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_new_deposit(spec, state): # fresh deposit = next validator index = validator appended to registry @@ -63,7 +63,7 @@ def test_new_deposit(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index) -@with_all_phases +@with_phases(['phase0', 'phase1']) @always_bls @spec_state_test def test_invalid_sig_new_deposit(spec, state): @@ -74,7 +74,7 @@ def test_invalid_sig_new_deposit(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_success_top_up(spec, state): validator_index = 0 @@ -84,7 +84,7 @@ def test_success_top_up(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index) -@with_all_phases +@with_phases(['phase0', 'phase1']) @always_bls @spec_state_test def test_invalid_sig_top_up(spec, state): @@ -96,7 +96,7 @@ def test_invalid_sig_top_up(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=True) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_invalid_withdrawal_credentials_top_up(spec, state): validator_index = 0 @@ -114,7 +114,7 @@ def test_invalid_withdrawal_credentials_top_up(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=True) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_wrong_index(spec, state): validator_index = len(state.validator_registry) @@ -129,7 +129,7 @@ def test_wrong_index(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_wrong_deposit_for_deposit_count(spec, state): deposit_data_leaves = [spec.ZERO_HASH] * len(state.validator_registry) @@ -175,7 +175,7 @@ def test_wrong_deposit_for_deposit_count(spec, state): # TODO: test invalid signature -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_bad_merkle_proof(spec, state): validator_index = len(state.validator_registry) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py index b35241859c..3c8e20d7e3 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_phases from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import privkeys from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing @@ -40,7 +40,7 @@ def run_proposer_slashing_processing(spec, state, proposer_slashing, valid=True) ) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_success(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) @@ -48,7 +48,7 @@ def test_success(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing) -@with_all_phases +@with_phases(['phase0', 'phase1']) @always_bls @spec_state_test def test_invalid_sig_1(spec, state): @@ -56,7 +56,7 @@ def test_invalid_sig_1(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @always_bls @spec_state_test def test_invalid_sig_2(spec, state): @@ -64,7 +64,7 @@ def test_invalid_sig_2(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @always_bls @spec_state_test def test_invalid_sig_1_and_2(spec, state): @@ -72,7 +72,7 @@ def test_invalid_sig_1_and_2(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_invalid_proposer_index(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) @@ -82,7 +82,7 @@ def test_invalid_proposer_index(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_epochs_are_different(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=False) @@ -94,7 +94,7 @@ def test_epochs_are_different(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_headers_are_same(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=False) @@ -105,7 +105,7 @@ def test_headers_are_same(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_proposer_is_not_activated(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) @@ -116,7 +116,7 @@ def test_proposer_is_not_activated(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_proposer_is_slashed(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) @@ -127,7 +127,7 @@ def test_proposer_is_slashed(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_proposer_is_withdrawn(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index 1294ca84a2..13de5f6b06 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_phases from eth2spec.test.helpers.state import next_epoch from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.transfers import get_valid_transfer @@ -36,7 +36,7 @@ def run_transfer_processing(spec, state, transfer, valid=True): assert state.balances[proposer_index] == pre_transfer_proposer_balance + transfer.fee -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_success_non_activated(spec, state): transfer = get_valid_transfer(spec, state, signed=True) @@ -46,7 +46,7 @@ def test_success_non_activated(spec, state): yield from run_transfer_processing(spec, state, transfer) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_success_withdrawable(spec, state): next_epoch(spec, state) @@ -60,7 +60,7 @@ def test_success_withdrawable(spec, state): yield from run_transfer_processing(spec, state, transfer) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_success_active_above_max_effective(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -70,7 +70,7 @@ def test_success_active_above_max_effective(spec, state): yield from run_transfer_processing(spec, state, transfer) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_success_active_above_max_effective_fee(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -80,7 +80,7 @@ def test_success_active_above_max_effective_fee(spec, state): yield from run_transfer_processing(spec, state, transfer) -@with_all_phases +@with_phases(['phase0', 'phase1']) @always_bls @spec_state_test def test_invalid_signature(spec, state): @@ -91,7 +91,7 @@ def test_invalid_signature(spec, state): yield from run_transfer_processing(spec, state, transfer, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_active_but_transfer_past_effective_balance(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -102,7 +102,7 @@ def test_active_but_transfer_past_effective_balance(spec, state): yield from run_transfer_processing(spec, state, transfer, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_incorrect_slot(spec, state): transfer = get_valid_transfer(spec, state, slot=state.slot + 1, signed=True) @@ -112,7 +112,7 @@ def test_incorrect_slot(spec, state): yield from run_transfer_processing(spec, state, transfer, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_insufficient_balance_for_fee(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -125,7 +125,7 @@ def test_insufficient_balance_for_fee(spec, state): yield from run_transfer_processing(spec, state, transfer, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_insufficient_balance(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -138,7 +138,7 @@ def test_insufficient_balance(spec, state): yield from run_transfer_processing(spec, state, transfer, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_no_dust_sender(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -158,7 +158,7 @@ def test_no_dust_sender(spec, state): yield from run_transfer_processing(spec, state, transfer, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_no_dust_recipient(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -172,7 +172,7 @@ def test_no_dust_recipient(spec, state): yield from run_transfer_processing(spec, state, transfer, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_invalid_pubkey(spec, state): transfer = get_valid_transfer(spec, state, signed=True) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py index 3359c5e789..4b3f7446cc 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_phases from eth2spec.test.helpers.keys import pubkey_to_privkey from eth2spec.test.helpers.voluntary_exits import build_voluntary_exit, sign_voluntary_exit @@ -31,7 +31,7 @@ def run_voluntary_exit_processing(spec, state, voluntary_exit, valid=True): assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_success(spec, state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit @@ -46,7 +46,7 @@ def test_success(spec, state): yield from run_voluntary_exit_processing(spec, state, voluntary_exit) -@with_all_phases +@with_phases(['phase0', 'phase1']) @always_bls @spec_state_test def test_invalid_signature(spec, state): @@ -62,7 +62,7 @@ def test_invalid_signature(spec, state): yield from run_voluntary_exit_processing(spec, state, voluntary_exit, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_success_exit_queue(spec, state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit @@ -114,7 +114,7 @@ def test_success_exit_queue(spec, state): ) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_validator_exit_in_future(spec, state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit @@ -138,7 +138,7 @@ def test_validator_exit_in_future(spec, state): yield from run_voluntary_exit_processing(spec, state, voluntary_exit, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_validator_invalid_validator_index(spec, state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit @@ -162,7 +162,7 @@ def test_validator_invalid_validator_index(spec, state): yield from run_voluntary_exit_processing(spec, state, voluntary_exit, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_validator_not_active(spec, state): current_epoch = spec.get_current_epoch(state) @@ -184,7 +184,7 @@ def test_validator_not_active(spec, state): yield from run_voluntary_exit_processing(spec, state, voluntary_exit, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_validator_already_exited(spec, state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow validator able to exit @@ -209,7 +209,7 @@ def test_validator_already_exited(spec, state): yield from run_voluntary_exit_processing(spec, state, voluntary_exit, False) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_validator_not_active_long_enough(spec, state): current_epoch = spec.get_current_epoch(state) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index 65d9586788..65bb6a0552 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -1,6 +1,6 @@ from copy import deepcopy -from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.context import spec_state_test, with_phases from eth2spec.test.helpers.state import ( next_epoch, next_slot @@ -37,7 +37,7 @@ def run_process_crosslinks(spec, state, valid=True): yield 'post', state -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_no_attestations(spec, state): yield from run_process_crosslinks(spec, state) @@ -46,7 +46,7 @@ def test_no_attestations(spec, state): assert state.previous_crosslinks[shard] == state.current_crosslinks[shard] -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_single_crosslink_update_from_current_epoch(spec, state): next_epoch(spec, state) @@ -67,7 +67,7 @@ def test_single_crosslink_update_from_current_epoch(spec, state): assert pre_crosslink != state.current_crosslinks[shard] -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_single_crosslink_update_from_previous_epoch(spec, state): next_epoch(spec, state) @@ -98,7 +98,7 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): assert crosslink_deltas[1][index] == 0 -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_double_late_crosslink(spec, state): if spec.get_epoch_committee_count(state, spec.get_current_epoch(state)) < spec.SHARD_COUNT: diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py index e6679f8448..daa52dba99 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py @@ -1,7 +1,7 @@ from eth2spec.phase0.spec import state_transition from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block from eth2spec.test.helpers.state import next_epoch -from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.context import spec_state_test, with_phases def run_process_registry_updates(spec, state, valid=True): @@ -31,7 +31,7 @@ def run_process_registry_updates(spec, state, valid=True): yield 'post', state -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_activation(spec, state): index = 0 @@ -56,7 +56,7 @@ def test_activation(spec, state): ) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_ejection(spec, state): index = 0 diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py index 30f19e031e..f028217c63 100644 --- a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py @@ -1,7 +1,7 @@ from eth2spec.test.helpers.custody import get_valid_early_derived_secret_reveal from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.state import next_epoch, get_balance -from eth2spec.test.context import with_phase1, spec_state_test, expect_assertion_error +from eth2spec.test.context import with_phases, spec_state_test, expect_assertion_error def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, valid=True): @@ -35,7 +35,7 @@ def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, v yield 'post', state -@with_phase1 +@with_phases(['phase1']) @spec_state_test def test_success(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state) @@ -43,7 +43,7 @@ def test_success(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal) -@with_phase1 +@with_phases(['phase1']) @spec_state_test def test_reveal_from_current_epoch(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state)) @@ -51,7 +51,7 @@ def test_reveal_from_current_epoch(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -@with_phase1 +@with_phases(['phase1']) @spec_state_test def test_reveal_from_past_epoch(spec, state): next_epoch(spec, state) @@ -61,7 +61,7 @@ def test_reveal_from_past_epoch(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -@with_phase1 +@with_phases(['phase1']) @spec_state_test def test_reveal_with_custody_padding(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal( @@ -72,7 +72,7 @@ def test_reveal_with_custody_padding(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, True) -@with_phase1 +@with_phases(['phase1']) @spec_state_test def test_reveal_with_custody_padding_minus_one(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal( @@ -83,7 +83,7 @@ def test_reveal_with_custody_padding_minus_one(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, True) -@with_phase1 +@with_phases(['phase1']) @spec_state_test def test_double_reveal(spec, state): randao_key_reveal1 = get_valid_early_derived_secret_reveal( @@ -107,7 +107,7 @@ def test_double_reveal(spec, state): yield 'post', post_state -@with_phase1 +@with_phases(['phase1']) @spec_state_test def test_revealer_is_slashed(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state)) @@ -116,7 +116,7 @@ def test_revealer_is_slashed(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -@with_phase1 +@with_phases(['phase1']) @spec_state_test def test_far_future_epoch(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal( diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 587c377429..c08dd83cdd 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -13,10 +13,10 @@ from eth2spec.test.helpers.attestations import get_valid_attestation from eth2spec.test.helpers.deposits import prepare_state_and_deposit -from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +from eth2spec.test.context import spec_state_test, never_bls, with_phases -@with_all_phases +@with_phases(['phase0', 'phase1']) @never_bls @spec_state_test def test_empty_block_transition(spec, state): @@ -35,7 +35,7 @@ def test_empty_block_transition(spec, state): assert spec.get_block_root_at_slot(state, pre_slot) == block.parent_root -@with_all_phases +@with_phases(['phase0', 'phase1']) @never_bls @spec_state_test def test_skipped_slots(spec, state): @@ -55,7 +55,7 @@ def test_skipped_slots(spec, state): assert spec.get_block_root_at_slot(state, slot) == block.parent_root -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_empty_epoch_transition(spec, state): pre_slot = state.slot @@ -74,7 +74,7 @@ def test_empty_epoch_transition(spec, state): assert spec.get_block_root_at_slot(state, slot) == block.parent_root -# @with_all_phases +# @with_phases(['phase0', 'phase1']) # @spec_state_test # def test_empty_epoch_transition_not_finalizing(spec, state): # # copy for later balance lookups. @@ -95,7 +95,7 @@ def test_empty_epoch_transition(spec, state): # assert get_balance(state, index) < get_balance(pre_state, index) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_proposer_slashing(spec, state): # copy for later balance lookups. @@ -127,7 +127,7 @@ def test_proposer_slashing(spec, state): assert get_balance(state, validator_index) < get_balance(pre_state, validator_index) -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_attester_slashing(spec, state): # copy for later balance lookups. @@ -169,7 +169,7 @@ def test_attester_slashing(spec, state): # TODO update functions below to be like above, i.e. with @spec_state_test and yielding data to put into the test vector -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_deposit_in_block(spec, state): initial_registry_len = len(state.validator_registry) @@ -196,7 +196,7 @@ def test_deposit_in_block(spec, state): assert state.validator_registry[validator_index].pubkey == pubkeys[validator_index] -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_deposit_top_up(spec, state): validator_index = 0 @@ -223,7 +223,7 @@ def test_deposit_top_up(spec, state): assert get_balance(state, validator_index) == validator_pre_balance + amount -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_attestation(spec, state): state.slot = spec.SLOTS_PER_EPOCH @@ -257,7 +257,7 @@ def test_attestation(spec, state): assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_voluntary_exit(spec, state): validator_index = spec.get_active_validator_indices( @@ -303,7 +303,7 @@ def test_voluntary_exit(spec, state): assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_transfer(spec, state): # overwrite default 0 to test @@ -337,7 +337,7 @@ def test_transfer(spec, state): assert recipient_balance == pre_transfer_recipient_balance + amount -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_balance_driven_status_transitions(spec, state): current_epoch = spec.get_current_epoch(state) @@ -362,7 +362,7 @@ def test_balance_driven_status_transitions(spec, state): assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_historical_batch(spec, state): state.slot += spec.SLOTS_PER_HISTORICAL_ROOT - (state.slot % spec.SLOTS_PER_HISTORICAL_ROOT) - 1 @@ -381,7 +381,7 @@ def test_historical_batch(spec, state): assert len(state.historical_roots) == pre_historical_roots_len + 1 -# @with_all_phases +# @with_phases(['phase0', 'phase1']) # @spec_state_test # def test_eth1_data_votes(spec, state): # yield 'pre', state diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_slots.py b/test_libs/pyspec/eth2spec/test/sanity/test_slots.py index 6ef6be4d38..feddd65be3 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_slots.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_slots.py @@ -1,8 +1,8 @@ from eth2spec.test.helpers.state import get_state_root -from eth2spec.test.context import spec_state_test, with_all_phases +from eth2spec.test.context import spec_state_test, with_phases -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_slots_1(spec, state): pre_slot = state.slot @@ -18,7 +18,7 @@ def test_slots_1(spec, state): assert get_state_root(spec, state, pre_slot) == pre_root -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_slots_2(spec, state): yield 'pre', state @@ -28,7 +28,7 @@ def test_slots_2(spec, state): yield 'post', state -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_empty_epoch(spec, state): yield 'pre', state @@ -38,7 +38,7 @@ def test_empty_epoch(spec, state): yield 'post', state -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_double_empty_epoch(spec, state): yield 'pre', state @@ -48,7 +48,7 @@ def test_double_empty_epoch(spec, state): yield 'post', state -@with_all_phases +@with_phases(['phase0', 'phase1']) @spec_state_test def test_over_epoch_boundary(spec, state): spec.process_slots(state, state.slot + (spec.SLOTS_PER_EPOCH // 2)) diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 801e8b4fd5..2d2e01432b 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -1,7 +1,7 @@ from copy import deepcopy from typing import List -from eth2spec.test.context import spec_state_test, never_bls, with_all_phases +from eth2spec.test.context import spec_state_test, never_bls, with_phases from eth2spec.test.helpers.state import next_epoch from eth2spec.test.helpers.block import build_empty_block_for_next_slot, apply_empty_block from eth2spec.test.helpers.attestations import get_valid_attestation @@ -60,7 +60,7 @@ def next_epoch_with_attestations(spec, return state, blocks, post_state -@with_all_phases +@with_phases(['phase0', 'phase1']) @never_bls @spec_state_test def test_finality_rule_4(spec, state): @@ -89,7 +89,7 @@ def test_finality_rule_4(spec, state): yield 'post', state -@with_all_phases +@with_phases(['phase0', 'phase1']) @never_bls @spec_state_test def test_finality_rule_1(spec, state): @@ -120,7 +120,7 @@ def test_finality_rule_1(spec, state): yield 'post', state -@with_all_phases +@with_phases(['phase0', 'phase1']) @never_bls @spec_state_test def test_finality_rule_2(spec, state): @@ -153,7 +153,7 @@ def test_finality_rule_2(spec, state): yield 'post', state -@with_all_phases +@with_phases(['phase0', 'phase1']) @never_bls @spec_state_test def test_finality_rule_3(spec, state): From 67471a8d6e802115513c4014fa25c6abd28c7f35 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 6 Jun 2019 23:30:40 +0200 Subject: [PATCH 284/308] Rework of phases decorator --- test_libs/pyspec/eth2spec/test/context.py | 19 ++++++++++ .../test_process_attestation.py | 36 +++++++++---------- .../test_process_attester_slashing.py | 20 +++++------ .../test_process_block_header.py | 12 +++---- .../block_processing/test_process_deposit.py | 18 +++++----- .../test_process_proposer_slashing.py | 22 ++++++------ .../block_processing/test_process_transfer.py | 26 +++++++------- .../test_process_voluntary_exit.py | 18 +++++----- .../test_process_crosslinks.py | 10 +++--- .../test_process_registry_updates.py | 6 ++-- ...est_process_early_derived_secret_reveal.py | 18 +++++----- .../eth2spec/test/sanity/test_blocks.py | 30 ++++++++-------- .../pyspec/eth2spec/test/sanity/test_slots.py | 12 +++---- .../pyspec/eth2spec/test/test_finality.py | 10 +++--- 14 files changed, 138 insertions(+), 119 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 5dac3cefcb..cbc594cd8e 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -90,6 +90,25 @@ def entry(*args, **kw): return entry +all_phases = ['phase0', 'phase1'] + + +def with_all_phases(fn): + """ + A decorator for running a test wil every phase + """ + return with_phases(all_phases)(fn) + + +def with_all_phases_except(exclusion_phases): + """ + A decorator factory for running a tests with every phase except the ones listed + """ + def decorator(fn): + return with_phases([phase for phase in all_phases if phase not in exclusion_phases])(fn) + return decorator + + def with_phases(phases): """ Decorator factory that returns a decorator that runs a test for the appropriate phases diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py index 7840345b6d..2b34ab405e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py @@ -1,6 +1,6 @@ from copy import deepcopy -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_phases +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases from eth2spec.test.helpers.attestations import ( get_valid_attestation, sign_attestation, @@ -47,7 +47,7 @@ def run_attestation_processing(spec, state, attestation, valid=True): yield 'post', state -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_success(spec, state): attestation = get_valid_attestation(spec, state, signed=True) @@ -56,7 +56,7 @@ def test_success(spec, state): yield from run_attestation_processing(spec, state, attestation) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_success_previous_epoch(spec, state): attestation = get_valid_attestation(spec, state, signed=True) @@ -66,7 +66,7 @@ def test_success_previous_epoch(spec, state): yield from run_attestation_processing(spec, state, attestation) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_success_since_max_epochs_per_crosslink(spec, state): for _ in range(spec.MAX_EPOCHS_PER_CROSSLINK + 2): @@ -85,7 +85,7 @@ def test_success_since_max_epochs_per_crosslink(spec, state): yield from run_attestation_processing(spec, state, attestation) -@with_phases(['phase0', 'phase1']) +@with_all_phases @always_bls @spec_state_test def test_invalid_attestation_signature(spec, state): @@ -95,7 +95,7 @@ def test_invalid_attestation_signature(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_before_inclusion_delay(spec, state): attestation = get_valid_attestation(spec, state, signed=True) @@ -104,7 +104,7 @@ def test_before_inclusion_delay(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_after_epoch_slots(spec, state): attestation = get_valid_attestation(spec, state, signed=True) @@ -115,7 +115,7 @@ def test_after_epoch_slots(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_old_source_epoch(spec, state): state.slot = spec.SLOTS_PER_EPOCH * 5 @@ -135,7 +135,7 @@ def test_old_source_epoch(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_wrong_shard(spec, state): attestation = get_valid_attestation(spec, state) @@ -148,7 +148,7 @@ def test_wrong_shard(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_new_source_epoch(spec, state): attestation = get_valid_attestation(spec, state) @@ -161,7 +161,7 @@ def test_new_source_epoch(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_source_root_is_target_root(spec, state): attestation = get_valid_attestation(spec, state) @@ -174,7 +174,7 @@ def test_source_root_is_target_root(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_invalid_current_source_root(spec, state): state.slot = spec.SLOTS_PER_EPOCH * 5 @@ -201,7 +201,7 @@ def test_invalid_current_source_root(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_bad_source_root(spec, state): attestation = get_valid_attestation(spec, state) @@ -227,7 +227,7 @@ def test_non_zero_crosslink_data_root(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_bad_parent_crosslink(spec, state): next_epoch(spec, state) @@ -243,7 +243,7 @@ def test_bad_parent_crosslink(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_bad_crosslink_start_epoch(spec, state): next_epoch(spec, state) @@ -259,7 +259,7 @@ def test_bad_crosslink_start_epoch(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_bad_crosslink_end_epoch(spec, state): next_epoch(spec, state) @@ -275,7 +275,7 @@ def test_bad_crosslink_end_epoch(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_inconsistent_bitfields(spec, state): attestation = get_valid_attestation(spec, state) @@ -301,7 +301,7 @@ def test_non_empty_custody_bitfield(spec, state): yield from run_attestation_processing(spec, state, attestation, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_empty_aggregation_bitfield(spec, state): attestation = get_valid_attestation(spec, state) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py index aad68cc4b2..6c7637d599 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attester_slashing.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_phases +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases from eth2spec.test.helpers.attestations import sign_indexed_attestation from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing from eth2spec.test.helpers.block import apply_empty_block @@ -54,7 +54,7 @@ def run_attester_slashing_processing(spec, state, attester_slashing, valid=True) yield 'post', state -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_success_double(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) @@ -62,7 +62,7 @@ def test_success_double(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_success_surround(spec, state): next_epoch(spec, state) @@ -80,7 +80,7 @@ def test_success_surround(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing) -@with_phases(['phase0', 'phase1']) +@with_all_phases @always_bls @spec_state_test def test_invalid_sig_1(spec, state): @@ -88,7 +88,7 @@ def test_invalid_sig_1(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @always_bls @spec_state_test def test_invalid_sig_2(spec, state): @@ -96,7 +96,7 @@ def test_invalid_sig_2(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @always_bls @spec_state_test def test_invalid_sig_1_and_2(spec, state): @@ -104,7 +104,7 @@ def test_invalid_sig_1_and_2(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_same_data(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) @@ -115,7 +115,7 @@ def test_same_data(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_no_double_or_surround(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) @@ -126,7 +126,7 @@ def test_no_double_or_surround(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_participants_already_slashed(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=True, signed_2=True) @@ -140,7 +140,7 @@ def test_participants_already_slashed(spec, state): yield from run_attester_slashing_processing(spec, state, attester_slashing, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_custody_bit_0_and_1(spec, state): attester_slashing = get_valid_attester_slashing(spec, state, signed_1=False, signed_2=True) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py index b2d22232fb..f3c017982d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_block_header.py @@ -1,6 +1,6 @@ from copy import deepcopy -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_phases +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, sign_block @@ -34,14 +34,14 @@ def run_block_header_processing(spec, state, block, valid=True): yield 'post', state -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_success_block_header(spec, state): block = build_empty_block_for_next_slot(spec, state, signed=True) yield from run_block_header_processing(spec, state, block) -@with_phases(['phase0', 'phase1']) +@with_all_phases @always_bls @spec_state_test def test_invalid_sig_block_header(spec, state): @@ -49,7 +49,7 @@ def test_invalid_sig_block_header(spec, state): yield from run_block_header_processing(spec, state, block, valid=False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_invalid_slot_block_header(spec, state): block = build_empty_block_for_next_slot(spec, state) @@ -59,7 +59,7 @@ def test_invalid_slot_block_header(spec, state): yield from run_block_header_processing(spec, state, block, valid=False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_invalid_parent_root(spec, state): block = build_empty_block_for_next_slot(spec, state) @@ -69,7 +69,7 @@ def test_invalid_parent_root(spec, state): yield from run_block_header_processing(spec, state, block, valid=False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_proposer_slashed(spec, state): # use stub state to get proposer index of next slot diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 302a3d37f4..c50b11f2ef 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_phases +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases from eth2spec.test.helpers.deposits import ( build_deposit, prepare_state_and_deposit, @@ -52,7 +52,7 @@ def run_deposit_processing(spec, state, deposit, validator_index, valid=True, ef assert state.deposit_index == state.latest_eth1_data.deposit_count -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_new_deposit(spec, state): # fresh deposit = next validator index = validator appended to registry @@ -63,7 +63,7 @@ def test_new_deposit(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index) -@with_phases(['phase0', 'phase1']) +@with_all_phases @always_bls @spec_state_test def test_invalid_sig_new_deposit(spec, state): @@ -74,7 +74,7 @@ def test_invalid_sig_new_deposit(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_success_top_up(spec, state): validator_index = 0 @@ -84,7 +84,7 @@ def test_success_top_up(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index) -@with_phases(['phase0', 'phase1']) +@with_all_phases @always_bls @spec_state_test def test_invalid_sig_top_up(spec, state): @@ -96,7 +96,7 @@ def test_invalid_sig_top_up(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=True) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_invalid_withdrawal_credentials_top_up(spec, state): validator_index = 0 @@ -114,7 +114,7 @@ def test_invalid_withdrawal_credentials_top_up(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index, valid=True, effective=True) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_wrong_index(spec, state): validator_index = len(state.validator_registry) @@ -129,7 +129,7 @@ def test_wrong_index(spec, state): yield from run_deposit_processing(spec, state, deposit, validator_index, valid=False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_wrong_deposit_for_deposit_count(spec, state): deposit_data_leaves = [spec.ZERO_HASH] * len(state.validator_registry) @@ -175,7 +175,7 @@ def test_wrong_deposit_for_deposit_count(spec, state): # TODO: test invalid signature -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_bad_merkle_proof(spec, state): validator_index = len(state.validator_registry) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py index 3c8e20d7e3..b35241859c 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_proposer_slashing.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_phases +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases from eth2spec.test.helpers.block_header import sign_block_header from eth2spec.test.helpers.keys import privkeys from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing @@ -40,7 +40,7 @@ def run_proposer_slashing_processing(spec, state, proposer_slashing, valid=True) ) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_success(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) @@ -48,7 +48,7 @@ def test_success(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing) -@with_phases(['phase0', 'phase1']) +@with_all_phases @always_bls @spec_state_test def test_invalid_sig_1(spec, state): @@ -56,7 +56,7 @@ def test_invalid_sig_1(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @always_bls @spec_state_test def test_invalid_sig_2(spec, state): @@ -64,7 +64,7 @@ def test_invalid_sig_2(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @always_bls @spec_state_test def test_invalid_sig_1_and_2(spec, state): @@ -72,7 +72,7 @@ def test_invalid_sig_1_and_2(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_invalid_proposer_index(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) @@ -82,7 +82,7 @@ def test_invalid_proposer_index(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_epochs_are_different(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=False) @@ -94,7 +94,7 @@ def test_epochs_are_different(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_headers_are_same(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=False) @@ -105,7 +105,7 @@ def test_headers_are_same(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_proposer_is_not_activated(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) @@ -116,7 +116,7 @@ def test_proposer_is_not_activated(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_proposer_is_slashed(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) @@ -127,7 +127,7 @@ def test_proposer_is_slashed(spec, state): yield from run_proposer_slashing_processing(spec, state, proposer_slashing, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_proposer_is_withdrawn(spec, state): proposer_slashing = get_valid_proposer_slashing(spec, state, signed_1=True, signed_2=True) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py index 13de5f6b06..1294ca84a2 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_transfer.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_phases +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases from eth2spec.test.helpers.state import next_epoch from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.transfers import get_valid_transfer @@ -36,7 +36,7 @@ def run_transfer_processing(spec, state, transfer, valid=True): assert state.balances[proposer_index] == pre_transfer_proposer_balance + transfer.fee -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_success_non_activated(spec, state): transfer = get_valid_transfer(spec, state, signed=True) @@ -46,7 +46,7 @@ def test_success_non_activated(spec, state): yield from run_transfer_processing(spec, state, transfer) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_success_withdrawable(spec, state): next_epoch(spec, state) @@ -60,7 +60,7 @@ def test_success_withdrawable(spec, state): yield from run_transfer_processing(spec, state, transfer) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_success_active_above_max_effective(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -70,7 +70,7 @@ def test_success_active_above_max_effective(spec, state): yield from run_transfer_processing(spec, state, transfer) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_success_active_above_max_effective_fee(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -80,7 +80,7 @@ def test_success_active_above_max_effective_fee(spec, state): yield from run_transfer_processing(spec, state, transfer) -@with_phases(['phase0', 'phase1']) +@with_all_phases @always_bls @spec_state_test def test_invalid_signature(spec, state): @@ -91,7 +91,7 @@ def test_invalid_signature(spec, state): yield from run_transfer_processing(spec, state, transfer, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_active_but_transfer_past_effective_balance(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -102,7 +102,7 @@ def test_active_but_transfer_past_effective_balance(spec, state): yield from run_transfer_processing(spec, state, transfer, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_incorrect_slot(spec, state): transfer = get_valid_transfer(spec, state, slot=state.slot + 1, signed=True) @@ -112,7 +112,7 @@ def test_incorrect_slot(spec, state): yield from run_transfer_processing(spec, state, transfer, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_insufficient_balance_for_fee(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -125,7 +125,7 @@ def test_insufficient_balance_for_fee(spec, state): yield from run_transfer_processing(spec, state, transfer, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_insufficient_balance(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -138,7 +138,7 @@ def test_insufficient_balance(spec, state): yield from run_transfer_processing(spec, state, transfer, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_no_dust_sender(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -158,7 +158,7 @@ def test_no_dust_sender(spec, state): yield from run_transfer_processing(spec, state, transfer, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_no_dust_recipient(spec, state): sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] @@ -172,7 +172,7 @@ def test_no_dust_recipient(spec, state): yield from run_transfer_processing(spec, state, transfer, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_invalid_pubkey(spec, state): transfer = get_valid_transfer(spec, state, signed=True) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py index 4b3f7446cc..3359c5e789 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_voluntary_exit.py @@ -1,4 +1,4 @@ -from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_phases +from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases from eth2spec.test.helpers.keys import pubkey_to_privkey from eth2spec.test.helpers.voluntary_exits import build_voluntary_exit, sign_voluntary_exit @@ -31,7 +31,7 @@ def run_voluntary_exit_processing(spec, state, voluntary_exit, valid=True): assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_success(spec, state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit @@ -46,7 +46,7 @@ def test_success(spec, state): yield from run_voluntary_exit_processing(spec, state, voluntary_exit) -@with_phases(['phase0', 'phase1']) +@with_all_phases @always_bls @spec_state_test def test_invalid_signature(spec, state): @@ -62,7 +62,7 @@ def test_invalid_signature(spec, state): yield from run_voluntary_exit_processing(spec, state, voluntary_exit, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_success_exit_queue(spec, state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit @@ -114,7 +114,7 @@ def test_success_exit_queue(spec, state): ) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_validator_exit_in_future(spec, state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit @@ -138,7 +138,7 @@ def test_validator_exit_in_future(spec, state): yield from run_voluntary_exit_processing(spec, state, voluntary_exit, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_validator_invalid_validator_index(spec, state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow for exit @@ -162,7 +162,7 @@ def test_validator_invalid_validator_index(spec, state): yield from run_voluntary_exit_processing(spec, state, voluntary_exit, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_validator_not_active(spec, state): current_epoch = spec.get_current_epoch(state) @@ -184,7 +184,7 @@ def test_validator_not_active(spec, state): yield from run_voluntary_exit_processing(spec, state, voluntary_exit, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_validator_already_exited(spec, state): # move state forward PERSISTENT_COMMITTEE_PERIOD epochs to allow validator able to exit @@ -209,7 +209,7 @@ def test_validator_already_exited(spec, state): yield from run_voluntary_exit_processing(spec, state, voluntary_exit, False) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_validator_not_active_long_enough(spec, state): current_epoch = spec.get_current_epoch(state) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py index 65bb6a0552..65d9586788 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py @@ -1,6 +1,6 @@ from copy import deepcopy -from eth2spec.test.context import spec_state_test, with_phases +from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.helpers.state import ( next_epoch, next_slot @@ -37,7 +37,7 @@ def run_process_crosslinks(spec, state, valid=True): yield 'post', state -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_no_attestations(spec, state): yield from run_process_crosslinks(spec, state) @@ -46,7 +46,7 @@ def test_no_attestations(spec, state): assert state.previous_crosslinks[shard] == state.current_crosslinks[shard] -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_single_crosslink_update_from_current_epoch(spec, state): next_epoch(spec, state) @@ -67,7 +67,7 @@ def test_single_crosslink_update_from_current_epoch(spec, state): assert pre_crosslink != state.current_crosslinks[shard] -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_single_crosslink_update_from_previous_epoch(spec, state): next_epoch(spec, state) @@ -98,7 +98,7 @@ def test_single_crosslink_update_from_previous_epoch(spec, state): assert crosslink_deltas[1][index] == 0 -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_double_late_crosslink(spec, state): if spec.get_epoch_committee_count(state, spec.get_current_epoch(state)) < spec.SHARD_COUNT: diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py index daa52dba99..e6679f8448 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py @@ -1,7 +1,7 @@ from eth2spec.phase0.spec import state_transition from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block from eth2spec.test.helpers.state import next_epoch -from eth2spec.test.context import spec_state_test, with_phases +from eth2spec.test.context import spec_state_test, with_all_phases def run_process_registry_updates(spec, state, valid=True): @@ -31,7 +31,7 @@ def run_process_registry_updates(spec, state, valid=True): yield 'post', state -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_activation(spec, state): index = 0 @@ -56,7 +56,7 @@ def test_activation(spec, state): ) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_ejection(spec, state): index = 0 diff --git a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py index f028217c63..110231d77e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py +++ b/test_libs/pyspec/eth2spec/test/phase_1/block_processing/test_process_early_derived_secret_reveal.py @@ -1,7 +1,7 @@ from eth2spec.test.helpers.custody import get_valid_early_derived_secret_reveal from eth2spec.test.helpers.block import apply_empty_block from eth2spec.test.helpers.state import next_epoch, get_balance -from eth2spec.test.context import with_phases, spec_state_test, expect_assertion_error +from eth2spec.test.context import with_all_phases_except, spec_state_test, expect_assertion_error def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, valid=True): @@ -35,7 +35,7 @@ def run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, v yield 'post', state -@with_phases(['phase1']) +@with_all_phases_except(['phase0']) @spec_state_test def test_success(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state) @@ -43,7 +43,7 @@ def test_success(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal) -@with_phases(['phase1']) +@with_all_phases_except(['phase0']) @spec_state_test def test_reveal_from_current_epoch(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state)) @@ -51,7 +51,7 @@ def test_reveal_from_current_epoch(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -@with_phases(['phase1']) +@with_all_phases_except(['phase0']) @spec_state_test def test_reveal_from_past_epoch(spec, state): next_epoch(spec, state) @@ -61,7 +61,7 @@ def test_reveal_from_past_epoch(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -@with_phases(['phase1']) +@with_all_phases_except(['phase0']) @spec_state_test def test_reveal_with_custody_padding(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal( @@ -72,7 +72,7 @@ def test_reveal_with_custody_padding(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, True) -@with_phases(['phase1']) +@with_all_phases_except(['phase0']) @spec_state_test def test_reveal_with_custody_padding_minus_one(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal( @@ -83,7 +83,7 @@ def test_reveal_with_custody_padding_minus_one(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, True) -@with_phases(['phase1']) +@with_all_phases_except(['phase0']) @spec_state_test def test_double_reveal(spec, state): randao_key_reveal1 = get_valid_early_derived_secret_reveal( @@ -107,7 +107,7 @@ def test_double_reveal(spec, state): yield 'post', post_state -@with_phases(['phase1']) +@with_all_phases_except(['phase0']) @spec_state_test def test_revealer_is_slashed(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal(spec, state, spec.get_current_epoch(state)) @@ -116,7 +116,7 @@ def test_revealer_is_slashed(spec, state): yield from run_early_derived_secret_reveal_processing(spec, state, randao_key_reveal, False) -@with_phases(['phase1']) +@with_all_phases_except(['phase0']) @spec_state_test def test_far_future_epoch(spec, state): randao_key_reveal = get_valid_early_derived_secret_reveal( diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index c08dd83cdd..587c377429 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -13,10 +13,10 @@ from eth2spec.test.helpers.attestations import get_valid_attestation from eth2spec.test.helpers.deposits import prepare_state_and_deposit -from eth2spec.test.context import spec_state_test, never_bls, with_phases +from eth2spec.test.context import spec_state_test, never_bls, with_all_phases -@with_phases(['phase0', 'phase1']) +@with_all_phases @never_bls @spec_state_test def test_empty_block_transition(spec, state): @@ -35,7 +35,7 @@ def test_empty_block_transition(spec, state): assert spec.get_block_root_at_slot(state, pre_slot) == block.parent_root -@with_phases(['phase0', 'phase1']) +@with_all_phases @never_bls @spec_state_test def test_skipped_slots(spec, state): @@ -55,7 +55,7 @@ def test_skipped_slots(spec, state): assert spec.get_block_root_at_slot(state, slot) == block.parent_root -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_empty_epoch_transition(spec, state): pre_slot = state.slot @@ -74,7 +74,7 @@ def test_empty_epoch_transition(spec, state): assert spec.get_block_root_at_slot(state, slot) == block.parent_root -# @with_phases(['phase0', 'phase1']) +# @with_all_phases # @spec_state_test # def test_empty_epoch_transition_not_finalizing(spec, state): # # copy for later balance lookups. @@ -95,7 +95,7 @@ def test_empty_epoch_transition(spec, state): # assert get_balance(state, index) < get_balance(pre_state, index) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_proposer_slashing(spec, state): # copy for later balance lookups. @@ -127,7 +127,7 @@ def test_proposer_slashing(spec, state): assert get_balance(state, validator_index) < get_balance(pre_state, validator_index) -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_attester_slashing(spec, state): # copy for later balance lookups. @@ -169,7 +169,7 @@ def test_attester_slashing(spec, state): # TODO update functions below to be like above, i.e. with @spec_state_test and yielding data to put into the test vector -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_deposit_in_block(spec, state): initial_registry_len = len(state.validator_registry) @@ -196,7 +196,7 @@ def test_deposit_in_block(spec, state): assert state.validator_registry[validator_index].pubkey == pubkeys[validator_index] -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_deposit_top_up(spec, state): validator_index = 0 @@ -223,7 +223,7 @@ def test_deposit_top_up(spec, state): assert get_balance(state, validator_index) == validator_pre_balance + amount -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_attestation(spec, state): state.slot = spec.SLOTS_PER_EPOCH @@ -257,7 +257,7 @@ def test_attestation(spec, state): assert spec.hash_tree_root(state.previous_epoch_attestations) == pre_current_attestations_root -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_voluntary_exit(spec, state): validator_index = spec.get_active_validator_indices( @@ -303,7 +303,7 @@ def test_voluntary_exit(spec, state): assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_transfer(spec, state): # overwrite default 0 to test @@ -337,7 +337,7 @@ def test_transfer(spec, state): assert recipient_balance == pre_transfer_recipient_balance + amount -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_balance_driven_status_transitions(spec, state): current_epoch = spec.get_current_epoch(state) @@ -362,7 +362,7 @@ def test_balance_driven_status_transitions(spec, state): assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_historical_batch(spec, state): state.slot += spec.SLOTS_PER_HISTORICAL_ROOT - (state.slot % spec.SLOTS_PER_HISTORICAL_ROOT) - 1 @@ -381,7 +381,7 @@ def test_historical_batch(spec, state): assert len(state.historical_roots) == pre_historical_roots_len + 1 -# @with_phases(['phase0', 'phase1']) +# @with_all_phases # @spec_state_test # def test_eth1_data_votes(spec, state): # yield 'pre', state diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_slots.py b/test_libs/pyspec/eth2spec/test/sanity/test_slots.py index feddd65be3..6ef6be4d38 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_slots.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_slots.py @@ -1,8 +1,8 @@ from eth2spec.test.helpers.state import get_state_root -from eth2spec.test.context import spec_state_test, with_phases +from eth2spec.test.context import spec_state_test, with_all_phases -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_slots_1(spec, state): pre_slot = state.slot @@ -18,7 +18,7 @@ def test_slots_1(spec, state): assert get_state_root(spec, state, pre_slot) == pre_root -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_slots_2(spec, state): yield 'pre', state @@ -28,7 +28,7 @@ def test_slots_2(spec, state): yield 'post', state -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_empty_epoch(spec, state): yield 'pre', state @@ -38,7 +38,7 @@ def test_empty_epoch(spec, state): yield 'post', state -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_double_empty_epoch(spec, state): yield 'pre', state @@ -48,7 +48,7 @@ def test_double_empty_epoch(spec, state): yield 'post', state -@with_phases(['phase0', 'phase1']) +@with_all_phases @spec_state_test def test_over_epoch_boundary(spec, state): spec.process_slots(state, state.slot + (spec.SLOTS_PER_EPOCH // 2)) diff --git a/test_libs/pyspec/eth2spec/test/test_finality.py b/test_libs/pyspec/eth2spec/test/test_finality.py index 2d2e01432b..801e8b4fd5 100644 --- a/test_libs/pyspec/eth2spec/test/test_finality.py +++ b/test_libs/pyspec/eth2spec/test/test_finality.py @@ -1,7 +1,7 @@ from copy import deepcopy from typing import List -from eth2spec.test.context import spec_state_test, never_bls, with_phases +from eth2spec.test.context import spec_state_test, never_bls, with_all_phases from eth2spec.test.helpers.state import next_epoch from eth2spec.test.helpers.block import build_empty_block_for_next_slot, apply_empty_block from eth2spec.test.helpers.attestations import get_valid_attestation @@ -60,7 +60,7 @@ def next_epoch_with_attestations(spec, return state, blocks, post_state -@with_phases(['phase0', 'phase1']) +@with_all_phases @never_bls @spec_state_test def test_finality_rule_4(spec, state): @@ -89,7 +89,7 @@ def test_finality_rule_4(spec, state): yield 'post', state -@with_phases(['phase0', 'phase1']) +@with_all_phases @never_bls @spec_state_test def test_finality_rule_1(spec, state): @@ -120,7 +120,7 @@ def test_finality_rule_1(spec, state): yield 'post', state -@with_phases(['phase0', 'phase1']) +@with_all_phases @never_bls @spec_state_test def test_finality_rule_2(spec, state): @@ -153,7 +153,7 @@ def test_finality_rule_2(spec, state): yield 'post', state -@with_phases(['phase0', 'phase1']) +@with_all_phases @never_bls @spec_state_test def test_finality_rule_3(spec, state): From cf23a252b4a0f96f4f313f9d2bd55160297df1f2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 7 Jun 2019 15:35:52 -0400 Subject: [PATCH 285/308] fix path --- deposit_contract/tests/contracts/test_deposit.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/deposit_contract/tests/contracts/test_deposit.py b/deposit_contract/tests/contracts/test_deposit.py index 3d0b6f8453..8492d63478 100644 --- a/deposit_contract/tests/contracts/test_deposit.py +++ b/deposit_contract/tests/contracts/test_deposit.py @@ -1,6 +1,3 @@ -from hashlib import ( - sha256, -) from random import ( randint, ) @@ -18,8 +15,7 @@ DepositData, ) from eth2spec.utils.hash_function import hash -from eth2spec.utils.minimal_ssz import ( - SSZType, +from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, ) From 96237c74e1a7f687ba3fc2481ac670e3156b9e67 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 7 Jun 2019 15:30:27 -0400 Subject: [PATCH 286/308] two venv caches --- .circleci/config.yml | 61 ++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e0fac0d40..91578f8617 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,18 +31,31 @@ commands: - save_cache: key: << parameters.venv_name >>-venv-<< parameters.reqs_checksum >> paths: << parameters.venv_path >> - restore_default_cached_venv: - description: "Restore the cache with default keys" + restore_pyspec_cached_venv: + description: "Restore the cache with pyspec keys" steps: - restore_cached_venv: venv_name: v2-pyspec - reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} - save_default_cached_venv: - description: Save a venv into a cache with default keys" + reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} + save_pyspec_cached_venv: + description: Save a venv into a cache with pyspec keys" steps: - save_cached_venv: venv_name: v2-pyspec - reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} + reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }} + venv_path: ./test_libs/pyspec/venv + restore_deposit_contract_cached_venv: + description: "Restore the cache with deposit_contract keys" + steps: + - restore_cached_venv: + venv_name: v2-deposit-contract + reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} + save_deposit_contract_cached_venv: + description: Save a venv into a cache with deposit_contract keys" + steps: + - save_cached_venv: + venv_name: v2-deposit-contract + reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./test_libs/pyspec/venv jobs: checkout_specs: @@ -65,18 +78,18 @@ jobs: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} paths: - ~/specs-repo - install_env: + install_pyspec_test: docker: - image: circleci/python:3.6 working_directory: ~/specs-repo steps: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_default_cached_venv + - restore_pyspec_cached_venv - run: name: Install pyspec requirements - command: make install_test && make install_lint && make install_deposit_contract_test - - save_default_cached_venv + command: make install_test + - save_pyspec_cached_venv test: docker: - image: circleci/python:3.6 @@ -84,7 +97,7 @@ jobs: steps: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_default_cached_venv + - restore_pyspec_cached_venv - run: name: Run py-tests command: make citest @@ -97,10 +110,22 @@ jobs: steps: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_default_cached_venv + - restore_pyspec_cached_venv - run: name: Run linter command: make lint + install_deposit_contract_test: + docker: + - image: circleci/python:3.6 + working_directory: ~/specs-repo + steps: + - restore_cache: + key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} + - restore_deposit_contract_cached_venv + - run: + name: Install deposit contract requirements + command: make install_deposit_contract_test + - save_deposit_contract_cached_venv deposit_contract: docker: - image: circleci/python:3.6 @@ -108,7 +133,7 @@ jobs: steps: - restore_cache: key: v1-specs-repo-{{ .Branch }}-{{ .Revision }} - - restore_default_cached_venv + - restore_deposit_contract_cached_venv - run: name: Run deposit contract test command: make test_deposit_contract @@ -117,16 +142,18 @@ workflows: test_spec: jobs: - checkout_specs - - install_env: + - install_pyspec_test: requires: - checkout_specs - test: requires: - - install_env + - install_pyspec_test - lint: requires: - test + - install_deposit_contract_test: + requires: + - checkout_specs - deposit_contract: requires: - - install_env - - lint + - install_deposit_contract_test From 5b8cca831458be07ec8e9a0ca580d60354bfd024 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 7 Jun 2019 22:19:11 -0400 Subject: [PATCH 287/308] deposit_contract/venv PR feedback + bump eth-tester --- .circleci/config.yml | 6 +++--- Makefile | 15 ++++++--------- deposit_contract/README.md | 13 +++++++------ deposit_contract/requirements-testing.txt | 5 ++--- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 91578f8617..6b785e6ed7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,15 +48,15 @@ commands: description: "Restore the cache with deposit_contract keys" steps: - restore_cached_venv: - venv_name: v2-deposit-contract + venv_name: v3-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_deposit_contract_cached_venv: description: Save a venv into a cache with deposit_contract keys" steps: - save_cached_venv: - venv_name: v2-deposit-contract + venv_name: v3-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} - venv_path: ./test_libs/pyspec/venv + venv_path: ./deposit_contract/venv jobs: checkout_specs: docker: diff --git a/Makefile b/Makefile index 66c5ba3ec4..12b8d79fb9 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,7 @@ clean: rm -rf $(GENERATOR_VENVS) rm -rf $(PY_SPEC_DIR)/venv $(PY_SPEC_DIR)/.pytest_cache rm -rf $(PY_SPEC_ALL_TARGETS) + rm -rf $(DEPOSIT_CONTRACT_DIR)/venv $(DEPOSIT_CONTRACT_DIR)/.pytest_cache # "make gen_yaml_tests" to run generators gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS) @@ -50,19 +51,15 @@ lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ flake8 --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec; +install_deposit_contract_test: + cd $(DEPOSIT_CONTRACT_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt + compile_deposit_contract: - cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; \ - cd ../..; cd $(DEPOSIT_CONTRACT_DIR); \ + cd $(DEPOSIT_CONTRACT_DIR); . venv/bin/activate; \ python tool/compile_deposit_contract.py contracts/validator_registration.v.py; -install_deposit_contract_test: - cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; \ - cd ../..; cd $(DEPOSIT_CONTRACT_DIR); \ - pip3 install -r requirements-testing.txt - test_deposit_contract: $(PY_SPEC_ALL_TARGETS) - cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; \ - cd ../.. && cd $(DEPOSIT_CONTRACT_DIR); \ + cd $(DEPOSIT_CONTRACT_DIR); . venv/bin/activate; \ pip3 install -r requirements-testing.txt; \ python -m pytest . diff --git a/deposit_contract/README.md b/deposit_contract/README.md index 266eb93e42..16779e7778 100644 --- a/deposit_contract/README.md +++ b/deposit_contract/README.md @@ -1,21 +1,22 @@ # Deposit contract -## How to compile the contract? +## How to set up the testing environment? Under the `eth2.0-specs` directory, execute: ```sh -make compile_deposit_contract +make install_deposit_contract_test ``` -The ABI and bytecode will be updated at [`contracts/validator_registration.json`](./contracts/validator_registration.json). - -## How to set up the testing environment? +## How to compile the contract? ```sh -make install_deposit_contract_test +make compile_deposit_contract ``` +The ABI and bytecode will be updated at [`contracts/validator_registration.json`](./contracts/validator_registration.json). + + ## How to run tests? ```sh diff --git a/deposit_contract/requirements-testing.txt b/deposit_contract/requirements-testing.txt index c1f96a0120..b3a90a88a7 100644 --- a/deposit_contract/requirements-testing.txt +++ b/deposit_contract/requirements-testing.txt @@ -1,6 +1,5 @@ -tox==3.0.0 -eth-tester[py-evm]==0.1.0b29 +eth-tester[py-evm]==0.1.0b39 vyper==0.1.0b9 -web3==4.8.3 +web3==5.0.0b2 pytest==3.6.1 ../test_libs/pyspec From e52c4a5526866deba675ef11ac6c4cf8499dba65 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 7 Jun 2019 23:00:28 -0400 Subject: [PATCH 288/308] fix --- .circleci/config.yml | 4 ++-- Makefile | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b785e6ed7..6e9a77c49b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,13 +48,13 @@ commands: description: "Restore the cache with deposit_contract keys" steps: - restore_cached_venv: - venv_name: v3-deposit-contract + venv_name: v4-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} save_deposit_contract_cached_venv: description: Save a venv into a cache with deposit_contract keys" steps: - save_cached_venv: - venv_name: v3-deposit-contract + venv_name: v4-deposit-contract reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "deposit_contract/requirements-testing.txt" }} venv_path: ./deposit_contract/venv jobs: diff --git a/Makefile b/Makefile index 12b8d79fb9..ab0806c246 100644 --- a/Makefile +++ b/Makefile @@ -51,16 +51,15 @@ lint: $(PY_SPEC_ALL_TARGETS) cd $(PY_SPEC_DIR); . venv/bin/activate; \ flake8 --ignore=E252,W504,W503 --max-line-length=120 ./eth2spec; -install_deposit_contract_test: +install_deposit_contract_test: $(PY_SPEC_ALL_TARGETS) cd $(DEPOSIT_CONTRACT_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt compile_deposit_contract: cd $(DEPOSIT_CONTRACT_DIR); . venv/bin/activate; \ python tool/compile_deposit_contract.py contracts/validator_registration.v.py; -test_deposit_contract: $(PY_SPEC_ALL_TARGETS) +test_deposit_contract: cd $(DEPOSIT_CONTRACT_DIR); . venv/bin/activate; \ - pip3 install -r requirements-testing.txt; \ python -m pytest . # "make pyspec" to create the pyspec for all phases. From bce6c899f65ad05587a1067c235251a5c3fff9fd Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 8 Jun 2019 13:30:47 +0200 Subject: [PATCH 289/308] improve makefile: declare new targets as non-file --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ab0806c246..f79b89dada 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ PY_SPEC_PHASE_1_DEPS = $(SPEC_DIR)/core/1_*.md PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) $(PY_SPEC_PHASE_1_TARGETS) -.PHONY: clean all test citest gen_yaml_tests pyspec phase0 phase1 install_test +.PHONY: clean all test citest lint gen_yaml_tests pyspec phase0 phase1 install_test install_deposit_contract_test test_deposit_contract compile_deposit_contract all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS) From 11f2cd189a333613201eb35649095365ba5d4406 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 8 Jun 2019 18:48:35 -0400 Subject: [PATCH 290/308] Fix the Vyper contract link (#1154) --- specs/core/0_deposit-contract.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_deposit-contract.md b/specs/core/0_deposit-contract.md index 5d93985894..e80dad1c50 100644 --- a/specs/core/0_deposit-contract.md +++ b/specs/core/0_deposit-contract.md @@ -82,7 +82,7 @@ When `CHAIN_START_FULL_DEPOSIT_THRESHOLD` of full deposits have been made, the d ## Vyper code -The source for the Vyper contract lives in a [separate repository](https://github.com/ethereum/deposit_contract) at [https://github.com/ethereum/deposit_contract/blob/master/deposit_contract/contracts/validator_registration.v.py](https://github.com/ethereum/deposit_contract/blob/master/deposit_contract/contracts/validator_registration.v.py). +The source for the Vyper contract lives [here](./../../deposit_contract/contracts/validator_registration.v.py). *Note*: To save ~10x on gas, this contract uses a somewhat unintuitive progressive Merkle root calculation algo that requires only O(log(n)) storage. See https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py for an implementation of the same algo in Python tested for correctness. From 8b54c90fd6c6111bf6c027b2009e89d99c8a6d83 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Sun, 9 Jun 2019 12:09:54 -0700 Subject: [PATCH 291/308] Update 0_beacon-chain-validator.md --- specs/validator/0_beacon-chain-validator.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index 2f5aa4264b..e7a3a1a9de 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -297,11 +297,11 @@ Set `attestation_data.beacon_block_root = signing_root(head_block)`. * Set `attestation_data.source_epoch = head_state.current_justified_epoch`. * Set `attestation_data.source_root = head_state.current_justified_root`. * Set `attestation_data.target_epoch = get_current_epoch(head_state)` -* Set `attestation_data.target_root = signing_root(epoch_boundary_block)` where `epoch_boundary_block` is the block at the most recent epoch boundary. +* Set `attestation_data.target_root = epoch_boundary_block_root` where `epoch_boundary_block_root` is the root of block at the most recent epoch boundary. -*Note*: `epoch_boundary_block` can be looked up in the state using: +*Note*: `epoch_boundary_block_root` can be looked up in the state using: * Let `epoch_start_slot = get_epoch_start_slot(get_current_epoch(head_state))`. -* Let `epoch_boundary_block = head if epoch_start_slot == head_state.slot else get_block_root(state, epoch_start_slot)`. +* Let `epoch_boundary_block_root = signing_root(head_block) if epoch_start_slot == head_state.slot else get_block_root(state, epoch_start_slot)`. ##### Crosslink vote From b56539a47d31d40a8e0f07d7915b9fa9a65edd9c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 10 Jun 2019 15:14:42 -0600 Subject: [PATCH 292/308] add missing colon to v-guide --- specs/validator/0_beacon-chain-validator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/validator/0_beacon-chain-validator.md b/specs/validator/0_beacon-chain-validator.md index e7a3a1a9de..29724da514 100644 --- a/specs/validator/0_beacon-chain-validator.md +++ b/specs/validator/0_beacon-chain-validator.md @@ -147,7 +147,7 @@ def get_committee_assignment( committees_per_slot = get_epoch_committee_count(state, epoch) // SLOTS_PER_EPOCH epoch_start_slot = get_epoch_start_slot(epoch) - for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH) + for slot in range(epoch_start_slot, epoch_start_slot + SLOTS_PER_EPOCH): offset = committees_per_slot * (slot % SLOTS_PER_EPOCH) slot_start_shard = (get_epoch_start_shard(state, epoch) + offset) % SHARD_COUNT for i in range(committees_per_slot): From c4bf772d30b4a557c484e9527c2c670cdbacfe89 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 10 Jun 2019 17:26:39 -0600 Subject: [PATCH 293/308] demo phase restricted generators for epoch processing --- test_generators/epoch_processing/main.py | 10 +++++----- test_libs/gen_helpers/gen_from_tests/gen.py | 7 ++++--- test_libs/pyspec/eth2spec/test/context.py | 13 +++++++++++-- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/test_generators/epoch_processing/main.py b/test_generators/epoch_processing/main.py index 2ce895fc5b..679ef4c6ba 100644 --- a/test_generators/epoch_processing/main.py +++ b/test_generators/epoch_processing/main.py @@ -2,7 +2,7 @@ from eth2spec.phase0 import spec as spec_phase0 from eth2spec.phase1 import spec as spec_phase1 -from eth2spec.test.epoch_processing import ( +from eth2spec.test.phase_0.epoch_processing import ( test_process_crosslinks, test_process_registry_updates ) @@ -33,8 +33,8 @@ def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: if __name__ == "__main__": gen_runner.run_generator("epoch_processing", [ - create_suite('crosslinks', 'minimal', lambda: generate_from_tests(test_process_crosslinks)), - create_suite('crosslinks', 'mainnet', lambda: generate_from_tests(test_process_crosslinks)), - create_suite('registry_updates', 'minimal', lambda: generate_from_tests(test_process_registry_updates)), - create_suite('registry_updates', 'mainnet', lambda: generate_from_tests(test_process_registry_updates)), + create_suite('crosslinks', 'minimal', lambda: generate_from_tests(test_process_crosslinks, 'phase0')), + create_suite('crosslinks', 'mainnet', lambda: generate_from_tests(test_process_crosslinks, 'phase0')), + create_suite('registry_updates', 'minimal', lambda: generate_from_tests(test_process_registry_updates, 'phase0')), + create_suite('registry_updates', 'mainnet', lambda: generate_from_tests(test_process_registry_updates, 'phase0')), ]) diff --git a/test_libs/gen_helpers/gen_from_tests/gen.py b/test_libs/gen_helpers/gen_from_tests/gen.py index e7d8011310..3810c385e0 100644 --- a/test_libs/gen_helpers/gen_from_tests/gen.py +++ b/test_libs/gen_helpers/gen_from_tests/gen.py @@ -1,9 +1,10 @@ from inspect import getmembers, isfunction -def generate_from_tests(src, bls_active=True): +def generate_from_tests(src, phase, bls_active=True): """ Generate a list of test cases by running tests from the given src in generator-mode. - :param src: to retrieve tests from (discovered using inspect.getmembers) + :param src: to retrieve tests from (discovered using inspect.getmembers). + :param phase: to run tests against particular phase. :param bls_active: optional, to override BLS switch preference. Defaults to True. :return: the list of test cases. """ @@ -16,7 +17,7 @@ def generate_from_tests(src, bls_active=True): for name in fn_names: tfn = getattr(src, name) try: - test_case = tfn(generator_mode=True, bls_active=bls_active) + test_case = tfn(generator_mode=True, phase=phase, bls_active=bls_active) # If no test case data is returned, the test is ignored. if test_case is not None: out.append(test_case) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index cbc594cd8e..520eb9745a 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -119,9 +119,18 @@ def run_with_spec_version(spec, *args, **kw): fn(*args, **kw) def wrapper(*args, **kw): - if 'phase0' in phases: + run_phases = phases + + # limit phases if one explicitly specified + if 'phase' in kw: + phase = kw.pop('phase') + if phase not in phases: + return + run_phases = [phase] + + if 'phase0' in run_phases: run_with_spec_version(spec_phase0, *args, **kw) - if 'phase1' in phases: + if 'phase1' in run_phases: run_with_spec_version(spec_phase1, *args, **kw) return wrapper return decorator From 94d4e3a94402f65a58a23d434f40d021c4f80259 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 10 Jun 2019 20:05:43 -0600 Subject: [PATCH 294/308] operations suite generator for phase0 --- test_generators/epoch_processing/main.py | 6 +++-- test_generators/operations/main.py | 32 +++++++++++------------ test_libs/pyspec/eth2spec/test/context.py | 7 ++--- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/test_generators/epoch_processing/main.py b/test_generators/epoch_processing/main.py index 679ef4c6ba..846f463a18 100644 --- a/test_generators/epoch_processing/main.py +++ b/test_generators/epoch_processing/main.py @@ -35,6 +35,8 @@ def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: gen_runner.run_generator("epoch_processing", [ create_suite('crosslinks', 'minimal', lambda: generate_from_tests(test_process_crosslinks, 'phase0')), create_suite('crosslinks', 'mainnet', lambda: generate_from_tests(test_process_crosslinks, 'phase0')), - create_suite('registry_updates', 'minimal', lambda: generate_from_tests(test_process_registry_updates, 'phase0')), - create_suite('registry_updates', 'mainnet', lambda: generate_from_tests(test_process_registry_updates, 'phase0')), + create_suite('registry_updates', 'minimal', + lambda: generate_from_tests(test_process_registry_updates, 'phase0')), + create_suite('registry_updates', 'mainnet', + lambda: generate_from_tests(test_process_registry_updates, 'phase0')), ]) diff --git a/test_generators/operations/main.py b/test_generators/operations/main.py index 82e05b3075..38fa42f689 100644 --- a/test_generators/operations/main.py +++ b/test_generators/operations/main.py @@ -1,13 +1,13 @@ from typing import Callable, Iterable -from eth2spec.test.block_processing import ( +from eth2spec.test.phase_0.block_processing import ( test_process_attestation, test_process_attester_slashing, test_process_block_header, test_process_deposit, test_process_proposer_slashing, test_process_transfer, - test_process_voluntary_exit + test_process_voluntary_exit, ) from gen_base import gen_runner, gen_suite, gen_typing @@ -38,18 +38,18 @@ def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: if __name__ == "__main__": gen_runner.run_generator("operations", [ - create_suite('attestation', 'minimal', lambda: generate_from_tests(test_process_attestation)), - create_suite('attestation', 'mainnet', lambda: generate_from_tests(test_process_attestation)), - create_suite('attester_slashing', 'minimal', lambda: generate_from_tests(test_process_attester_slashing)), - create_suite('attester_slashing', 'mainnet', lambda: generate_from_tests(test_process_attester_slashing)), - create_suite('block_header', 'minimal', lambda: generate_from_tests(test_process_block_header)), - create_suite('block_header', 'mainnet', lambda: generate_from_tests(test_process_block_header)), - create_suite('deposit', 'minimal', lambda: generate_from_tests(test_process_deposit)), - create_suite('deposit', 'mainnet', lambda: generate_from_tests(test_process_deposit)), - create_suite('proposer_slashing', 'minimal', lambda: generate_from_tests(test_process_proposer_slashing)), - create_suite('proposer_slashing', 'mainnet', lambda: generate_from_tests(test_process_proposer_slashing)), - create_suite('transfer', 'minimal', lambda: generate_from_tests(test_process_transfer)), - create_suite('transfer', 'mainnet', lambda: generate_from_tests(test_process_transfer)), - create_suite('voluntary_exit', 'minimal', lambda: generate_from_tests(test_process_voluntary_exit)), - create_suite('voluntary_exit', 'mainnet', lambda: generate_from_tests(test_process_voluntary_exit)), + create_suite('attestation', 'minimal', lambda: generate_from_tests(test_process_attestation, 'phase0')), + create_suite('attestation', 'mainnet', lambda: generate_from_tests(test_process_attestation, 'phase0')), + create_suite('attester_slashing', 'minimal', lambda: generate_from_tests(test_process_attester_slashing, 'phase0')), + create_suite('attester_slashing', 'mainnet', lambda: generate_from_tests(test_process_attester_slashing, 'phase0')), + create_suite('block_header', 'minimal', lambda: generate_from_tests(test_process_block_header, 'phase0')), + create_suite('block_header', 'mainnet', lambda: generate_from_tests(test_process_block_header, 'phase0')), + create_suite('deposit', 'minimal', lambda: generate_from_tests(test_process_deposit, 'phase0')), + create_suite('deposit', 'mainnet', lambda: generate_from_tests(test_process_deposit, 'phase0')), + create_suite('proposer_slashing', 'minimal', lambda: generate_from_tests(test_process_proposer_slashing, 'phase0')), + create_suite('proposer_slashing', 'mainnet', lambda: generate_from_tests(test_process_proposer_slashing, 'phase0')), + create_suite('transfer', 'minimal', lambda: generate_from_tests(test_process_transfer, 'phase0')), + create_suite('transfer', 'mainnet', lambda: generate_from_tests(test_process_transfer, 'phase0')), + create_suite('voluntary_exit', 'minimal', lambda: generate_from_tests(test_process_voluntary_exit, 'phase0')), + create_suite('voluntary_exit', 'mainnet', lambda: generate_from_tests(test_process_voluntary_exit, 'phase0')), ]) diff --git a/test_libs/pyspec/eth2spec/test/context.py b/test_libs/pyspec/eth2spec/test/context.py index 520eb9745a..97266acf2c 100644 --- a/test_libs/pyspec/eth2spec/test/context.py +++ b/test_libs/pyspec/eth2spec/test/context.py @@ -116,7 +116,7 @@ def with_phases(phases): def decorator(fn): def run_with_spec_version(spec, *args, **kw): kw['spec'] = spec - fn(*args, **kw) + return fn(*args, **kw) def wrapper(*args, **kw): run_phases = phases @@ -129,8 +129,9 @@ def wrapper(*args, **kw): run_phases = [phase] if 'phase0' in run_phases: - run_with_spec_version(spec_phase0, *args, **kw) + ret = run_with_spec_version(spec_phase0, *args, **kw) if 'phase1' in run_phases: - run_with_spec_version(spec_phase1, *args, **kw) + ret = run_with_spec_version(spec_phase1, *args, **kw) + return ret return wrapper return decorator From 46d8422510e5235cecb9da08982dc5c8c77f8920 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 16:42:50 +0200 Subject: [PATCH 295/308] fix ssz container recognition for generators --- test_libs/pyspec/eth2spec/test/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/utils.py b/test_libs/pyspec/eth2spec/test/utils.py index b61801c3dd..817c952b7e 100644 --- a/test_libs/pyspec/eth2spec/test/utils.py +++ b/test_libs/pyspec/eth2spec/test/utils.py @@ -1,5 +1,6 @@ from typing import Dict, Any, Callable, Iterable from eth2spec.debug.encode import encode +from eth2spec.utils.ssz.ssz_typing import Container def spectest(description: str = None): @@ -30,9 +31,13 @@ def entry(*args, **kw): else: # Otherwise, try to infer the type, but keep it as-is if it's not a SSZ container. (key, value) = data - if hasattr(value.__class__, 'fields'): + if isinstance(value, Container): out[key] = encode(value, value.__class__) else: + # not a ssz value. + # It could be vector or bytes still, but it is a rare case, + # and lists can't be inferred fully (generics lose element type). + # In such cases, explicitly state the type of the yielded value as a third yielded object. out[key] = value if has_contents: return out From 481c9664bee8746a9eb7cb5388ffcf314e05fcda Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 17:07:48 +0200 Subject: [PATCH 296/308] update naming of 2 constants, fix comments in minimal config (mainnet time estimates do not hold with different amount of slots per epoch) --- configs/constant_presets/mainnet.yaml | 4 ++-- configs/constant_presets/minimal.yaml | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index 6ac3f422f5..bca7febc72 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -48,7 +48,7 @@ GENESIS_FORK_VERSION: 0x00000000 GENESIS_SLOT: 0 # 2**64 - 1 FAR_FUTURE_EPOCH: 18446744073709551615 -BLS_WITHDRAWAL_PREFIX_BYTE: 0x00 +BLS_WITHDRAWAL_PREFIX: 0x00 # Time parameters @@ -90,7 +90,7 @@ LATEST_SLASHED_EXIT_LENGTH: 8192 # Reward and penalty quotients # --------------------------------------------------------------- # 2**5 (= 32) -BASE_REWARD_QUOTIENT: 32 +BASE_REWARD_FACTOR: 32 # 2**9 (= 512) WHISTLEBLOWING_REWARD_QUOTIENT: 512 # 2**3 (= 8) diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 73448c3c60..23bc6e8193 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -47,7 +47,7 @@ GENESIS_FORK_VERSION: 0x00000000 GENESIS_SLOT: 0 # 2**64 - 1 FAR_FUTURE_EPOCH: 18446744073709551615 -BLS_WITHDRAWAL_PREFIX_BYTE: 0x00 +BLS_WITHDRAWAL_PREFIX: 0x00 # Time parameters @@ -58,23 +58,23 @@ SECONDS_PER_SLOT: 6 MIN_ATTESTATION_INCLUSION_DELAY: 2 # [customized] fast epochs SLOTS_PER_EPOCH: 8 -# 2**0 (= 1) epochs 6.4 minutes +# 2**0 (= 1) epochs MIN_SEED_LOOKAHEAD: 1 -# 2**2 (= 4) epochs 25.6 minutes +# 2**2 (= 4) epochs ACTIVATION_EXIT_DELAY: 4 # [customized] higher frequency new deposits from eth1 for testing SLOTS_PER_ETH1_VOTING_PERIOD: 16 # [customized] smaller state SLOTS_PER_HISTORICAL_ROOT: 64 -# 2**8 (= 256) epochs ~27 hours +# 2**8 (= 256) epochs MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 -# 2**11 (= 2,048) epochs 9 days +# 2**11 (= 2,048) epochs PERSISTENT_COMMITTEE_PERIOD: 2048 -# 2**6 (= 64) epochs ~7 hours +# 2**6 (= 64) epochs MAX_EPOCHS_PER_CROSSLINK: 64 -# 2**2 (= 4) epochs 25.6 minutes +# 2**2 (= 4) epochs MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 -# [customized] 2**12 (= 4,096) epochs 18 days +# [customized] 2**12 (= 4,096) epochs EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 4096 @@ -91,7 +91,7 @@ LATEST_SLASHED_EXIT_LENGTH: 64 # Reward and penalty quotients # --------------------------------------------------------------- # 2**5 (= 32) -BASE_REWARD_QUOTIENT: 32 +BASE_REWARD_FACTOR: 32 # 2**9 (= 512) WHISTLEBLOWING_REWARD_QUOTIENT: 512 # 2**3 (= 8) From 98a798bd54d7d1969cf84d32a2e6de073e4a68e0 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 11 Jun 2019 09:10:09 -0600 Subject: [PATCH 297/308] sanity tests gens to use explicit phase0 --- test_generators/sanity/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_generators/sanity/main.py b/test_generators/sanity/main.py index a9c0fe1603..805ee61394 100644 --- a/test_generators/sanity/main.py +++ b/test_generators/sanity/main.py @@ -30,8 +30,8 @@ def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: if __name__ == "__main__": gen_runner.run_generator("sanity", [ - create_suite('blocks', 'minimal', lambda: generate_from_tests(test_blocks)), - create_suite('blocks', 'mainnet', lambda: generate_from_tests(test_blocks)), - create_suite('slots', 'minimal', lambda: generate_from_tests(test_slots)), - create_suite('slots', 'mainnet', lambda: generate_from_tests(test_slots)), + create_suite('blocks', 'minimal', lambda: generate_from_tests(test_blocks, 'phase0')), + create_suite('blocks', 'mainnet', lambda: generate_from_tests(test_blocks, 'phase0')), + create_suite('slots', 'minimal', lambda: generate_from_tests(test_slots, 'phase0')), + create_suite('slots', 'mainnet', lambda: generate_from_tests(test_slots, 'phase0')), ]) From e86771250b4fd34d4a1aa95369b3fd2c12ee9792 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 17:16:02 +0200 Subject: [PATCH 298/308] Fix tests to use new constant name --- test_libs/pyspec/eth2spec/test/helpers/deposits.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/transfers.py | 2 +- .../test/phase_0/block_processing/test_process_deposit.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index 8f437ec89e..b84970044a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -67,7 +67,7 @@ def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_c # insecurely use pubkey as withdrawal key if no credentials provided if withdrawal_credentials is None: - withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(pubkey)[1:] + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(pubkey)[1:] deposit, root, deposit_data_leaves = build_deposit( spec, diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index 83af566211..23451b06b5 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -5,7 +5,7 @@ def build_mock_validator(spec, i: int, balance: int): pubkey = pubkeys[i] # insecurely use pubkey as withdrawal key as well - withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(pubkey)[1:] + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(pubkey)[1:] return spec.Validator( pubkey=pubkeys[i], withdrawal_credentials=withdrawal_credentials, diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index e619c5569d..eea12a36d4 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -32,7 +32,7 @@ def get_valid_transfer(spec, state, slot=None, sender_index=None, amount=None, f # ensure withdrawal_credentials reproducible state.validator_registry[transfer.sender].withdrawal_credentials = ( - spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(transfer.pubkey)[1:] + spec.BLS_WITHDRAWAL_PREFIX + spec.hash(transfer.pubkey)[1:] ) return transfer diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index c50b11f2ef..42de447ece 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -101,7 +101,7 @@ def test_invalid_sig_top_up(spec, state): def test_invalid_withdrawal_credentials_top_up(spec, state): validator_index = 0 amount = spec.MAX_EFFECTIVE_BALANCE // 4 - withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX_BYTE + spec.hash(b"junk")[1:] + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(b"junk")[1:] deposit = prepare_state_and_deposit( spec, state, From ef1ef753a3ecff39b8a85cca5c407e7ff17a359d Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 11 Jun 2019 09:16:37 -0600 Subject: [PATCH 299/308] comment out transfer test from sanity tests --- .../eth2spec/test/sanity/test_blocks.py | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 587c377429..593202dab2 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -303,38 +303,38 @@ def test_voluntary_exit(spec, state): assert state.validator_registry[validator_index].exit_epoch < spec.FAR_FUTURE_EPOCH -@with_all_phases -@spec_state_test -def test_transfer(spec, state): +# @with_all_phases +# @spec_state_test +# def test_transfer(spec, state): # overwrite default 0 to test - spec.MAX_TRANSFERS = 1 + # spec.MAX_TRANSFERS = 1 - sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] - amount = get_balance(state, sender_index) + # sender_index = spec.get_active_validator_indices(state, spec.get_current_epoch(state))[-1] + # amount = get_balance(state, sender_index) - transfer = get_valid_transfer(spec, state, state.slot + 1, sender_index, amount, signed=True) - recipient_index = transfer.recipient - pre_transfer_recipient_balance = get_balance(state, recipient_index) + # transfer = get_valid_transfer(spec, state, state.slot + 1, sender_index, amount, signed=True) + # recipient_index = transfer.recipient + # pre_transfer_recipient_balance = get_balance(state, recipient_index) # un-activate so validator can transfer - state.validator_registry[sender_index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH + # state.validator_registry[sender_index].activation_eligibility_epoch = spec.FAR_FUTURE_EPOCH - yield 'pre', state + # yield 'pre', state # Add to state via block transition - block = build_empty_block_for_next_slot(spec, state) - block.body.transfers.append(transfer) - sign_block(spec, state, block) + # block = build_empty_block_for_next_slot(spec, state) + # block.body.transfers.append(transfer) + # sign_block(spec, state, block) - yield 'blocks', [block], List[spec.BeaconBlock] + # yield 'blocks', [block], List[spec.BeaconBlock] - spec.state_transition(state, block) - yield 'post', state + # spec.state_transition(state, block) + # yield 'post', state - sender_balance = get_balance(state, sender_index) - recipient_balance = get_balance(state, recipient_index) - assert sender_balance == 0 - assert recipient_balance == pre_transfer_recipient_balance + amount + # sender_balance = get_balance(state, sender_index) + # recipient_balance = get_balance(state, recipient_index) + # assert sender_balance == 0 + # assert recipient_balance == pre_transfer_recipient_balance + amount @with_all_phases From cc625c7d6d0c25664b46b119e5389dc7aee3bda9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 11 Jun 2019 09:26:23 -0600 Subject: [PATCH 300/308] lint --- test_libs/pyspec/eth2spec/test/sanity/test_blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py index 593202dab2..2275161c8e 100644 --- a/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py +++ b/test_libs/pyspec/eth2spec/test/sanity/test_blocks.py @@ -5,7 +5,7 @@ from eth2spec.utils.bls import bls_sign from eth2spec.test.helpers.state import get_balance -from eth2spec.test.helpers.transfers import get_valid_transfer +# from eth2spec.test.helpers.transfers import get_valid_transfer from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block from eth2spec.test.helpers.keys import privkeys, pubkeys from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing From 2d636ddf5acff71808a044148095b28e8bf52522 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 17:27:34 +0200 Subject: [PATCH 301/308] BLS withdrawal byte is formatted as int now, but still one byte. Justin changed spec, now fix tests + configs --- configs/constant_presets/mainnet.yaml | 2 +- configs/constant_presets/minimal.yaml | 2 +- test_libs/pyspec/eth2spec/test/helpers/deposits.py | 2 +- test_libs/pyspec/eth2spec/test/helpers/transfers.py | 2 +- .../test/phase_0/block_processing/test_process_deposit.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/configs/constant_presets/mainnet.yaml b/configs/constant_presets/mainnet.yaml index bca7febc72..dca2a69ba8 100644 --- a/configs/constant_presets/mainnet.yaml +++ b/configs/constant_presets/mainnet.yaml @@ -48,7 +48,7 @@ GENESIS_FORK_VERSION: 0x00000000 GENESIS_SLOT: 0 # 2**64 - 1 FAR_FUTURE_EPOCH: 18446744073709551615 -BLS_WITHDRAWAL_PREFIX: 0x00 +BLS_WITHDRAWAL_PREFIX: 0 # Time parameters diff --git a/configs/constant_presets/minimal.yaml b/configs/constant_presets/minimal.yaml index 23bc6e8193..fa337cfbf8 100644 --- a/configs/constant_presets/minimal.yaml +++ b/configs/constant_presets/minimal.yaml @@ -47,7 +47,7 @@ GENESIS_FORK_VERSION: 0x00000000 GENESIS_SLOT: 0 # 2**64 - 1 FAR_FUTURE_EPOCH: 18446744073709551615 -BLS_WITHDRAWAL_PREFIX: 0x00 +BLS_WITHDRAWAL_PREFIX: 0 # Time parameters diff --git a/test_libs/pyspec/eth2spec/test/helpers/deposits.py b/test_libs/pyspec/eth2spec/test/helpers/deposits.py index b84970044a..1d4761753f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/deposits.py +++ b/test_libs/pyspec/eth2spec/test/helpers/deposits.py @@ -67,7 +67,7 @@ def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_c # insecurely use pubkey as withdrawal key if no credentials provided if withdrawal_credentials is None: - withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(pubkey)[1:] + withdrawal_credentials = spec.int_to_bytes(spec.BLS_WITHDRAWAL_PREFIX, length=1) + spec.hash(pubkey)[1:] deposit, root, deposit_data_leaves = build_deposit( spec, diff --git a/test_libs/pyspec/eth2spec/test/helpers/transfers.py b/test_libs/pyspec/eth2spec/test/helpers/transfers.py index eea12a36d4..4d40deee70 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/transfers.py +++ b/test_libs/pyspec/eth2spec/test/helpers/transfers.py @@ -32,7 +32,7 @@ def get_valid_transfer(spec, state, slot=None, sender_index=None, amount=None, f # ensure withdrawal_credentials reproducible state.validator_registry[transfer.sender].withdrawal_credentials = ( - spec.BLS_WITHDRAWAL_PREFIX + spec.hash(transfer.pubkey)[1:] + spec.int_to_bytes(spec.BLS_WITHDRAWAL_PREFIX, length=1) + spec.hash(transfer.pubkey)[1:] ) return transfer diff --git a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py index 42de447ece..603a07c3db 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_deposit.py @@ -101,7 +101,7 @@ def test_invalid_sig_top_up(spec, state): def test_invalid_withdrawal_credentials_top_up(spec, state): validator_index = 0 amount = spec.MAX_EFFECTIVE_BALANCE // 4 - withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(b"junk")[1:] + withdrawal_credentials = spec.int_to_bytes(spec.BLS_WITHDRAWAL_PREFIX, length=1) + spec.hash(b"junk")[1:] deposit = prepare_state_and_deposit( spec, state, From 7fafebd6cf7f0747e162e23ca327d5e3354053e6 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 17:32:42 +0200 Subject: [PATCH 302/308] missed one case, fix withdrawal byte <> int problem --- test_libs/pyspec/eth2spec/test/helpers/genesis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/genesis.py b/test_libs/pyspec/eth2spec/test/helpers/genesis.py index 23451b06b5..a3ca915ef5 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/genesis.py +++ b/test_libs/pyspec/eth2spec/test/helpers/genesis.py @@ -5,7 +5,7 @@ def build_mock_validator(spec, i: int, balance: int): pubkey = pubkeys[i] # insecurely use pubkey as withdrawal key as well - withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(pubkey)[1:] + withdrawal_credentials = spec.int_to_bytes(spec.BLS_WITHDRAWAL_PREFIX, length=1) + spec.hash(pubkey)[1:] return spec.Validator( pubkey=pubkeys[i], withdrawal_credentials=withdrawal_credentials, From 85b0c237ca2f58e210d0aa410e4892e4c9be6671 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 17:58:13 +0200 Subject: [PATCH 303/308] fix formatting sanity generator file names --- test_generators/sanity/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_generators/sanity/main.py b/test_generators/sanity/main.py index 805ee61394..fbef4da96d 100644 --- a/test_generators/sanity/main.py +++ b/test_generators/sanity/main.py @@ -16,7 +16,7 @@ def suite_definition(configs_path: str) -> gen_typing.TestSuiteOutput: spec_phase0.apply_constants_preset(presets) spec_phase1.apply_constants_preset(presets) - return ("%sanity_s_%s" % (handler_name, config_name), handler_name, gen_suite.render_suite( + return ("sanity_%s_%s" % (handler_name, config_name), handler_name, gen_suite.render_suite( title="sanity testing", summary="Sanity test suite, %s type, generated from pytests" % handler_name, forks_timeline="testing", From 22f624e12e5b7765c052b8a82b34b050e65fdada Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 18:08:27 +0200 Subject: [PATCH 304/308] re-order type check for more speed --- test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 368041f90a..832fd5ae23 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -513,13 +513,11 @@ def read_vector_elem_type(vector_typ: Type[Vector[T, L]]) -> T: def read_elem_type(typ): - if typ == bytes: + if issubclass(typ, bytes): # bytes or bytesN return byte elif is_list_type(typ): return read_list_elem_type(typ) elif is_vector_type(typ): return read_vector_elem_type(typ) - elif issubclass(typ, bytes): # bytes or bytesN - return byte else: raise TypeError("Unexpected type: {}".format(typ)) From d6fd19ce44f96dc7976ca9f48c43f93048a4478a Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 18:30:50 +0200 Subject: [PATCH 305/308] fix ssz-static generator --- test_generators/ssz_static/main.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/test_generators/ssz_static/main.py b/test_generators/ssz_static/main.py index 7de5237d1f..9d9af8c5e5 100644 --- a/test_generators/ssz_static/main.py +++ b/test_generators/ssz_static/main.py @@ -1,7 +1,10 @@ from random import Random +from inspect import getmembers, isclass + from eth2spec.debug import random_value, encode from eth2spec.phase0 import spec +from eth2spec.utils.ssz.ssz_typing import Container from eth2spec.utils.ssz.ssz_impl import ( hash_tree_root, signing_root, @@ -27,17 +30,23 @@ def create_test_case_contents(value, typ): @to_dict -def create_test_case(rng: Random, name: str, mode: random_value.RandomizationMode, chaos: bool): - typ = spec.get_ssz_type_by_name(name) +def create_test_case(rng: Random, name: str, typ, mode: random_value.RandomizationMode, chaos: bool): value = random_value.get_random_ssz_object(rng, typ, MAX_BYTES_LENGTH, MAX_LIST_LENGTH, mode, chaos) yield name, create_test_case_contents(value, typ) +def get_spec_ssz_types(): + return [ + (name, value) for (name, value) in getmembers(spec, isclass) + if issubclass(value, Container) and value != Container # only the subclasses, not the imported base class + ] + + @to_tuple def ssz_static_cases(rng: Random, mode: random_value.RandomizationMode, chaos: bool, count: int): - for type_name in spec.ssz_types: + for (name, ssz_type) in get_spec_ssz_types(): for i in range(count): - yield create_test_case(rng, type_name, mode, chaos) + yield create_test_case(rng, name, ssz_type, mode, chaos) def get_ssz_suite(seed: int, config_name: str, mode: random_value.RandomizationMode, chaos: bool, cases_if_random: int): @@ -81,8 +90,6 @@ def ssz_suite(configs_path: str) -> gen_typing.TestSuiteOutput: settings.append((seed, "mainnet", random_value.RandomizationMode.mode_random, False, 5)) seed += 1 - print("Settings: %d, SSZ-types: %d" % (len(settings), len(spec.ssz_types))) - gen_runner.run_generator("ssz_static", [ get_ssz_suite(seed, config_name, mode, chaos, cases_if_random) for (seed, config_name, mode, chaos, cases_if_random) in settings From 1157d9b8d8f17c475b4bdb34d60c598075e478be Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 18:34:49 +0200 Subject: [PATCH 306/308] fix shuffling generator --- test_generators/shuffling/main.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/test_generators/shuffling/main.py b/test_generators/shuffling/main.py index 862c4d9107..291aa2c47d 100644 --- a/test_generators/shuffling/main.py +++ b/test_generators/shuffling/main.py @@ -1,5 +1,4 @@ -from eth2spec.phase0 import spec as spec_phase0 -from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.phase0 import spec as spec from eth_utils import ( to_dict, to_tuple ) @@ -8,7 +7,7 @@ @to_dict -def shuffling_case(seed: spec.Bytes32, count: int): +def shuffling_case(seed, count): yield 'seed', '0x' + seed.hex() yield 'count', count yield 'shuffled', [spec.get_shuffled_index(i, count, seed) for i in range(count)] @@ -23,8 +22,7 @@ def shuffling_test_cases(): def mini_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput: presets = loader.load_presets(configs_path, 'minimal') - spec_phase0.apply_constants_preset(presets) - spec_phase1.apply_constants_preset(presets) + spec.apply_constants_preset(presets) return ("shuffling_minimal", "core", gen_suite.render_suite( title="Swap-or-Not Shuffling tests with minimal config", @@ -39,8 +37,7 @@ def mini_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput: def full_shuffling_suite(configs_path: str) -> gen_typing.TestSuiteOutput: presets = loader.load_presets(configs_path, 'mainnet') - spec_phase0.apply_constants_preset(presets) - spec_phase1.apply_constants_preset(presets) + spec.apply_constants_preset(presets) return ("shuffling_full", "core", gen_suite.render_suite( title="Swap-or-Not Shuffling tests with mainnet config", From f4814862fe3245f43bbdb3570fe6eda88207e217 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 19:23:45 +0200 Subject: [PATCH 307/308] fix typing check, add zero-hash cache to hash function --- test_libs/pyspec/eth2spec/utils/hash_function.py | 16 +++++++++++++++- .../pyspec/eth2spec/utils/ssz/ssz_typing.py | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/hash_function.py b/test_libs/pyspec/eth2spec/utils/hash_function.py index f965827d06..4aecc57f83 100644 --- a/test_libs/pyspec/eth2spec/utils/hash_function.py +++ b/test_libs/pyspec/eth2spec/utils/hash_function.py @@ -1,5 +1,19 @@ from hashlib import sha256 +ZERO_BYTES32 = b'\x00' * 32 -def hash(x): +def _hash(x): return sha256(x).digest() + +zerohashes = [(None, ZERO_BYTES32)] +for layer in range(1, 32): + k = zerohashes[layer - 1][1] + zerohashes[layer - 1][1] + zerohashes.append((k, _hash(k))) +zerohashes = zerohashes[1:] + + +def hash(x): + for (k, h) in zerohashes: + if x == k: + return h + return _hash(x) diff --git a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py index 832fd5ae23..55ced4ee2a 100644 --- a/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py +++ b/test_libs/pyspec/eth2spec/utils/ssz/ssz_typing.py @@ -513,7 +513,7 @@ def read_vector_elem_type(vector_typ: Type[Vector[T, L]]) -> T: def read_elem_type(typ): - if issubclass(typ, bytes): # bytes or bytesN + if typ == bytes or (isinstance(typ, type) and issubclass(typ, bytes)): # bytes or bytesN return byte elif is_list_type(typ): return read_list_elem_type(typ) From 9ec395c04fedf9572b1bfd815c6da1f9bd4f4b1c Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 11 Jun 2019 21:53:38 +0200 Subject: [PATCH 308/308] fig linting + improve docs + structure of hash optimization --- .../pyspec/eth2spec/utils/hash_function.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/utils/hash_function.py b/test_libs/pyspec/eth2spec/utils/hash_function.py index 4aecc57f83..2c9b5a579a 100644 --- a/test_libs/pyspec/eth2spec/utils/hash_function.py +++ b/test_libs/pyspec/eth2spec/utils/hash_function.py @@ -2,18 +2,27 @@ ZERO_BYTES32 = b'\x00' * 32 + def _hash(x): return sha256(x).digest() -zerohashes = [(None, ZERO_BYTES32)] -for layer in range(1, 32): - k = zerohashes[layer - 1][1] + zerohashes[layer - 1][1] - zerohashes.append((k, _hash(k))) -zerohashes = zerohashes[1:] + +# Minimal collection of (key, value) pairs, for fast hash-retrieval, to save on repetitive computation cost. +# Key = the hash input +# Value = the hash output +hash_cache = [] + + +def add_zero_hashes_to_cache(): + zerohashes = [(None, ZERO_BYTES32)] + for layer in range(1, 32): + k = zerohashes[layer - 1][1] + zerohashes[layer - 1][1] + zerohashes.append((k, _hash(k))) + hash_cache.extend(zerohashes[1:]) def hash(x): - for (k, h) in zerohashes: + for (k, h) in hash_cache: if x == k: return h return _hash(x)