Merge pull request #1333 from ethereum/v08x

v0.8.2 release into master

.circleci/config.yml

@@ -35,13 +35,13 @@ commands:
     description: "Restore the cache with pyspec keys"
     steps:
       - restore_cached_venv:
-          venv_name: v3-pyspec-bump2
+          venv_name: v4-pyspec
           reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}
   save_pyspec_cached_venv:
     description: Save a venv into a cache with pyspec keys"
     steps:
       - save_cached_venv:
-          venv_name: v3-pyspec-bump2
+          venv_name: v4-pyspec
           reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}
           venv_path: ./test_libs/pyspec/venv
   restore_deposit_contract_cached_venv:

Makefile

@@ -2,17 +2,20 @@ SPEC_DIR = ./specs
 SCRIPT_DIR = ./scripts
 TEST_LIBS_DIR = ./test_libs
 PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec
-YAML_TEST_DIR = ./eth2.0-spec-tests/tests
+TEST_VECTOR_DIR = ./eth2.0-spec-tests/tests
 GENERATOR_DIR = ./test_generators
 DEPOSIT_CONTRACT_DIR = ./deposit_contract
 CONFIGS_DIR = ./configs
 
 # Collect a list of generator names
-GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/)))
-# Map this list of generator paths to a list of test output paths
-YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERATORS))
+GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.)))
+# Map this list of generator paths to "gen_{generator name}" entries
+GENERATOR_TARGETS = $(patsubst $(GENERATOR_DIR)/%/, gen_%, $(GENERATORS))
 GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS))
 
+# To check generator matching:
+#$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}])
+
 PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py
 PY_SPEC_PHASE_0_DEPS = $(SPEC_DIR)/core/0_*.md
 
@@ -24,14 +27,14 @@ PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) $(PY_SPEC_PHASE_1_TARGETS)
 COV_HTML_OUT=.htmlcov
 COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html
 
-.PHONY: clean all test citest lint gen_yaml_tests pyspec phase0 phase1 install_test open_cov \
+.PHONY: clean partial_clean all test citest lint generate_tests pyspec phase0 phase1 install_test open_cov \
         install_deposit_contract_test test_deposit_contract compile_deposit_contract
 
-all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS)
+all: $(PY_SPEC_ALL_TARGETS)
 
 # deletes everything except the venvs
 partial_clean:
-	rm -rf $(YAML_TEST_DIR)
+	rm -rf $(TEST_VECTOR_DIR)
 	rm -rf $(GENERATOR_VENVS)
 	rm -rf $(PY_SPEC_DIR)/.pytest_cache
 	rm -rf $(PY_SPEC_ALL_TARGETS)
@@ -44,8 +47,8 @@ clean: partial_clean
 	rm -rf $(PY_SPEC_DIR)/venv
 	rm -rf $(DEPOSIT_CONTRACT_DIR)/venv
 
-# "make gen_yaml_tests" to run generators
-gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS)
+# "make generate_tests" to run all generators
+generate_tests: $(PY_SPEC_ALL_TARGETS) $(GENERATOR_TARGETS)
 
 # installs the packages to run pyspec tests
 install_test:
@@ -90,8 +93,8 @@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS)
 
 CURRENT_DIR = ${CURDIR}
 
-# The function that builds a set of suite files, by calling a generator for the given type (param 1)
-define build_yaml_tests
+# Runs a generator, identified by param 1
+define run_generator
 	# Started!
 	# Create output directory
 	# Navigate to the generator
@@ -100,24 +103,25 @@ define build_yaml_tests
 	# Install all the necessary requirements
 	# Run the generator. The generator is assumed to have an "main.py" file.
 	# We output to the tests dir (generator program should accept a "-o <filepath>" argument.
+	# `-l minimal general` can be added to the generator call to filter to smaller configs, when testing.
 	echo "generator $(1) started"; \
-	mkdir -p $(YAML_TEST_DIR)$(1); \
-	cd $(GENERATOR_DIR)$(1); \
+	mkdir -p $(TEST_VECTOR_DIR); \
+	cd $(GENERATOR_DIR)/$(1); \
 	if ! test -d venv; then python3 -m venv venv; fi; \
 	. venv/bin/activate; \
 	pip3 install -r requirements.txt; \
-	python3 main.py -o $(CURRENT_DIR)/$(YAML_TEST_DIR)$(1) -c $(CURRENT_DIR)/$(CONFIGS_DIR); \
+	python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) -c $(CURRENT_DIR)/$(CONFIGS_DIR); \
 	echo "generator $(1) finished"
 endef
 
 # The tests dir itself is simply build by creating the directory (recursively creating deeper directories if necessary)
-$(YAML_TEST_DIR):
-	$(info creating directory, to output yaml targets to: ${YAML_TEST_TARGETS})
+$(TEST_VECTOR_DIR):
+	$(info creating test output directory, for generators: ${GENERATOR_TARGETS})
 	mkdir -p $@
-$(YAML_TEST_DIR)/:
-	$(info ignoring duplicate yaml tests dir)
+$(TEST_VECTOR_DIR)/:
+	$(info ignoring duplicate tests dir)
 
-# For any target within the tests dir, build it using the build_yaml_tests function.
+# For any generator, build it using the run_generator function.
 # (creation of output dir is a dependency)
-$(YAML_TEST_DIR)%: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR)
-	$(call build_yaml_tests,$*)
+gen_%: $(PY_SPEC_ALL_TARGETS) $(TEST_VECTOR_DIR)
+	$(call run_generator,$*)

README.md

@@ -21,6 +21,12 @@ Core specifications for Eth 2.0 client validation can be found in [specs/core](s
 * [Custody Game](specs/core/1_custody-game.md)
 * [Shard Data Chains](specs/core/1_shard-data-chains.md)
 
+### Phase 2
+
+Phase 2 is still actively in R&D and does not yet have any formal specifications.
+
+See the [Eth 2.0 Phase 2 Wiki](https://hackmd.io/UzysWse1Th240HELswKqVA?view) for current progress, discussions, and definitions regarding this work.
+
 ### Accompanying documents can be found in [specs](specs) and include:
 
 * [SimpleSerialize (SSZ) spec](specs/simple-serialize.md)

configs/README.md

@@ -0,0 +1,36 @@
+# Configs
+
+This directory contains a set of constants presets used for testing, testnets, and mainnet.
+
+A preset file contains all the constants known for its target.
+Later-fork constants can be ignored, e.g. ignore phase1 constants as a client that only supports phase 0 currently.
+
+
+## Forking
+
+Configs are not replaced, but extended with forks. This is to support syncing from one state to the other over a fork boundary, without hot-swapping a config.
+Instead, for forks that introduce changes in a constant, the constant name is prefixed with a short abbreviation of the fork.
+
+Over time, the need to sync an older state may be deprecated.
+In this case, the prefix on the new constant may be removed, and the old constant will keep a special name before completely being removed.
+
+A previous iteration of forking made use of "timelines", but this collides with the definitions used in the spec (constants for special forking slots etc.),
+ and was not integrated sufficiently in any of the spec tools or implementations.
+Instead, the config essentially doubles as fork definition now, changing the value for e.g. `PHASE_1_GENESIS_SLOT` changes the fork.
+
+Another reason to prefer forking through constants is the ability to program a forking moment based on context, instead of being limited to a static slot number.
+
+
+## Format
+
+Each preset is a key-value mapping.
+
+**Key**: an `UPPER_SNAKE_CASE` (a.k.a. "macro case") formatted string, name of the constant.
+
+**Value** can be either:
+ - an unsigned integer number, can be up to 64 bits (incl.)
+ - a hexadecimal string, prefixed with `0x`
+
+Presets may contain comments to describe the values.
+
+See [`mainnet.yaml`](./mainnet.yaml) for a complete example.

configs/constant_presets/mainnet.yaml → configs/mainnet.yaml

@@ -128,3 +128,6 @@ DOMAIN_ATTESTATION: 0x02000000
 DOMAIN_DEPOSIT: 0x03000000
 DOMAIN_VOLUNTARY_EXIT: 0x04000000
 DOMAIN_TRANSFER: 0x05000000
+DOMAIN_CUSTODY_BIT_CHALLENGE: 0x06000000
+DOMAIN_SHARD_PROPOSER: 0x80000000
+DOMAIN_SHARD_ATTESTER: 0x81000000

configs/constant_presets/minimal.yaml → configs/minimal.yaml

@@ -125,4 +125,7 @@ DOMAIN_RANDAO: 0x01000000
 DOMAIN_ATTESTATION: 0x02000000
 DOMAIN_DEPOSIT: 0x03000000
 DOMAIN_VOLUNTARY_EXIT: 0x04000000
-DOMAIN_TRANSFER: 0x05000000
+DOMAIN_TRANSFER: 0x05000000
+DOMAIN_CUSTODY_BIT_CHALLENGE: 0x06000000
+DOMAIN_SHARD_PROPOSER: 0x80000000
+DOMAIN_SHARD_ATTESTER: 0x81000000

scripts/build_spec.py

@@ -49,7 +49,7 @@
     hash_tree_root,
     signing_root,
     serialize,
-    is_empty,
+    is_zero,
 )
 from eth2spec.utils.ssz.ssz_typing import (
     bit, boolean, Container, List, Vector, Bytes, uint64,

specs/core/0_beacon-chain.md

@@ -207,6 +207,7 @@ The following values are (non-configurable) constants used throughout the specif
 
 | Name | Value | Unit | Duration |
 | - | - | :-: | :-: |
+| `SECONDS_PER_SLOT` | `6` | seconds | 6 seconds |
 | `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | slots | 6 seconds |
 | `SLOTS_PER_EPOCH` | `2**6` (= 64) | slots | 6.4 minutes |
 | `MIN_SEED_LOOKAHEAD` | `2**0` (= 1) | epochs | 6.4 minutes |
@@ -1270,7 +1271,7 @@ def get_unslashed_attesting_indices(state: BeaconState,
     output = set()  # type: Set[ValidatorIndex]
     for a in attestations:
         output = output.union(get_attesting_indices(state, a.data, a.aggregation_bits))
-    return set(filter(lambda index: not state.validators[index].slashed, list(output)))
+    return set(filter(lambda index: not state.validators[index].slashed, output))
 ```
 
 ```python
@@ -1283,10 +1284,10 @@ def get_winning_crosslink_and_attesting_indices(state: BeaconState,
                                                 epoch: Epoch,
                                                 shard: Shard) -> Tuple[Crosslink, Set[ValidatorIndex]]:
     attestations = [a for a in get_matching_source_attestations(state, epoch) if a.data.crosslink.shard == shard]
-    crosslinks = list(filter(
+    crosslinks = filter(
         lambda c: hash_tree_root(state.current_crosslinks[shard]) in (c.parent_root, hash_tree_root(c)),
         [a.data.crosslink for a in attestations]
-    ))
+    )
     # Winning crosslink has the crosslink data root with the most balance voting for it (ties broken lexicographically)
     winning_crosslink = max(crosslinks, key=lambda c: (
         get_attesting_balance(state, [a for a in attestations if a.data.crosslink == c]), c.data_root
@@ -1503,8 +1504,6 @@ def process_final_updates(state: BeaconState) -> None:
         HALF_INCREMENT = EFFECTIVE_BALANCE_INCREMENT // 2
         if balance < validator.effective_balance or validator.effective_balance + 3 * HALF_INCREMENT < balance:
             validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE)
-    # Update start shard
-    state.start_shard = Shard((state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT)
     # Set active index root
     index_epoch = Epoch(next_epoch + ACTIVATION_EXIT_DELAY)
     index_root_position = index_epoch % EPOCHS_PER_HISTORICAL_VECTOR
@@ -1521,6 +1520,8 @@ def process_final_updates(state: BeaconState) -> None:
     if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0:
         historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots)
         state.historical_roots.append(hash_tree_root(historical_batch))
+    # Update start shard
+    state.start_shard = Shard((state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT)
     # Rotate current/previous epoch attestations
     state.previous_epoch_attestations = state.current_epoch_attestations
     state.current_epoch_attestations = []
@@ -1653,6 +1654,9 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None:
     attestation_slot = get_attestation_data_slot(state, data)
     assert attestation_slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation_slot + SLOTS_PER_EPOCH
 
+    committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard)
+    assert len(attestation.aggregation_bits) == len(attestation.custody_bits) == len(committee)
+
     pending_attestation = PendingAttestation(
         data=data,
         aggregation_bits=attestation.aggregation_bits,

specs/core/0_fork-choice.md

@@ -8,8 +8,6 @@
 - [Ethereum 2.0 Phase 0 -- Beacon Chain Fork Choice](#ethereum-20-phase-0----beacon-chain-fork-choice)
     - [Table of contents](#table-of-contents)
     - [Introduction](#introduction)
-    - [Configuration](#configuration)
-        - [Time parameters](#time-parameters)
     - [Fork choice](#fork-choice)
         - [Helpers](#helpers)
             - [`LatestMessage`](#latestmessage)
@@ -29,14 +27,6 @@
 
 This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0. It assumes the [beacon chain state transition function spec](./0_beacon-chain.md).
 
-## Configuration
-
-### Time parameters
-
-| Name | Value | Unit | Duration |
-| - | - | :-: | :-: |
-| `SECONDS_PER_SLOT` | `6` | seconds | 6 seconds |
-
 ## Fork choice
 
 The head block root associated with a `store` is defined as `get_head(store)`. At genesis, let `store = get_genesis_store(genesis_state)` and update `store` by running:
@@ -101,8 +91,12 @@ def get_genesis_store(genesis_state: BeaconState) -> Store:
 ```python
 def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash:
     block = store.blocks[root]
-    assert block.slot >= slot
-    return root if block.slot == slot else get_ancestor(store, block.parent_root, slot)
+    if block.slot > slot:
+        return get_ancestor(store, block.parent_root, slot)
+    elif block.slot == slot:
+        return root
+    else:
+        return Bytes32()  # root is older than queried slot: no results. 
 ```
 
 #### `get_latest_attesting_balance`