Merge pull request #1444 from ethereum/v08x

Release `v0.8.4` to `master`

test_generators/bls/main.py

@@ -11,6 +11,12 @@
 from gen_base import gen_runner, gen_typing
 
 from py_ecc import bls
+from hashlib import sha256
+
+
+def hash(x):
+    return sha256(x).digest()
+
 
 F2Q_COEFF_LEN = 48
 G2_COMPRESSED_Z_LEN = 48
@@ -122,7 +128,8 @@ def case04_sign_messages():
         for message in MESSAGES:
             for domain in DOMAINS:
                 sig = bls.sign(message, privkey, domain)
-                yield f'sign_msg_{int_to_hex(privkey)}_{encode_hex(message)}_{encode_hex(domain)}', {
+                full_name = f'{int_to_hex(privkey)}_{encode_hex(message)}_{encode_hex(domain)}'
+                yield f'sign_msg_case_{(hash(bytes(full_name, "utf-8"))[:8]).hex()}', {
                     'input': {
                         'privkey': int_to_hex(privkey),
                         'message': encode_hex(message),

test_generators/epoch_processing/main.py

@@ -7,6 +7,7 @@
     test_process_final_updates,
     test_process_justification_and_finalization,
     test_process_registry_updates,
+    test_process_rewards_and_penalties,
     test_process_slashings
 )
 from gen_base import gen_runner, gen_typing
@@ -43,6 +44,9 @@ def cases_fn() -> Iterable[gen_typing.TestCase]:
         create_provider('justification_and_finalization', test_process_justification_and_finalization, 'mainnet'),
         create_provider('registry_updates', test_process_registry_updates, 'minimal'),
         create_provider('registry_updates', test_process_registry_updates, 'mainnet'),
+        create_provider('rewards_and_penalties', test_process_rewards_and_penalties, 'minimal'),
+        # runs full epochs filled with data, with uncached ssz. Disabled for now.
+        # create_provider('rewards_and_penalties', test_process_rewards_and_penalties, 'mainnet'),
         create_provider('slashings', test_process_slashings, 'minimal'),
         create_provider('slashings', test_process_slashings, 'mainnet'),
     ])

test_libs/config_helpers/requirements.txt

@@ -1 +1 @@
-ruamel.yaml==0.15.96
+ruamel.yaml==0.16.5

test_libs/config_helpers/setup.py

@@ -4,6 +4,6 @@
     name='config_helpers',
     packages=['preset_loader'],
     install_requires=[
-        "ruamel.yaml==0.15.96"
+        "ruamel.yaml==0.16.5"
     ]
 )

test_libs/pyspec/eth2spec/test/context.py

@@ -6,15 +6,67 @@
 
 from .utils import vector_test, with_meta_tags
 
+from typing import Any, Callable, Sequence
 
-def with_state(fn):
-    def entry(*args, **kw):
-        try:
-            kw['state'] = create_genesis_state(spec=kw['spec'], num_validators=spec_phase0.SLOTS_PER_EPOCH * 8)
-        except KeyError:
-            raise TypeError('Spec decorator must come within state decorator to inject spec into state.')
-        return fn(*args, **kw)
-    return entry
+
+def with_custom_state(balances_fn: Callable[[Any], Sequence[int]],
+                      threshold_fn: Callable[[Any], int]):
+    def deco(fn):
+        def entry(*args, **kw):
+            try:
+                spec = kw['spec']
+
+                balances = balances_fn(spec)
+                activation_threshold = threshold_fn(spec)
+
+                kw['state'] = create_genesis_state(spec=spec, validator_balances=balances,
+                                                   activation_threshold=activation_threshold)
+            except KeyError:
+                raise TypeError('Spec decorator must come within state decorator to inject spec into state.')
+            return fn(*args, **kw)
+        return entry
+    return deco
+
+
+def default_activation_threshold(spec):
+    """
+    Helper method to use the default balance activation threshold for state creation for tests.
+    Usage: `@with_custom_state(threshold_fn=default_activation_threshold, ...)`
+    """
+    return spec.MAX_EFFECTIVE_BALANCE
+
+
+def default_balances(spec):
+    """
+    Helper method to create a series of default balances.
+    Usage: `@with_custom_state(balances_fn=default_balances, ...)`
+    """
+    num_validators = spec.SLOTS_PER_EPOCH * 8
+    return [spec.MAX_EFFECTIVE_BALANCE] * num_validators
+
+
+with_state = with_custom_state(default_balances, default_activation_threshold)
+
+
+def low_balances(spec):
+    """
+    Helper method to create a series of low balances.
+    Usage: `@with_custom_state(balances_fn=low_balances, ...)`
+    """
+    num_validators = spec.SLOTS_PER_EPOCH * 8
+    # Technically the balances cannot be this low starting from genesis, but it is useful for testing
+    low_balance = 18 * 10 ** 9
+    return [low_balance] * num_validators
+
+
+def misc_balances(spec):
+    """
+    Helper method to create a series of balances that includes some misc. balances.
+    Usage: `@with_custom_state(balances_fn=misc_balances, ...)`
+    """
+    num_validators = spec.SLOTS_PER_EPOCH * 8
+    num_misc_validators = spec.SLOTS_PER_EPOCH
+    return [spec.MAX_EFFECTIVE_BALANCE] * num_validators + [spec.MIN_DEPOSIT_AMOUNT] * num_misc_validators
 
 
 # BLS is turned off by default *for performance purposes during TESTING*.

test_libs/pyspec/eth2spec/test/genesis/test_initialization.py

@@ -8,7 +8,7 @@
 @spec_test
 def test_initialize_beacon_state_from_eth1(spec):
     deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT
-    deposits, deposit_root = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True)
+    deposits, deposit_root, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True)
 
     eth1_block_hash = b'\x12' * 32
     eth1_timestamp = spec.MIN_GENESIS_TIME
@@ -25,6 +25,43 @@ def test_initialize_beacon_state_from_eth1(spec):
     assert state.eth1_data.deposit_root == deposit_root
     assert state.eth1_data.deposit_count == deposit_count
     assert state.eth1_data.block_hash == eth1_block_hash
+    assert spec.get_total_active_balance(state) == deposit_count * spec.MAX_EFFECTIVE_BALANCE
+
+    # yield state
+    yield 'state', state
+
+
+@with_phases(['phase0'])
+@spec_test
+def test_initialize_beacon_state_some_small_balances(spec):
+    main_deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT
+    main_deposits, _, deposit_data_list = prepare_genesis_deposits(spec, main_deposit_count,
+                                                                   spec.MAX_EFFECTIVE_BALANCE, signed=True)
+    # For deposits above, and for another deposit_count, add a balance of EFFECTIVE_BALANCE_INCREMENT
+    small_deposit_count = main_deposit_count * 2
+    small_deposits, deposit_root, _ = prepare_genesis_deposits(spec, small_deposit_count,
+                                                               spec.MIN_DEPOSIT_AMOUNT,
+                                                               signed=True,
+                                                               deposit_data_list=deposit_data_list)
+    deposits = main_deposits + small_deposits
+
+    eth1_block_hash = b'\x12' * 32
+    eth1_timestamp = spec.MIN_GENESIS_TIME
+
+    yield 'eth1_block_hash', eth1_block_hash
+    yield 'eth1_timestamp', eth1_timestamp
+    yield 'deposits', deposits
+
+    # initialize beacon_state
+    state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits)
+
+    assert state.genesis_time == eth1_timestamp - eth1_timestamp % spec.SECONDS_PER_DAY + 2 * spec.SECONDS_PER_DAY
+    assert len(state.validators) == small_deposit_count
+    assert state.eth1_data.deposit_root == deposit_root
+    assert state.eth1_data.deposit_count == len(deposits)
+    assert state.eth1_data.block_hash == eth1_block_hash
+    # only main deposits participate to the active balance
+    assert spec.get_total_active_balance(state) == main_deposit_count * spec.MAX_EFFECTIVE_BALANCE
 
     # yield state
     yield 'state', state

test_libs/pyspec/eth2spec/test/genesis/test_validity.py

@@ -6,7 +6,7 @@
 
 def create_valid_beacon_state(spec):
     deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT
-    deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True)
+    deposits, _, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True)
 
     eth1_block_hash = b'\x12' * 32
     eth1_timestamp = spec.MIN_GENESIS_TIME
@@ -65,7 +65,7 @@ def test_is_valid_genesis_state_true_more_balance(spec):
 @spec_test
 def test_is_valid_genesis_state_true_one_more_validator(spec):
     deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT + 1
-    deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True)
+    deposits, _, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True)
 
     eth1_block_hash = b'\x12' * 32
     eth1_timestamp = spec.MIN_GENESIS_TIME
@@ -78,7 +78,7 @@ def test_is_valid_genesis_state_true_one_more_validator(spec):
 @spec_test
 def test_is_valid_genesis_state_false_not_enough_validator(spec):
     deposit_count = spec.MIN_GENESIS_ACTIVE_VALIDATOR_COUNT - 1
-    deposits, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True)
+    deposits, _, _ = prepare_genesis_deposits(spec, deposit_count, spec.MAX_EFFECTIVE_BALANCE, signed=True)
 
     eth1_block_hash = b'\x12' * 32
     eth1_timestamp = spec.MIN_GENESIS_TIME

test_libs/pyspec/eth2spec/test/helpers/attestations.py

@@ -128,7 +128,7 @@ def get_attestation_signature(spec, state, attestation_data, privkey, custody_bi
     )
 
 
-def fill_aggregate_attestation(spec, state, attestation):
+def fill_aggregate_attestation(spec, state, attestation, signed=False):
     crosslink_committee = spec.get_crosslink_committee(
         state,
         attestation.data.target.epoch,
@@ -137,11 +137,15 @@ def fill_aggregate_attestation(spec, state, attestation):
     for i in range(len(crosslink_committee)):
         attestation.aggregation_bits[i] = True
 
+    if signed:
+        sign_attestation(spec, state, attestation)
+
 
-def add_attestation_to_state(spec, state, attestation, slot):
+def add_attestations_to_state(spec, state, attestations, slot):
     block = build_empty_block_for_next_slot(spec, state)
     block.slot = slot
-    block.body.attestations.append(attestation)
+    for attestation in attestations:
+        block.body.attestations.append(attestation)
     spec.process_slots(state, block.slot)
     sign_block(spec, state, block)
     spec.state_transition(state, block)

test_libs/pyspec/eth2spec/test/helpers/deposits.py

@@ -55,8 +55,9 @@ def build_deposit(spec,
     return deposit, root, deposit_data_list
 
 
-def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False):
-    deposit_data_list = []
+def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False, deposit_data_list=None):
+    if deposit_data_list is None:
+        deposit_data_list = []
     genesis_deposits = []
     for validator_index in range(genesis_validator_count):
         pubkey = pubkeys[validator_index]
@@ -75,7 +76,7 @@ def prepare_genesis_deposits(spec, genesis_validator_count, amount, signed=False
         )
         genesis_deposits.append(deposit)
 
-    return genesis_deposits, root
+    return genesis_deposits, root, deposit_data_list
 
 
 def prepare_state_and_deposit(spec, state, validator_index, amount, withdrawal_credentials=None, signed=False):

test_libs/pyspec/eth2spec/test/helpers/genesis.py

@@ -1,6 +1,7 @@
 from eth2spec.test.helpers.keys import pubkeys
 from eth2spec.utils.ssz.ssz_impl import hash_tree_root
 from eth2spec.utils.ssz.ssz_typing import List
+import copy
 
 
 def build_mock_validator(spec, i: int, balance: int):
@@ -18,28 +19,28 @@ def build_mock_validator(spec, i: int, balance: int):
     )
 
 
-def create_genesis_state(spec, num_validators):
+def create_genesis_state(spec, validator_balances, activation_threshold):
     deposit_root = b'\x42' * 32
 
     state = spec.BeaconState(
         genesis_time=0,
-        eth1_deposit_index=num_validators,
+        eth1_deposit_index=len(validator_balances),
         eth1_data=spec.Eth1Data(
             deposit_root=deposit_root,
-            deposit_count=num_validators,
+            deposit_count=len(validator_balances),
             block_hash=spec.Hash(),
         ),
         latest_block_header=spec.BeaconBlockHeader(body_root=spec.hash_tree_root(spec.BeaconBlockBody())),
     )
 
     # We "hack" in the initial validators,
     #  as it is much faster than creating and processing genesis deposits for every single test case.
-    state.balances = [spec.MAX_EFFECTIVE_BALANCE] * num_validators
-    state.validators = [build_mock_validator(spec, i, state.balances[i]) for i in range(num_validators)]
+    state.balances = copy.deepcopy(validator_balances)
+    state.validators = [build_mock_validator(spec, i, state.balances[i]) for i in range(len(validator_balances))]
 
     # Process genesis activations
     for validator in state.validators:
-        if validator.effective_balance >= spec.MAX_EFFECTIVE_BALANCE:
+        if validator.effective_balance >= activation_threshold:
             validator.activation_eligibility_epoch = spec.GENESIS_EPOCH
             validator.activation_epoch = spec.GENESIS_EPOCH
 

test_libs/pyspec/eth2spec/test/phase_0/block_processing/test_process_attestation.py

@@ -1,4 +1,5 @@
-from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, with_all_phases, with_phases
+from eth2spec.test.context import spec_state_test, expect_assertion_error, always_bls, \
+    with_all_phases, with_phases, spec_test, low_balances, with_custom_state
 from eth2spec.test.helpers.attestations import (
     get_valid_attestation,
     sign_aggregate_attestation,
@@ -56,6 +57,17 @@ def test_success(spec, state):
     yield from run_attestation_processing(spec, state, attestation)
 
 
+@with_all_phases
+@spec_test
+@with_custom_state(balances_fn=low_balances, threshold_fn=lambda spec: spec.EJECTION_BALANCE)
+def test_success_multi_proposer_index_iterations(spec, state):
+    state.slot += spec.SLOTS_PER_EPOCH * 2
+    attestation = get_valid_attestation(spec, state, signed=True)
+    state.slot += spec.MIN_ATTESTATION_INCLUSION_DELAY
+
+    yield from run_attestation_processing(spec, state, attestation)
+
+
 @with_all_phases
 @spec_state_test
 def test_success_previous_epoch(spec, state):