Merge pull request #2202 from ethereum/fork-choice-test-vectors

fork-choice test vectors: starting with `get_head` tests

tests/core/pyspec/eth2spec/test/helpers/fork_choice.py

@@ -1,3 +1,5 @@
+from eth_utils import encode_hex
+
 from eth2spec.phase0 import spec as phase0_spec
 
 
@@ -18,7 +20,23 @@ def add_block_to_store(spec, store, signed_block):
     spec.on_block(store, signed_block)
 
 
-def add_attestation_to_store(spec, store, attestation):
+def tick_and_run_on_block(spec, store, signed_block, test_steps=None):
+    if test_steps is None:
+        test_steps = []
+
+    pre_state = store.block_states[signed_block.message.parent_root]
+    block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT
+
+    if store.time < block_time:
+        on_tick_and_append_step(spec, store, block_time, test_steps)
+
+    yield from run_on_block(spec, store, signed_block, test_steps)
+
+
+def tick_and_run_on_attestation(spec, store, attestation, test_steps=None):
+    if test_steps is None:
+        test_steps = []
+
     parent_block = store.blocks[attestation.data.beacon_block_root]
     pre_state = store.block_states[spec.hash_tree_root(parent_block)]
     block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT
@@ -26,12 +44,72 @@ def add_attestation_to_store(spec, store, attestation):
 
     if store.time < next_epoch_time:
         spec.on_tick(store, next_epoch_time)
+        test_steps.append({'tick': int(next_epoch_time)})
 
     spec.on_attestation(store, attestation)
+    yield get_attestation_file_name(attestation), attestation
+    test_steps.append({'attestation': get_attestation_file_name(attestation)})
 
 
 def get_genesis_forkchoice_store(spec, genesis_state):
+    store, _ = get_genesis_forkchoice_store_and_block(spec, genesis_state)
+    return store
+
+
+def get_genesis_forkchoice_store_and_block(spec, genesis_state):
     assert genesis_state.slot == spec.GENESIS_SLOT
     # The genesis block must be a Phase 0 `BeaconBlock`
     genesis_block = phase0_spec.BeaconBlock(state_root=genesis_state.hash_tree_root())
-    return spec.get_forkchoice_store(genesis_state, genesis_block)
+    return spec.get_forkchoice_store(genesis_state, genesis_block), genesis_block
+
+
+def get_block_file_name(block):
+    return f"block_{encode_hex(block.hash_tree_root())}"
+
+
+def get_attestation_file_name(attestation):
+    return f"attestation_{encode_hex(attestation.hash_tree_root())}"
+
+
+def on_tick_and_append_step(spec, store, time, test_steps):
+    spec.on_tick(store, time)
+    test_steps.append({'tick': int(time)})
+
+
+def run_on_block(spec, store, signed_block, test_steps, valid=True):
+    if not valid:
+        try:
+            spec.on_block(store, signed_block)
+
+        except AssertionError:
+            return
+        else:
+            assert False
+
+    spec.on_block(store, signed_block)
+    yield get_block_file_name(signed_block), signed_block
+    test_steps.append({'block': get_block_file_name(signed_block)})
+
+    # An on_block step implies receiving block's attestations
+    for attestation in signed_block.message.body.attestations:
+        spec.on_attestation(store, attestation)
+
+    assert store.blocks[signed_block.message.hash_tree_root()] == signed_block.message
+    test_steps.append({
+        'checks': {
+            'time': int(store.time),
+            'head': get_formatted_head_output(spec, store),
+            'justified_checkpoint_root': encode_hex(store.justified_checkpoint.root),
+            'finalized_checkpoint_root': encode_hex(store.finalized_checkpoint.root),
+            'best_justified_checkpoint': encode_hex(store.best_justified_checkpoint.root),
+        }
+    })
+
+
+def get_formatted_head_output(spec, store):
+    head = spec.get_head(store)
+    slot = store.blocks[head].slot
+    return {
+        'slot': int(slot),
+        'root': encode_hex(head),
+    }

tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py

@@ -1,10 +1,22 @@
-from eth2spec.test.context import with_all_phases, spec_state_test
+from eth_utils import encode_hex
+
+from eth2spec.test.context import (
+    MINIMAL,
+    is_post_altair,
+    spec_state_test,
+    with_all_phases,
+    with_configs,
+)
 from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations
 from eth2spec.test.helpers.block import build_empty_block_for_next_slot
 from eth2spec.test.helpers.fork_choice import (
-    add_attestation_to_store,
-    add_block_to_store, get_anchor_root,
-    get_genesis_forkchoice_store,
+    tick_and_run_on_attestation,
+    tick_and_run_on_block,
+    get_anchor_root,
+    get_genesis_forkchoice_store_and_block,
+    get_formatted_head_output,
+    on_tick_and_append_step,
+    run_on_block,
 )
 from eth2spec.test.helpers.state import (
     next_epoch,
@@ -15,119 +27,180 @@ from eth2spec.test.helpers.state import (
 @with_all_phases
 @spec_state_test
 def test_genesis(spec, state):
+    test_steps = []
     # Initialization
-    store = get_genesis_forkchoice_store(spec, state)
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
+
     anchor_root = get_anchor_root(spec, state)
     assert spec.get_head(store) == anchor_root
+    test_steps.append({
+        'checks': {
+            'genesis_time': int(store.genesis_time),
+            'head': get_formatted_head_output(spec, store),
+        }
+    })
+
+    yield 'steps', test_steps
+
+    if is_post_altair(spec):
+        yield 'description', 'meta', f"Although it's not phase 0, we may use {spec.fork} spec to start testnets."
 
 
 @with_all_phases
 @spec_state_test
 def test_chain_no_attestations(spec, state):
+    test_steps = []
     # Initialization
-    store = get_genesis_forkchoice_store(spec, state)
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
+
     anchor_root = get_anchor_root(spec, state)
     assert spec.get_head(store) == anchor_root
+    test_steps.append({
+        'checks': {
+            'head': get_formatted_head_output(spec, store),
+        }
+    })
 
     # On receiving a block of `GENESIS_SLOT + 1` slot
     block_1 = build_empty_block_for_next_slot(spec, state)
     signed_block_1 = state_transition_and_sign_block(spec, state, block_1)
-    add_block_to_store(spec, store, signed_block_1)
+    yield from tick_and_run_on_block(spec, store, signed_block_1, test_steps)
 
     # On receiving a block of next epoch
     block_2 = build_empty_block_for_next_slot(spec, state)
     signed_block_2 = state_transition_and_sign_block(spec, state, block_2)
-    add_block_to_store(spec, store, signed_block_2)
+    yield from tick_and_run_on_block(spec, store, signed_block_2, test_steps)
 
     assert spec.get_head(store) == spec.hash_tree_root(block_2)
+    test_steps.append({
+        'checks': {
+            'head': get_formatted_head_output(spec, store),
+        }
+    })
+
+    yield 'steps', test_steps
 
 
 @with_all_phases
 @spec_state_test
 def test_split_tie_breaker_no_attestations(spec, state):
+    test_steps = []
     genesis_state = state.copy()
 
     # Initialization
-    store = get_genesis_forkchoice_store(spec, state)
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
     anchor_root = get_anchor_root(spec, state)
     assert spec.get_head(store) == anchor_root
+    test_steps.append({
+        'checks': {
+            'head': get_formatted_head_output(spec, store),
+        }
+    })
 
     # block at slot 1
     block_1_state = genesis_state.copy()
     block_1 = build_empty_block_for_next_slot(spec, block_1_state)
     signed_block_1 = state_transition_and_sign_block(spec, block_1_state, block_1)
-    add_block_to_store(spec, store, signed_block_1)
+    yield from tick_and_run_on_block(spec, store, signed_block_1, test_steps)
 
     # additional block at slot 1
     block_2_state = genesis_state.copy()
     block_2 = build_empty_block_for_next_slot(spec, block_2_state)
     block_2.body.graffiti = b'\x42' * 32
     signed_block_2 = state_transition_and_sign_block(spec, block_2_state, block_2)
-    add_block_to_store(spec, store, signed_block_2)
+    yield from tick_and_run_on_block(spec, store, signed_block_2, test_steps)
 
     highest_root = max(spec.hash_tree_root(block_1), spec.hash_tree_root(block_2))
-
     assert spec.get_head(store) == highest_root
+    test_steps.append({
+        'checks': {
+            'head': get_formatted_head_output(spec, store),
+        }
+    })
+
+    yield 'steps', test_steps
 
 
 @with_all_phases
 @spec_state_test
 def test_shorter_chain_but_heavier_weight(spec, state):
+    test_steps = []
     genesis_state = state.copy()
 
     # Initialization
-    store = get_genesis_forkchoice_store(spec, state)
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
     anchor_root = get_anchor_root(spec, state)
     assert spec.get_head(store) == anchor_root
+    test_steps.append({
+        'checks': {
+            'head': get_formatted_head_output(spec, store),
+        }
+    })
 
     # build longer tree
     long_state = genesis_state.copy()
     for _ in range(3):
         long_block = build_empty_block_for_next_slot(spec, long_state)
         signed_long_block = state_transition_and_sign_block(spec, long_state, long_block)
-        add_block_to_store(spec, store, signed_long_block)
+        yield from tick_and_run_on_block(spec, store, signed_long_block, test_steps)
 
     # build short tree
     short_state = genesis_state.copy()
     short_block = build_empty_block_for_next_slot(spec, short_state)
     short_block.body.graffiti = b'\x42' * 32
     signed_short_block = state_transition_and_sign_block(spec, short_state, short_block)
-    add_block_to_store(spec, store, signed_short_block)
+    yield from tick_and_run_on_block(spec, store, signed_short_block, test_steps)
 
     short_attestation = get_valid_attestation(spec, short_state, short_block.slot, signed=True)
-    add_attestation_to_store(spec, store, short_attestation)
+    yield from tick_and_run_on_attestation(spec, store, short_attestation, test_steps)
 
     assert spec.get_head(store) == spec.hash_tree_root(short_block)
+    test_steps.append({
+        'checks': {
+            'head': get_formatted_head_output(spec, store),
+        }
+    })
+
+    yield 'steps', test_steps
 
 
 @with_all_phases
 @spec_state_test
+@with_configs([MINIMAL], reason="too slow")
 def test_filtered_block_tree(spec, state):
+    test_steps = []
     # Initialization
-    store = get_genesis_forkchoice_store(spec, state)
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
     anchor_root = get_anchor_root(spec, state)
+    assert spec.get_head(store) == anchor_root
+    test_steps.append({
+        'checks': {
+            'head': get_formatted_head_output(spec, store),
+        }
+    })
 
     # transition state past initial couple of epochs
     next_epoch(spec, state)
     next_epoch(spec, state)
-
-    assert spec.get_head(store) == anchor_root
-
     # fill in attestations for entire epoch, justifying the recent epoch
     prev_state, signed_blocks, state = next_epoch_with_attestations(spec, state, True, False)
-    attestations = [
-        attestation for signed_block in signed_blocks
-        for attestation in signed_block.message.body.attestations
-    ]
     assert state.current_justified_checkpoint.epoch > prev_state.current_justified_checkpoint.epoch
 
     # tick time forward and add blocks and attestations to store
     current_time = state.slot * spec.SECONDS_PER_SLOT + store.genesis_time
-    spec.on_tick(store, current_time)
+    on_tick_and_append_step(spec, store, current_time, test_steps)
     for signed_block in signed_blocks:
-        spec.on_block(store, signed_block)
-    for attestation in attestations:
-        spec.on_attestation(store, attestation)
+        yield from run_on_block(spec, store, signed_block, test_steps)
 
     assert store.justified_checkpoint == state.current_justified_checkpoint
 
@@ -135,6 +208,13 @@ def test_filtered_block_tree(spec, state):
     expected_head_root = spec.hash_tree_root(signed_blocks[-1].message)
     assert spec.get_head(store) == expected_head_root
 
+    test_steps.append({
+        'checks': {
+            'head': get_formatted_head_output(spec, store),
+            'justified_checkpoint_root': encode_hex(store.justified_checkpoint.hash_tree_root()),
+        }
+    })
+
     #
     # create branch containing the justified block but not containing enough on
     # chain votes to justify that block
@@ -164,12 +244,20 @@ def test_filtered_block_tree(spec, state):
 
     # tick time forward to be able to include up to the latest attestation
     current_time = (attestations[-1].data.slot + 1) * spec.SECONDS_PER_SLOT + store.genesis_time
-    spec.on_tick(store, current_time)
+    on_tick_and_append_step(spec, store, current_time, test_steps)
 
     # include rogue block and associated attestations in the store
-    spec.on_block(store, signed_rogue_block)
+    yield from run_on_block(spec, store, signed_rogue_block, test_steps)
+
     for attestation in attestations:
-        spec.on_attestation(store, attestation)
+        yield from tick_and_run_on_attestation(spec, store, attestation, test_steps)
 
     # ensure that get_head still returns the head from the previous branch
     assert spec.get_head(store) == expected_head_root
+    test_steps.append({
+        'checks': {
+            'head': get_formatted_head_output(spec, store)
+        }
+    })
+
+    yield 'steps', test_steps

tests/formats/fork_choice/README.md

@@ -0,0 +1,111 @@
+# Fork choice tests
+
+The aim of the fork choice tests is to provide test coverage of the various components of the fork choice.
+
+## Test case format
+
+### `meta.yaml`
+
+```yaml
+description: string    -- Optional. Description of test case, purely for debugging purposes.
+bls_setting: int       -- see general test-format spec.
+```
+
+### `anchor_state.ssz_snappy`
+
+An SSZ-snappy encoded `BeaconState`, the state to initialize store with `get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock)` helper.
+
+### `anchor_block.ssz_snappy`
+
+An SSZ-snappy encoded `BeaconBlock`, the block to initialize store with `get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock)` helper.
+
+### `steps.yaml`
+
+The steps to execute in sequence. There may be multiple items of the following types:
+
+#### `on_tick` execution step
+
+The parameter that is required for executing `on_tick(store, time)`.
+
+```yaml
+{ tick: int }  --  to execute `on_tick(store, time)`
+```
+
+After this step, the `store` object may have been updated.
+
+#### `on_attestation` execution step
+
+The parameter that is required for executing `on_attestation(store, attestation)`.
+
+```yaml
+{ attestation: string }  -- the name of the `attestation_<32-byte-root>.ssz_snappy` file. To execute `on_attestation(store, attestation)` with the given attestation.
+```
+The file is located in the same folder (see below).
+
+After this step, the `store` object may have been updated.
+
+#### `on_block` execution step
+
+The parameter that is required for executing `on_block(store, block)`.
+
+```yaml
+{ block: string }  -- the name of the `block_<32-byte-root>.ssz_snappy` file. To execute `on_block(store, block)` with the given attestation.
+```
+The file is located in the same folder (see below).
+
+After this step, the `store` object may have been updated.
+
+#### Checks step
+
+The checks to verify the current status of `store`.
+
+```yaml
+checks: {<store_attibute>: value}  -- the assertions.
+```
+
+`<store_attibute>` is the field member or property of [`Store`](../../../specs/phase0/fork-choice.md#store) object that maintained by client implementation. Currently, the possible fields included:
+
+```yaml
+head: {                                     -- Encoded 32-byte value from get_head(store)
+    slot: slot,
+    root: string,
+}
+time: int                                   -- store.time
+genesis_time: int                           -- store.genesis_time
+justified_checkpoint_root: string           -- Encoded 32-byte value from store.justified_checkpoint.root
+finalized_checkpoint_root: string           -- Encoded 32-byte value from store.finalized_checkpoint.root
+best_justified_checkpoint_root: string      -- Encoded 32-byte value from store.best_justified_checkpoint.root
+```
+
+For example:
+```yaml
+- checks:
+    time: 144
+    genesis_time: 0
+    head: {slot: 17, root: '0xd2724c86002f7e1f8656ab44a341a409ad80e6e70a5225fd94835566deebb66f'}
+    justified_checkpoint_root: '0xcea6ecd3d3188e32ebf611f960eebd45b6c6f477a7cff242fa567a42653bfc7c'
+    finalized_checkpoint_root: '0xcea6ecd3d3188e32ebf611f960eebd45b6c6f477a7cff242fa567a42653bfc7c'
+    best_justified_checkpoint: '0xcea6ecd3d3188e32ebf611f960eebd45b6c6f477a7cff242fa567a42653bfc7c'
+```
+
+*Note*: Each `checks` step may include one or multiple items. Each item has to be checked against the current store.
+
+### `attestation_<32-byte-root>.ssz_snappy`
+
+`<32-byte-root>` is the hash tree root of the given attestation.
+
+Each file is an SSZ-snappy encoded `Attestation`.
+
+### `block_<32-byte-root>.ssz_snappy`
+
+`<32-byte-root>` is the hash tree root of the given block.
+
+Each file is an SSZ-snappy encoded `SignedBeaconBlock`.
+
+## Condition
+
+1. Deserialize `anchor_state.ssz_snappy` and `anchor_block.ssz_snappy` to initialize the local store object by with `get_forkchoice_store(anchor_state, anchor_block)` helper.
+2. Iterate sequentially through `steps.yaml`
+    - For each execution, look up the corresponding ssz_snappy file. Execute the corresponding helper function on the current store.
+        - For the `on_block` execution step: if `len(block.message.body.attestations) > 0`, execute each attestation with `on_attestation(store, attestation)` after executing `on_block(store, block)`.
+    - For each `checks` step, the assertions on the current store must be satisfied.

tests/generators/fork_choice/README.md

@@ -0,0 +1,5 @@
+# Fork choice tests
+
+Fork choice tests cover the different forking cases with fork choice helper functions.
+
+Information on the format of the tests can be found in the [fork choice test formats documentation](../../formats/fork_choice/README.md).

tests/generators/fork_choice/main.py

@@ -0,0 +1,25 @@
+from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators
+from eth2spec.phase0 import spec as spec_phase0
+from eth2spec.altair import spec as spec_altair
+from eth2spec.phase1 import spec as spec_phase1
+from eth2spec.test.context import PHASE0, PHASE1, ALTAIR
+
+
+specs = (spec_phase0, spec_altair, spec_phase1)
+
+
+if __name__ == "__main__":
+    phase_0_mods = {key: 'eth2spec.test.phase0.fork_choice.test_' + key for key in [
+        'get_head',
+    ]}
+    # No additional Altair or phase 1 specific finality tests, yet.
+    altair_mods = phase_0_mods
+    phase_1_mods = phase_0_mods
+
+    all_mods = {
+        PHASE0: phase_0_mods,
+        ALTAIR: altair_mods,
+        PHASE1: phase_1_mods,
+    }
+
+    run_state_test_generators(runner_name="fork_choice", specs=specs, all_mods=all_mods)

tests/generators/fork_choice/requirements.txt

@@ -0,0 +1,2 @@
+pytest>=4.4
+../../../[generator]