diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py index 85437e98a..2ca37768b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -1,3 +1,5 @@ +from eth_utils import encode_hex + from eth2spec.phase0 import spec as phase0_spec @@ -18,7 +20,23 @@ def add_block_to_store(spec, store, signed_block): spec.on_block(store, signed_block) -def add_attestation_to_store(spec, store, attestation): +def tick_and_run_on_block(spec, store, signed_block, test_steps=None): + if test_steps is None: + test_steps = [] + + pre_state = store.block_states[signed_block.message.parent_root] + block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT + + if store.time < block_time: + on_tick_and_append_step(spec, store, block_time, test_steps) + + yield from run_on_block(spec, store, signed_block, test_steps) + + +def tick_and_run_on_attestation(spec, store, attestation, test_steps=None): + if test_steps is None: + test_steps = [] + parent_block = store.blocks[attestation.data.beacon_block_root] pre_state = store.block_states[spec.hash_tree_root(parent_block)] block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT @@ -26,12 +44,72 @@ def add_attestation_to_store(spec, store, attestation): if store.time < next_epoch_time: spec.on_tick(store, next_epoch_time) + test_steps.append({'tick': int(next_epoch_time)}) spec.on_attestation(store, attestation) + yield get_attestation_file_name(attestation), attestation + test_steps.append({'attestation': get_attestation_file_name(attestation)}) def get_genesis_forkchoice_store(spec, genesis_state): + store, _ = get_genesis_forkchoice_store_and_block(spec, genesis_state) + return store + + +def get_genesis_forkchoice_store_and_block(spec, genesis_state): assert genesis_state.slot == spec.GENESIS_SLOT # The genesis block must be a Phase 0 `BeaconBlock` genesis_block = phase0_spec.BeaconBlock(state_root=genesis_state.hash_tree_root()) - return spec.get_forkchoice_store(genesis_state, genesis_block) + return spec.get_forkchoice_store(genesis_state, genesis_block), genesis_block + + +def get_block_file_name(block): + return f"block_{encode_hex(block.hash_tree_root())}" + + +def get_attestation_file_name(attestation): + return f"attestation_{encode_hex(attestation.hash_tree_root())}" + + +def on_tick_and_append_step(spec, store, time, test_steps): + spec.on_tick(store, time) + test_steps.append({'tick': int(time)}) + + +def run_on_block(spec, store, signed_block, test_steps, valid=True): + if not valid: + try: + spec.on_block(store, signed_block) + + except AssertionError: + return + else: + assert False + + spec.on_block(store, signed_block) + yield get_block_file_name(signed_block), signed_block + test_steps.append({'block': get_block_file_name(signed_block)}) + + # An on_block step implies receiving block's attestations + for attestation in signed_block.message.body.attestations: + spec.on_attestation(store, attestation) + + assert store.blocks[signed_block.message.hash_tree_root()] == signed_block.message + test_steps.append({ + 'checks': { + 'time': int(store.time), + 'head': get_formatted_head_output(spec, store), + 'justified_checkpoint_root': encode_hex(store.justified_checkpoint.root), + 'finalized_checkpoint_root': encode_hex(store.finalized_checkpoint.root), + 'best_justified_checkpoint': encode_hex(store.best_justified_checkpoint.root), + } + }) + + +def get_formatted_head_output(spec, store): + head = spec.get_head(store) + slot = store.blocks[head].slot + return { + 'slot': int(slot), + 'root': encode_hex(head), + } diff --git a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/__init__.py b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py similarity index 59% rename from tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_get_head.py rename to tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py index b470ab079..318db496a 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py @@ -1,10 +1,22 @@ -from eth2spec.test.context import with_all_phases, spec_state_test +from eth_utils import encode_hex + +from eth2spec.test.context import ( + MINIMAL, + is_post_altair, + spec_state_test, + with_all_phases, + with_configs, +) from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations from eth2spec.test.helpers.block import build_empty_block_for_next_slot from eth2spec.test.helpers.fork_choice import ( - add_attestation_to_store, - add_block_to_store, get_anchor_root, - get_genesis_forkchoice_store, + tick_and_run_on_attestation, + tick_and_run_on_block, + get_anchor_root, + get_genesis_forkchoice_store_and_block, + get_formatted_head_output, + on_tick_and_append_step, + run_on_block, ) from eth2spec.test.helpers.state import ( next_epoch, @@ -15,119 +27,180 @@ from eth2spec.test.helpers.state import ( @with_all_phases @spec_state_test def test_genesis(spec, state): + test_steps = [] # Initialization - store = get_genesis_forkchoice_store(spec, state) + store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state) + yield 'anchor_state', state + yield 'anchor_block', anchor_block + anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root + test_steps.append({ + 'checks': { + 'genesis_time': int(store.genesis_time), + 'head': get_formatted_head_output(spec, store), + } + }) + + yield 'steps', test_steps + + if is_post_altair(spec): + yield 'description', 'meta', f"Although it's not phase 0, we may use {spec.fork} spec to start testnets." @with_all_phases @spec_state_test def test_chain_no_attestations(spec, state): + test_steps = [] # Initialization - store = get_genesis_forkchoice_store(spec, state) + store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state) + yield 'anchor_state', state + yield 'anchor_block', anchor_block + anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root + test_steps.append({ + 'checks': { + 'head': get_formatted_head_output(spec, store), + } + }) # On receiving a block of `GENESIS_SLOT + 1` slot block_1 = build_empty_block_for_next_slot(spec, state) signed_block_1 = state_transition_and_sign_block(spec, state, block_1) - add_block_to_store(spec, store, signed_block_1) + yield from tick_and_run_on_block(spec, store, signed_block_1, test_steps) # On receiving a block of next epoch block_2 = build_empty_block_for_next_slot(spec, state) signed_block_2 = state_transition_and_sign_block(spec, state, block_2) - add_block_to_store(spec, store, signed_block_2) + yield from tick_and_run_on_block(spec, store, signed_block_2, test_steps) assert spec.get_head(store) == spec.hash_tree_root(block_2) + test_steps.append({ + 'checks': { + 'head': get_formatted_head_output(spec, store), + } + }) + + yield 'steps', test_steps @with_all_phases @spec_state_test def test_split_tie_breaker_no_attestations(spec, state): + test_steps = [] genesis_state = state.copy() # Initialization - store = get_genesis_forkchoice_store(spec, state) + store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state) + yield 'anchor_state', state + yield 'anchor_block', anchor_block anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root + test_steps.append({ + 'checks': { + 'head': get_formatted_head_output(spec, store), + } + }) # block at slot 1 block_1_state = genesis_state.copy() block_1 = build_empty_block_for_next_slot(spec, block_1_state) signed_block_1 = state_transition_and_sign_block(spec, block_1_state, block_1) - add_block_to_store(spec, store, signed_block_1) + yield from tick_and_run_on_block(spec, store, signed_block_1, test_steps) # additional block at slot 1 block_2_state = genesis_state.copy() block_2 = build_empty_block_for_next_slot(spec, block_2_state) block_2.body.graffiti = b'\x42' * 32 signed_block_2 = state_transition_and_sign_block(spec, block_2_state, block_2) - add_block_to_store(spec, store, signed_block_2) + yield from tick_and_run_on_block(spec, store, signed_block_2, test_steps) highest_root = max(spec.hash_tree_root(block_1), spec.hash_tree_root(block_2)) - assert spec.get_head(store) == highest_root + test_steps.append({ + 'checks': { + 'head': get_formatted_head_output(spec, store), + } + }) + + yield 'steps', test_steps @with_all_phases @spec_state_test def test_shorter_chain_but_heavier_weight(spec, state): + test_steps = [] genesis_state = state.copy() # Initialization - store = get_genesis_forkchoice_store(spec, state) + store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state) + yield 'anchor_state', state + yield 'anchor_block', anchor_block anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root + test_steps.append({ + 'checks': { + 'head': get_formatted_head_output(spec, store), + } + }) # build longer tree long_state = genesis_state.copy() for _ in range(3): long_block = build_empty_block_for_next_slot(spec, long_state) signed_long_block = state_transition_and_sign_block(spec, long_state, long_block) - add_block_to_store(spec, store, signed_long_block) + yield from tick_and_run_on_block(spec, store, signed_long_block, test_steps) # build short tree short_state = genesis_state.copy() short_block = build_empty_block_for_next_slot(spec, short_state) short_block.body.graffiti = b'\x42' * 32 signed_short_block = state_transition_and_sign_block(spec, short_state, short_block) - add_block_to_store(spec, store, signed_short_block) + yield from tick_and_run_on_block(spec, store, signed_short_block, test_steps) short_attestation = get_valid_attestation(spec, short_state, short_block.slot, signed=True) - add_attestation_to_store(spec, store, short_attestation) + yield from tick_and_run_on_attestation(spec, store, short_attestation, test_steps) assert spec.get_head(store) == spec.hash_tree_root(short_block) + test_steps.append({ + 'checks': { + 'head': get_formatted_head_output(spec, store), + } + }) + + yield 'steps', test_steps @with_all_phases @spec_state_test +@with_configs([MINIMAL], reason="too slow") def test_filtered_block_tree(spec, state): + test_steps = [] # Initialization - store = get_genesis_forkchoice_store(spec, state) + store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state) + yield 'anchor_state', state + yield 'anchor_block', anchor_block anchor_root = get_anchor_root(spec, state) + assert spec.get_head(store) == anchor_root + test_steps.append({ + 'checks': { + 'head': get_formatted_head_output(spec, store), + } + }) # transition state past initial couple of epochs next_epoch(spec, state) next_epoch(spec, state) - - assert spec.get_head(store) == anchor_root - # fill in attestations for entire epoch, justifying the recent epoch prev_state, signed_blocks, state = next_epoch_with_attestations(spec, state, True, False) - attestations = [ - attestation for signed_block in signed_blocks - for attestation in signed_block.message.body.attestations - ] assert state.current_justified_checkpoint.epoch > prev_state.current_justified_checkpoint.epoch # tick time forward and add blocks and attestations to store current_time = state.slot * spec.SECONDS_PER_SLOT + store.genesis_time - spec.on_tick(store, current_time) + on_tick_and_append_step(spec, store, current_time, test_steps) for signed_block in signed_blocks: - spec.on_block(store, signed_block) - for attestation in attestations: - spec.on_attestation(store, attestation) + yield from run_on_block(spec, store, signed_block, test_steps) assert store.justified_checkpoint == state.current_justified_checkpoint @@ -135,6 +208,13 @@ def test_filtered_block_tree(spec, state): expected_head_root = spec.hash_tree_root(signed_blocks[-1].message) assert spec.get_head(store) == expected_head_root + test_steps.append({ + 'checks': { + 'head': get_formatted_head_output(spec, store), + 'justified_checkpoint_root': encode_hex(store.justified_checkpoint.hash_tree_root()), + } + }) + # # create branch containing the justified block but not containing enough on # chain votes to justify that block @@ -164,12 +244,20 @@ def test_filtered_block_tree(spec, state): # tick time forward to be able to include up to the latest attestation current_time = (attestations[-1].data.slot + 1) * spec.SECONDS_PER_SLOT + store.genesis_time - spec.on_tick(store, current_time) + on_tick_and_append_step(spec, store, current_time, test_steps) # include rogue block and associated attestations in the store - spec.on_block(store, signed_rogue_block) + yield from run_on_block(spec, store, signed_rogue_block, test_steps) + for attestation in attestations: - spec.on_attestation(store, attestation) + yield from tick_and_run_on_attestation(spec, store, attestation, test_steps) # ensure that get_head still returns the head from the previous branch assert spec.get_head(store) == expected_head_root + test_steps.append({ + 'checks': { + 'head': get_formatted_head_output(spec, store) + } + }) + + yield 'steps', test_steps diff --git a/tests/formats/fork_choice/README.md b/tests/formats/fork_choice/README.md new file mode 100644 index 000000000..832ce9dd1 --- /dev/null +++ b/tests/formats/fork_choice/README.md @@ -0,0 +1,111 @@ +# Fork choice tests + +The aim of the fork choice tests is to provide test coverage of the various components of the fork choice. + +## Test case format + +### `meta.yaml` + +```yaml +description: string -- Optional. Description of test case, purely for debugging purposes. +bls_setting: int -- see general test-format spec. +``` + +### `anchor_state.ssz_snappy` + +An SSZ-snappy encoded `BeaconState`, the state to initialize store with `get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock)` helper. + +### `anchor_block.ssz_snappy` + +An SSZ-snappy encoded `BeaconBlock`, the block to initialize store with `get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock)` helper. + +### `steps.yaml` + +The steps to execute in sequence. There may be multiple items of the following types: + +#### `on_tick` execution step + +The parameter that is required for executing `on_tick(store, time)`. + +```yaml +{ tick: int } -- to execute `on_tick(store, time)` +``` + +After this step, the `store` object may have been updated. + +#### `on_attestation` execution step + +The parameter that is required for executing `on_attestation(store, attestation)`. + +```yaml +{ attestation: string } -- the name of the `attestation_<32-byte-root>.ssz_snappy` file. To execute `on_attestation(store, attestation)` with the given attestation. +``` +The file is located in the same folder (see below). + +After this step, the `store` object may have been updated. + +#### `on_block` execution step + +The parameter that is required for executing `on_block(store, block)`. + +```yaml +{ block: string } -- the name of the `block_<32-byte-root>.ssz_snappy` file. To execute `on_block(store, block)` with the given attestation. +``` +The file is located in the same folder (see below). + +After this step, the `store` object may have been updated. + +#### Checks step + +The checks to verify the current status of `store`. + +```yaml +checks: {: value} -- the assertions. +``` + +`` is the field member or property of [`Store`](../../../specs/phase0/fork-choice.md#store) object that maintained by client implementation. Currently, the possible fields included: + +```yaml +head: { -- Encoded 32-byte value from get_head(store) + slot: slot, + root: string, +} +time: int -- store.time +genesis_time: int -- store.genesis_time +justified_checkpoint_root: string -- Encoded 32-byte value from store.justified_checkpoint.root +finalized_checkpoint_root: string -- Encoded 32-byte value from store.finalized_checkpoint.root +best_justified_checkpoint_root: string -- Encoded 32-byte value from store.best_justified_checkpoint.root +``` + +For example: +```yaml +- checks: + time: 144 + genesis_time: 0 + head: {slot: 17, root: '0xd2724c86002f7e1f8656ab44a341a409ad80e6e70a5225fd94835566deebb66f'} + justified_checkpoint_root: '0xcea6ecd3d3188e32ebf611f960eebd45b6c6f477a7cff242fa567a42653bfc7c' + finalized_checkpoint_root: '0xcea6ecd3d3188e32ebf611f960eebd45b6c6f477a7cff242fa567a42653bfc7c' + best_justified_checkpoint: '0xcea6ecd3d3188e32ebf611f960eebd45b6c6f477a7cff242fa567a42653bfc7c' +``` + +*Note*: Each `checks` step may include one or multiple items. Each item has to be checked against the current store. + +### `attestation_<32-byte-root>.ssz_snappy` + +`<32-byte-root>` is the hash tree root of the given attestation. + +Each file is an SSZ-snappy encoded `Attestation`. + +### `block_<32-byte-root>.ssz_snappy` + +`<32-byte-root>` is the hash tree root of the given block. + +Each file is an SSZ-snappy encoded `SignedBeaconBlock`. + +## Condition + +1. Deserialize `anchor_state.ssz_snappy` and `anchor_block.ssz_snappy` to initialize the local store object by with `get_forkchoice_store(anchor_state, anchor_block)` helper. +2. Iterate sequentially through `steps.yaml` + - For each execution, look up the corresponding ssz_snappy file. Execute the corresponding helper function on the current store. + - For the `on_block` execution step: if `len(block.message.body.attestations) > 0`, execute each attestation with `on_attestation(store, attestation)` after executing `on_block(store, block)`. + - For each `checks` step, the assertions on the current store must be satisfied. diff --git a/tests/generators/fork_choice/README.md b/tests/generators/fork_choice/README.md new file mode 100644 index 000000000..e67b115ba --- /dev/null +++ b/tests/generators/fork_choice/README.md @@ -0,0 +1,5 @@ +# Fork choice tests + +Fork choice tests cover the different forking cases with fork choice helper functions. + +Information on the format of the tests can be found in the [fork choice test formats documentation](../../formats/fork_choice/README.md). diff --git a/tests/generators/fork_choice/main.py b/tests/generators/fork_choice/main.py new file mode 100644 index 000000000..492e596f4 --- /dev/null +++ b/tests/generators/fork_choice/main.py @@ -0,0 +1,25 @@ +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators +from eth2spec.phase0 import spec as spec_phase0 +from eth2spec.altair import spec as spec_altair +from eth2spec.phase1 import spec as spec_phase1 +from eth2spec.test.context import PHASE0, PHASE1, ALTAIR + + +specs = (spec_phase0, spec_altair, spec_phase1) + + +if __name__ == "__main__": + phase_0_mods = {key: 'eth2spec.test.phase0.fork_choice.test_' + key for key in [ + 'get_head', + ]} + # No additional Altair or phase 1 specific finality tests, yet. + altair_mods = phase_0_mods + phase_1_mods = phase_0_mods + + all_mods = { + PHASE0: phase_0_mods, + ALTAIR: altair_mods, + PHASE1: phase_1_mods, + } + + run_state_test_generators(runner_name="fork_choice", specs=specs, all_mods=all_mods) diff --git a/tests/generators/fork_choice/requirements.txt b/tests/generators/fork_choice/requirements.txt new file mode 100644 index 000000000..182248686 --- /dev/null +++ b/tests/generators/fork_choice/requirements.txt @@ -0,0 +1,2 @@ +pytest>=4.4 +../../../[generator]