Merge pull request #2487 from ethereum/on-block-tests

Rework + add some `on_block` tests

tests/core/pyspec/eth2spec/test/helpers/attestations.py

@@ -217,30 +217,13 @@ def next_slots_with_attestations(spec,
     post_state = state.copy()
     signed_blocks = []
     for _ in range(slot_count):
-        block = build_empty_block_for_next_slot(spec, post_state)
+        signed_block = state_transition_with_full_block(
-        if fill_cur_epoch and post_state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY:
-            slot_to_attest = post_state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1
-            if slot_to_attest >= spec.compute_start_slot_at_epoch(spec.get_current_epoch(post_state)):
-                attestations = _get_valid_attestation_at_slot(
-                    post_state,
             spec,
-                    slot_to_attest,
-                    participation_fn=participation_fn
-                )
-                for attestation in attestations:
-                    block.body.attestations.append(attestation)
-        if fill_prev_epoch:
-            slot_to_attest = post_state.slot - spec.SLOTS_PER_EPOCH + 1
-            attestations = _get_valid_attestation_at_slot(
             post_state,
-                spec,
+            fill_cur_epoch,
-                slot_to_attest,
+            fill_prev_epoch,
-                participation_fn=participation_fn
+            participation_fn,
         )
-            for attestation in attestations:
-                block.body.attestations.append(attestation)
-
-        signed_block = state_transition_and_sign_block(spec, post_state, block)
         signed_blocks.append(signed_block)
 
     return state, signed_blocks, post_state
@@ -249,7 +232,8 @@ def next_slots_with_attestations(spec,
 def next_epoch_with_attestations(spec,
                                  state,
                                  fill_cur_epoch,
-                                 fill_prev_epoch):
+                                 fill_prev_epoch,
+                                 participation_fn=None):
     assert state.slot % spec.SLOTS_PER_EPOCH == 0
 
     return next_slots_with_attestations(
@@ -258,9 +242,76 @@ def next_epoch_with_attestations(spec,
         spec.SLOTS_PER_EPOCH,
         fill_cur_epoch,
         fill_prev_epoch,
+        participation_fn,
     )
 
 
+def state_transition_with_full_block(spec, state, fill_cur_epoch, fill_prev_epoch, participation_fn=None):
+    """
+    Build and apply a block with attestions at the calculated `slot_to_attest` of current epoch and/or previous epoch.
+    """
+    block = build_empty_block_for_next_slot(spec, state)
+    if fill_cur_epoch and state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY:
+        slot_to_attest = state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1
+        if slot_to_attest >= spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)):
+            attestations = _get_valid_attestation_at_slot(
+                state,
+                spec,
+                slot_to_attest,
+                participation_fn=participation_fn
+            )
+            for attestation in attestations:
+                block.body.attestations.append(attestation)
+    if fill_prev_epoch:
+        slot_to_attest = state.slot - spec.SLOTS_PER_EPOCH + 1
+        attestations = _get_valid_attestation_at_slot(
+            state,
+            spec,
+            slot_to_attest,
+            participation_fn=participation_fn
+        )
+        for attestation in attestations:
+            block.body.attestations.append(attestation)
+
+    signed_block = state_transition_and_sign_block(spec, state, block)
+    return signed_block
+
+
+def state_transition_with_full_attestations_block(spec, state, fill_cur_epoch, fill_prev_epoch):
+    """
+    Build and apply a block with attestions at all valid slots of current epoch and/or previous epoch.
+    """
+    # Build a block with previous attestations
+    block = build_empty_block_for_next_slot(spec, state)
+    attestations = []
+
+    if fill_cur_epoch:
+        # current epoch
+        slots = state.slot % spec.SLOTS_PER_EPOCH
+        for slot_offset in range(slots):
+            target_slot = state.slot - slot_offset
+            attestations += _get_valid_attestation_at_slot(
+                state,
+                spec,
+                target_slot,
+            )
+
+    if fill_prev_epoch:
+        # attest previous epoch
+        slots = spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH
+        for slot_offset in range(1, slots):
+            target_slot = state.slot - (state.slot % spec.SLOTS_PER_EPOCH) - slot_offset
+            attestations += _get_valid_attestation_at_slot(
+                state,
+                spec,
+                target_slot,
+            )
+
+    block.body.attestations = attestations
+    signed_block = state_transition_and_sign_block(spec, state, block)
+    return signed_block
+
+
 def prepare_state_with_attestations(spec, state, participation_fn=None):
     """
     Prepare state with attestations according to the ``participation_fn``.

tests/core/pyspec/eth2spec/test/helpers/fork_choice.py

@@ -1,4 +1,8 @@
 from eth_utils import encode_hex
+from eth2spec.test.helpers.attestations import (
+    next_epoch_with_attestations,
+    next_slots_with_attestations,
+)
 
 
 def get_anchor_root(spec, state):
@@ -18,23 +22,20 @@ def add_block_to_store(spec, store, signed_block):
     spec.on_block(store, signed_block)
 
 
-def tick_and_run_on_block(spec, store, signed_block, test_steps=None):
+def tick_and_add_block(spec, store, signed_block, test_steps, valid=True, allow_invalid_attestations=False):
-    if test_steps is None:
-        test_steps = []
-
     pre_state = store.block_states[signed_block.message.parent_root]
     block_time = pre_state.genesis_time + signed_block.message.slot * spec.config.SECONDS_PER_SLOT
 
     if store.time < block_time:
         on_tick_and_append_step(spec, store, block_time, test_steps)
 
-    yield from run_on_block(spec, store, signed_block, test_steps)
+    post_state = yield from add_block(
+        spec, store, signed_block, test_steps, valid=valid, allow_invalid_attestations=allow_invalid_attestations)
+
+    return post_state
 
 
-def tick_and_run_on_attestation(spec, store, attestation, test_steps=None):
+def tick_and_run_on_attestation(spec, store, attestation, test_steps):
-    if test_steps is None:
-        test_steps = []
-
     parent_block = store.blocks[attestation.data.beacon_block_root]
     pre_state = store.block_states[spec.hash_tree_root(parent_block)]
     block_time = pre_state.genesis_time + parent_block.slot * spec.config.SECONDS_PER_SLOT
@@ -49,6 +50,37 @@ def tick_and_run_on_attestation(spec, store, attestation, test_steps=None):
     test_steps.append({'attestation': get_attestation_file_name(attestation)})
 
 
+def add_attestation(spec, store, attestation, test_steps, valid=True):
+    yield get_attestation_file_name(attestation), attestation
+
+    if not valid:
+        try:
+            run_on_attestation(spec, store, attestation, valid=True)
+        except AssertionError:
+            test_steps.append({
+                'attestation': get_attestation_file_name(attestation),
+                'valid': False,
+            })
+            return
+        else:
+            assert False
+
+    run_on_attestation(spec, store, attestation, valid=True)
+    test_steps.append({'attestation': get_attestation_file_name(attestation)})
+
+
+def run_on_attestation(spec, store, attestation, valid=True):
+    if not valid:
+        try:
+            spec.on_attestation(store, attestation)
+        except AssertionError:
+            return
+        else:
+            assert False
+
+    spec.on_attestation(store, attestation)
+
+
 def get_genesis_forkchoice_store(spec, genesis_state):
     store, _ = get_genesis_forkchoice_store_and_block(spec, genesis_state)
     return store
@@ -73,25 +105,53 @@ def on_tick_and_append_step(spec, store, time, test_steps):
     test_steps.append({'tick': int(time)})
 
 
-def run_on_block(spec, store, signed_block, test_steps, valid=True):
+def run_on_block(spec, store, signed_block, valid=True):
     if not valid:
         try:
             spec.on_block(store, signed_block)
-
         except AssertionError:
             return
         else:
             assert False
 
     spec.on_block(store, signed_block)
+    assert store.blocks[signed_block.message.hash_tree_root()] == signed_block.message
+
+
+def add_block(spec, store, signed_block, test_steps, valid=True, allow_invalid_attestations=False):
+    """
+    Run on_block and on_attestation
+    """
     yield get_block_file_name(signed_block), signed_block
+
+    if not valid:
+        try:
+            run_on_block(spec, store, signed_block, valid=True)
+        except AssertionError:
+            test_steps.append({
+                'block': get_block_file_name(signed_block),
+                'valid': False,
+            })
+            return
+        else:
+            assert False
+
+    run_on_block(spec, store, signed_block, valid=True)
     test_steps.append({'block': get_block_file_name(signed_block)})
 
     # An on_block step implies receiving block's attestations
+    try:
         for attestation in signed_block.message.body.attestations:
-        spec.on_attestation(store, attestation)
+            run_on_attestation(spec, store, attestation, valid=True)
+    except AssertionError:
+        if allow_invalid_attestations:
+            pass
+        else:
+            raise
 
-    assert store.blocks[signed_block.message.hash_tree_root()] == signed_block.message
+    block_root = signed_block.message.hash_tree_root()
+    assert store.blocks[block_root] == signed_block.message
+    assert store.block_states[block_root].hash_tree_root() == signed_block.message.state_root
     test_steps.append({
         'checks': {
             'time': int(store.time),
@@ -102,6 +162,8 @@ def run_on_block(spec, store, signed_block, test_steps, valid=True):
         }
     })
 
+    return store.block_states[signed_block.message.hash_tree_root()]
+
 
 def get_formatted_head_output(spec, store):
     head = spec.get_head(store)
@@ -110,3 +172,49 @@ def get_formatted_head_output(spec, store):
         'slot': int(slot),
         'root': encode_hex(head),
     }
+
+
+def apply_next_epoch_with_attestations(spec,
+                                       state,
+                                       store,
+                                       fill_cur_epoch,
+                                       fill_prev_epoch,
+                                       participation_fn=None,
+                                       test_steps=None):
+    if test_steps is None:
+        test_steps = []
+
+    _, new_signed_blocks, post_state = next_epoch_with_attestations(
+        spec, state, fill_cur_epoch, fill_prev_epoch, participation_fn=participation_fn)
+    for signed_block in new_signed_blocks:
+        block = signed_block.message
+        yield from tick_and_add_block(spec, store, signed_block, test_steps)
+        block_root = block.hash_tree_root()
+        assert store.blocks[block_root] == block
+        last_signed_block = signed_block
+
+    assert store.block_states[block_root].hash_tree_root() == post_state.hash_tree_root()
+
+    return post_state, store, last_signed_block
+
+
+def apply_next_slots_with_attestations(spec,
+                                       state,
+                                       store,
+                                       slots,
+                                       fill_cur_epoch,
+                                       fill_prev_epoch,
+                                       test_steps,
+                                       participation_fn=None):
+    _, new_signed_blocks, post_state = next_slots_with_attestations(
+        spec, state, slots, fill_cur_epoch, fill_prev_epoch, participation_fn=participation_fn)
+    for signed_block in new_signed_blocks:
+        block = signed_block.message
+        yield from tick_and_add_block(spec, store, signed_block, test_steps)
+        block_root = block.hash_tree_root()
+        assert store.blocks[block_root] == block
+        last_signed_block = signed_block
+
+    assert store.block_states[block_root].hash_tree_root() == post_state.hash_tree_root()
+
+    return post_state, store, last_signed_block

tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py

@@ -11,12 +11,12 @@ from eth2spec.test.helpers.block import build_empty_block_for_next_slot
 from eth2spec.test.helpers.constants import MINIMAL
 from eth2spec.test.helpers.fork_choice import (
     tick_and_run_on_attestation,
-    tick_and_run_on_block,
+    tick_and_add_block,
     get_anchor_root,
     get_genesis_forkchoice_store_and_block,
     get_formatted_head_output,
     on_tick_and_append_step,
-    run_on_block,
+    add_block,
 )
 from eth2spec.test.helpers.state import (
     next_epoch,
@@ -68,12 +68,12 @@ def test_chain_no_attestations(spec, state):
     # On receiving a block of `GENESIS_SLOT + 1` slot
     block_1 = build_empty_block_for_next_slot(spec, state)
     signed_block_1 = state_transition_and_sign_block(spec, state, block_1)
-    yield from tick_and_run_on_block(spec, store, signed_block_1, test_steps)
+    yield from tick_and_add_block(spec, store, signed_block_1, test_steps)
 
     # On receiving a block of next epoch
     block_2 = build_empty_block_for_next_slot(spec, state)
     signed_block_2 = state_transition_and_sign_block(spec, state, block_2)
-    yield from tick_and_run_on_block(spec, store, signed_block_2, test_steps)
+    yield from tick_and_add_block(spec, store, signed_block_2, test_steps)
 
     assert spec.get_head(store) == spec.hash_tree_root(block_2)
     test_steps.append({
@@ -107,14 +107,14 @@ def test_split_tie_breaker_no_attestations(spec, state):
     block_1_state = genesis_state.copy()
     block_1 = build_empty_block_for_next_slot(spec, block_1_state)
     signed_block_1 = state_transition_and_sign_block(spec, block_1_state, block_1)
-    yield from tick_and_run_on_block(spec, store, signed_block_1, test_steps)
+    yield from tick_and_add_block(spec, store, signed_block_1, test_steps)
 
     # additional block at slot 1
     block_2_state = genesis_state.copy()
     block_2 = build_empty_block_for_next_slot(spec, block_2_state)
     block_2.body.graffiti = b'\x42' * 32
     signed_block_2 = state_transition_and_sign_block(spec, block_2_state, block_2)
-    yield from tick_and_run_on_block(spec, store, signed_block_2, test_steps)
+    yield from tick_and_add_block(spec, store, signed_block_2, test_steps)
 
     highest_root = max(spec.hash_tree_root(block_1), spec.hash_tree_root(block_2))
     assert spec.get_head(store) == highest_root
@@ -150,14 +150,14 @@ def test_shorter_chain_but_heavier_weight(spec, state):
     for _ in range(3):
         long_block = build_empty_block_for_next_slot(spec, long_state)
         signed_long_block = state_transition_and_sign_block(spec, long_state, long_block)
-        yield from tick_and_run_on_block(spec, store, signed_long_block, test_steps)
+        yield from tick_and_add_block(spec, store, signed_long_block, test_steps)
 
     # build short tree
     short_state = genesis_state.copy()
     short_block = build_empty_block_for_next_slot(spec, short_state)
     short_block.body.graffiti = b'\x42' * 32
     signed_short_block = state_transition_and_sign_block(spec, short_state, short_block)
-    yield from tick_and_run_on_block(spec, store, signed_short_block, test_steps)
+    yield from tick_and_add_block(spec, store, signed_short_block, test_steps)
 
     short_attestation = get_valid_attestation(spec, short_state, short_block.slot, signed=True)
     yield from tick_and_run_on_attestation(spec, store, short_attestation, test_steps)
@@ -200,7 +200,7 @@ def test_filtered_block_tree(spec, state):
     current_time = state.slot * spec.config.SECONDS_PER_SLOT + store.genesis_time
     on_tick_and_append_step(spec, store, current_time, test_steps)
     for signed_block in signed_blocks:
-        yield from run_on_block(spec, store, signed_block, test_steps)
+        yield from add_block(spec, store, signed_block, test_steps)
 
     assert store.justified_checkpoint == state.current_justified_checkpoint
 
@@ -247,7 +247,7 @@ def test_filtered_block_tree(spec, state):
     on_tick_and_append_step(spec, store, current_time, test_steps)
 
     # include rogue block and associated attestations in the store
-    yield from run_on_block(spec, store, signed_rogue_block, test_steps)
+    yield from add_block(spec, store, signed_rogue_block, test_steps)
 
     for attestation in attestations:
         yield from tick_and_run_on_attestation(spec, store, attestation, test_steps)

tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_on_block.py

@@ -0,0 +1,689 @@
+import random
+
+from eth2spec.utils.ssz.ssz_impl import hash_tree_root
+from eth2spec.test.context import MINIMAL, spec_state_test, with_all_phases, with_presets
+from eth2spec.test.helpers.attestations import (
+    next_epoch_with_attestations,
+    next_slots_with_attestations,
+    state_transition_with_full_block,
+    state_transition_with_full_attestations_block,
+)
+from eth2spec.test.helpers.block import (
+    build_empty_block_for_next_slot,
+    build_empty_block,
+    transition_unsigned_block,
+    sign_block,
+)
+from eth2spec.test.helpers.fork_choice import (
+    get_genesis_forkchoice_store_and_block,
+    on_tick_and_append_step,
+    add_block,
+    tick_and_add_block,
+    apply_next_epoch_with_attestations,
+    apply_next_slots_with_attestations,
+)
+from eth2spec.test.helpers.state import (
+    next_epoch,
+    next_slots,
+    state_transition_and_sign_block,
+    transition_to,
+)
+
+
+rng = random.Random(2020)
+
+
+def _drop_random_one_third(_slot, _index, indices):
+    committee_len = len(indices)
+    assert committee_len >= 3
+    filter_len = committee_len // 3
+    participant_count = committee_len - filter_len
+    return rng.sample(indices, participant_count)
+
+
+@with_all_phases
+@spec_state_test
+def test_basic(spec, state):
+    test_steps = []
+    # Initialization
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
+    current_time = state.slot * spec.config.SECONDS_PER_SLOT + store.genesis_time
+    on_tick_and_append_step(spec, store, current_time, test_steps)
+    assert store.time == current_time
+
+    # On receiving a block of `GENESIS_SLOT + 1` slot
+    block = build_empty_block_for_next_slot(spec, state)
+    signed_block = state_transition_and_sign_block(spec, state, block)
+    yield from tick_and_add_block(spec, store, signed_block, test_steps)
+    assert spec.get_head(store) == signed_block.message.hash_tree_root()
+
+    # On receiving a block of next epoch
+    store.time = current_time + spec.config.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH
+    block = build_empty_block(spec, state, state.slot + spec.SLOTS_PER_EPOCH)
+    signed_block = state_transition_and_sign_block(spec, state, block)
+    yield from tick_and_add_block(spec, store, signed_block, test_steps)
+    assert spec.get_head(store) == signed_block.message.hash_tree_root()
+
+    yield 'steps', test_steps
+
+    # TODO: add tests for justified_root and finalized_root
+
+
+@with_all_phases
+@spec_state_test
+@with_presets([MINIMAL], reason="too slow")
+def test_on_block_checkpoints(spec, state):
+    test_steps = []
+    # Initialization
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
+    current_time = state.slot * spec.config.SECONDS_PER_SLOT + store.genesis_time
+    on_tick_and_append_step(spec, store, current_time, test_steps)
+    assert store.time == current_time
+
+    # Run for 1 epoch with full attestations
+    next_epoch(spec, state)
+    on_tick_and_append_step(spec, store, store.genesis_time + state.slot * spec.config.SECONDS_PER_SLOT, test_steps)
+
+    state, store, last_signed_block = yield from apply_next_epoch_with_attestations(
+        spec, state, store, True, False, test_steps=test_steps)
+    last_block_root = hash_tree_root(last_signed_block.message)
+    assert spec.get_head(store) == last_block_root
+
+    # Forward 1 epoch
+    next_epoch(spec, state)
+    on_tick_and_append_step(spec, store, store.genesis_time + state.slot * spec.config.SECONDS_PER_SLOT, test_steps)
+
+    # Mock the finalized_checkpoint and build a block on it
+    fin_state = store.block_states[last_block_root].copy()
+    fin_state.finalized_checkpoint = store.block_states[last_block_root].current_justified_checkpoint.copy()
+
+    block = build_empty_block_for_next_slot(spec, fin_state)
+    signed_block = state_transition_and_sign_block(spec, fin_state.copy(), block)
+    yield from tick_and_add_block(spec, store, signed_block, test_steps)
+    assert spec.get_head(store) == signed_block.message.hash_tree_root()
+    yield 'steps', test_steps
+
+
+@with_all_phases
+@spec_state_test
+def test_on_block_future_block(spec, state):
+    test_steps = []
+    # Initialization
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
+    current_time = state.slot * spec.config.SECONDS_PER_SLOT + store.genesis_time
+    on_tick_and_append_step(spec, store, current_time, test_steps)
+    assert store.time == current_time
+
+    # Do NOT tick time to `GENESIS_SLOT + 1` slot
+    # Fail receiving block of `GENESIS_SLOT + 1` slot
+    block = build_empty_block_for_next_slot(spec, state)
+    signed_block = state_transition_and_sign_block(spec, state, block)
+    yield from add_block(spec, store, signed_block, test_steps, valid=False)
+
+    yield 'steps', test_steps
+
+
+@with_all_phases
+@spec_state_test
+def test_on_block_bad_parent_root(spec, state):
+    test_steps = []
+    # Initialization
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
+    current_time = state.slot * spec.config.SECONDS_PER_SLOT + store.genesis_time
+    on_tick_and_append_step(spec, store, current_time, test_steps)
+    assert store.time == current_time
+
+    # Fail receiving block of `GENESIS_SLOT + 1` slot
+    block = build_empty_block_for_next_slot(spec, state)
+    transition_unsigned_block(spec, state, block)
+    block.state_root = state.hash_tree_root()
+
+    block.parent_root = b'\x45' * 32
+
+    signed_block = sign_block(spec, state, block)
+
+    yield from add_block(spec, store, signed_block, test_steps, valid=False)
+
+    yield 'steps', test_steps
+
+
+@with_all_phases
+@spec_state_test
+@with_presets([MINIMAL], reason="too slow")
+def test_on_block_before_finalized(spec, state):
+    test_steps = []
+    # Initialization
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
+    current_time = state.slot * spec.config.SECONDS_PER_SLOT + store.genesis_time
+    on_tick_and_append_step(spec, store, current_time, test_steps)
+    assert store.time == current_time
+
+    # Fork
+    another_state = state.copy()
+
+    # Create a finalized chain
+    for _ in range(4):
+        state, store, _ = yield from apply_next_epoch_with_attestations(
+            spec, state, store, True, False, test_steps=test_steps)
+    assert store.finalized_checkpoint.epoch == 2
+
+    # Fail receiving block of `GENESIS_SLOT + 1` slot
+    block = build_empty_block_for_next_slot(spec, another_state)
+    block.body.graffiti = b'\x12' * 32
+    signed_block = state_transition_and_sign_block(spec, another_state, block)
+    assert signed_block.message.hash_tree_root() not in store.blocks
+    yield from tick_and_add_block(spec, store, signed_block, test_steps, valid=False)
+
+    yield 'steps', test_steps
+
+
+@with_all_phases
+@spec_state_test
+@with_presets([MINIMAL], reason="too slow")
+def test_on_block_finalized_skip_slots(spec, state):
+    test_steps = []
+    # Initialization
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
+    current_time = state.slot * spec.config.SECONDS_PER_SLOT + store.genesis_time
+    on_tick_and_append_step(spec, store, current_time, test_steps)
+    assert store.time == current_time
+
+    # Create a finalized chain
+    for _ in range(4):
+        state, store, _ = yield from apply_next_epoch_with_attestations(
+            spec, state, store, True, False, test_steps=test_steps)
+    assert store.finalized_checkpoint.epoch == 2
+
+    # Another chain
+    another_state = store.block_states[store.finalized_checkpoint.root].copy()
+    # Build block that includes the skipped slots up to finality in chain
+    block = build_empty_block(spec,
+                              another_state,
+                              spec.compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + 2)
+    block.body.graffiti = b'\x12' * 32
+    signed_block = state_transition_and_sign_block(spec, another_state, block)
+
+    yield from tick_and_add_block(spec, store, signed_block, test_steps)
+
+    yield 'steps', test_steps
+
+
+@with_all_phases
+@spec_state_test
+@with_presets([MINIMAL], reason="too slow")
+def test_on_block_finalized_skip_slots_not_in_skip_chain(spec, state):
+    test_steps = []
+    # Initialization
+    transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH - 1)
+    block = build_empty_block_for_next_slot(spec, state)
+    transition_unsigned_block(spec, state, block)
+    block.state_root = state.hash_tree_root()
+    store = spec.get_forkchoice_store(state, block)
+    yield 'anchor_state', state
+    yield 'anchor_block', block
+
+    current_time = state.slot * spec.config.SECONDS_PER_SLOT + store.genesis_time
+    on_tick_and_append_step(spec, store, current_time, test_steps)
+    assert store.time == current_time
+
+    pre_finalized_checkpoint_epoch = store.finalized_checkpoint.epoch
+
+    # Finalized
+    for _ in range(3):
+        state, store, _ = yield from apply_next_epoch_with_attestations(
+            spec, state, store, True, False, test_steps=test_steps)
+    assert store.finalized_checkpoint.epoch == pre_finalized_checkpoint_epoch + 1
+
+    # Now build a block at later slot than finalized epoch
+    # Includes finalized block in chain, but not at appropriate skip slot
+    pre_state = store.block_states[block.hash_tree_root()].copy()
+    block = build_empty_block(spec,
+                              state=pre_state,
+                              slot=spec.compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + 2)
+    block.body.graffiti = b'\x12' * 32
+    signed_block = sign_block(spec, pre_state, block)
+    yield from tick_and_add_block(spec, store, signed_block, test_steps, valid=False)
+
+    yield 'steps', test_steps
+
+
+@with_all_phases
+@spec_state_test
+@with_presets([MINIMAL], reason="mainnet config requires too many pre-generated public/private keys")
+def test_on_block_update_justified_checkpoint_within_safe_slots(spec, state):
+    """
+    Test `should_update_justified_checkpoint`:
+    compute_slots_since_epoch_start(get_current_slot(store)) < SAFE_SLOTS_TO_UPDATE_JUSTIFIED
+    """
+    test_steps = []
+    # Initialization
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
+    current_time = state.slot * spec.config.SECONDS_PER_SLOT + store.genesis_time
+    on_tick_and_append_step(spec, store, current_time, test_steps)
+    assert store.time == current_time
+
+    # Skip epoch 0 & 1
+    for _ in range(2):
+        next_epoch(spec, state)
+    # Fill epoch 2
+    state, store, _ = yield from apply_next_epoch_with_attestations(
+        spec, state, store, True, False, test_steps=test_steps)
+    assert state.finalized_checkpoint.epoch == store.finalized_checkpoint.epoch == 0
+    assert state.current_justified_checkpoint.epoch == store.justified_checkpoint.epoch == 2
+    # Skip epoch 3 & 4
+    for _ in range(2):
+        next_epoch(spec, state)
+    # Epoch 5: Attest current epoch
+    state, store, _ = yield from apply_next_epoch_with_attestations(
+        spec, state, store, True, False, participation_fn=_drop_random_one_third, test_steps=test_steps)
+    assert state.finalized_checkpoint.epoch == store.finalized_checkpoint.epoch == 0
+    assert state.current_justified_checkpoint.epoch == 2
+    assert store.justified_checkpoint.epoch == 2
+    assert state.current_justified_checkpoint == store.justified_checkpoint
+
+    # Skip epoch 6
+    next_epoch(spec, state)
+
+    pre_state = state.copy()
+
+    # Build a block to justify epoch 5
+    signed_block = state_transition_with_full_block(spec, state, True, True)
+    assert state.finalized_checkpoint.epoch == 0
+    assert state.current_justified_checkpoint.epoch == 5
+    assert state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch
+    assert spec.get_current_slot(store) % spec.SLOTS_PER_EPOCH < spec.SAFE_SLOTS_TO_UPDATE_JUSTIFIED
+    # Run on_block
+    yield from tick_and_add_block(spec, store, signed_block, test_steps)
+    # Ensure justified_checkpoint has been changed but finality is unchanged
+    assert store.justified_checkpoint.epoch == 5
+    assert store.justified_checkpoint == state.current_justified_checkpoint
+    assert store.finalized_checkpoint.epoch == pre_state.finalized_checkpoint.epoch == 0
+
+    yield 'steps', test_steps
+
+
+@with_all_phases
+@with_presets([MINIMAL], reason="It assumes that `MAX_ATTESTATIONS` >= 2/3 attestations of an epoch")
+@spec_state_test
+def test_on_block_outside_safe_slots_but_finality(spec, state):
+    """
+    Test `should_update_justified_checkpoint` case
+    - compute_slots_since_epoch_start(get_current_slot(store)) > SAFE_SLOTS_TO_UPDATE_JUSTIFIED
+    - new_justified_checkpoint and store.justified_checkpoint.root are NOT conflicting
+
+    Thus should_update_justified_checkpoint returns True.
+
+    Part of this script is similar to `test_new_justified_is_later_than_store_justified`.
+    """
+    test_steps = []
+    # Initialization
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
+    current_time = state.slot * spec.config.SECONDS_PER_SLOT + store.genesis_time
+    on_tick_and_append_step(spec, store, current_time, test_steps)
+    assert store.time == current_time
+
+    # Skip epoch 0
+    next_epoch(spec, state)
+    # Fill epoch 1 to 3, attest current epoch
+    for _ in range(3):
+        state, store, _ = yield from apply_next_epoch_with_attestations(
+            spec, state, store, True, False, test_steps=test_steps)
+    assert state.finalized_checkpoint.epoch == store.finalized_checkpoint.epoch == 2
+    assert state.current_justified_checkpoint.epoch == store.justified_checkpoint.epoch == 3
+
+    # Skip epoch 4-6
+    for _ in range(3):
+        next_epoch(spec, state)
+
+    # epoch 7
+    state, store, _ = yield from apply_next_epoch_with_attestations(
+        spec, state, store, True, True, test_steps=test_steps)
+    assert state.finalized_checkpoint.epoch == 2
+    assert state.current_justified_checkpoint.epoch == 7
+
+    # epoch 8, attest the first 5 blocks
+    state, store, _ = yield from apply_next_slots_with_attestations(
+        spec, state, store, 5, True, True, test_steps)
+    assert state.finalized_checkpoint.epoch == store.finalized_checkpoint.epoch == 2
+    assert state.current_justified_checkpoint.epoch == store.justified_checkpoint.epoch == 7
+
+    # Propose a block at epoch 9, 5th slot
+    next_epoch(spec, state)
+    next_slots(spec, state, 4)
+    signed_block = state_transition_with_full_attestations_block(spec, state, True, True)
+    yield from tick_and_add_block(spec, store, signed_block, test_steps)
+    assert state.finalized_checkpoint.epoch == store.finalized_checkpoint.epoch == 2
+    assert state.current_justified_checkpoint.epoch == store.justified_checkpoint.epoch == 7
+
+    # Propose an empty block at epoch 10, SAFE_SLOTS_TO_UPDATE_JUSTIFIED + 2 slot
+    # This block would trigger justification and finality updates on store
+    next_epoch(spec, state)
+    next_slots(spec, state, 4)
+    block = build_empty_block_for_next_slot(spec, state)
+    signed_block = state_transition_and_sign_block(spec, state, block)
+    assert state.finalized_checkpoint.epoch == 7
+    assert state.current_justified_checkpoint.epoch == 8
+    # Step time past safe slots and run on_block
+    if store.time < spec.compute_time_at_slot(state, signed_block.message.slot):
+        time = store.genesis_time + signed_block.message.slot * spec.config.SECONDS_PER_SLOT
+        on_tick_and_append_step(spec, store, time, test_steps)
+    assert spec.get_current_slot(store) % spec.SLOTS_PER_EPOCH >= spec.SAFE_SLOTS_TO_UPDATE_JUSTIFIED
+    yield from add_block(spec, store, signed_block, test_steps)
+
+    # Ensure justified_checkpoint finality has been changed
+    assert store.finalized_checkpoint.epoch == 7
+    assert store.finalized_checkpoint == state.finalized_checkpoint
+    assert store.justified_checkpoint.epoch == 8
+    assert store.justified_checkpoint == state.current_justified_checkpoint
+
+    yield 'steps', test_steps
+
+
+@with_all_phases
+@with_presets([MINIMAL], reason="It assumes that `MAX_ATTESTATIONS` >= 2/3 attestations of an epoch")
+@spec_state_test
+def test_new_justified_is_later_than_store_justified(spec, state):
+    """
+    J: Justified
+    F: Finalized
+    fork_1_state (forked from genesis):
+        epoch
+        [0] <- [1] <- [2] <- [3] <- [4]
+         F                    J
+
+    fork_2_state (forked from fork_1_state's epoch 2):
+        epoch
+                       └──── [3] <- [4] <- [5] <- [6]
+         F                           J
+
+    fork_3_state (forked from genesis):
+        [0] <- [1] <- [2] <- [3] <- [4] <- [5]
+                              F      J
+    """
+    # The 1st fork, from genesis
+    fork_1_state = state.copy()
+    # The 3rd fork, from genesis
+    fork_3_state = state.copy()
+
+    test_steps = []
+    # Initialization
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
+    current_time = state.slot * spec.config.SECONDS_PER_SLOT + store.genesis_time
+    on_tick_and_append_step(spec, store, current_time, test_steps)
+    assert store.time == current_time
+
+    # ----- Process fork_1_state
+    # Skip epoch 0
+    next_epoch(spec, fork_1_state)
+    # Fill epoch 1 with previous epoch attestations
+    fork_1_state, store, _ = yield from apply_next_epoch_with_attestations(
+        spec, fork_1_state, store, False, True, test_steps=test_steps)
+
+    # Fork `fork_2_state` at the start of epoch 2
+    fork_2_state = fork_1_state.copy()
+    assert spec.get_current_epoch(fork_2_state) == 2
+
+    # Skip epoch 2
+    next_epoch(spec, fork_1_state)
+    # # Fill epoch 3 & 4 with previous epoch attestations
+    for _ in range(2):
+        fork_1_state, store, _ = yield from apply_next_epoch_with_attestations(
+            spec, fork_1_state, store, False, True, test_steps=test_steps)
+
+    assert fork_1_state.finalized_checkpoint.epoch == store.finalized_checkpoint.epoch == 0
+    assert fork_1_state.current_justified_checkpoint.epoch == store.justified_checkpoint.epoch == 3
+    assert store.justified_checkpoint == fork_1_state.current_justified_checkpoint
+
+    # ------ fork_2_state: Create a chain to set store.best_justified_checkpoint
+    # NOTE: The goal is to make `store.best_justified_checkpoint.epoch > store.justified_checkpoint.epoch`
+    all_blocks = []
+
+    # Proposed an empty block at epoch 2, 1st slot
+    block = build_empty_block_for_next_slot(spec, fork_2_state)
+    signed_block = state_transition_and_sign_block(spec, fork_2_state, block)
+    yield from tick_and_add_block(spec, store, signed_block, test_steps)
+    assert fork_2_state.current_justified_checkpoint.epoch == 0
+
+    # Skip to epoch 4
+    for _ in range(2):
+        next_epoch(spec, fork_2_state)
+        assert fork_2_state.current_justified_checkpoint.epoch == 0
+
+    # Propose a block at epoch 4, 5th slot
+    # Propose a block at epoch 5, 5th slot
+    for _ in range(2):
+        next_epoch(spec, fork_2_state)
+        next_slots(spec, fork_2_state, 4)
+        signed_block = state_transition_with_full_attestations_block(spec, fork_2_state, True, True)
+        yield from tick_and_add_block(spec, store, signed_block, test_steps)
+        assert fork_2_state.current_justified_checkpoint.epoch == 0
+
+    # Propose a block at epoch 6, SAFE_SLOTS_TO_UPDATE_JUSTIFIED + 2 slot
+    next_epoch(spec, fork_2_state)
+    next_slots(spec, fork_2_state, spec.SAFE_SLOTS_TO_UPDATE_JUSTIFIED + 2)
+    signed_block = state_transition_with_full_attestations_block(spec, fork_2_state, True, True)
+    assert fork_2_state.finalized_checkpoint.epoch == 0
+    assert fork_2_state.current_justified_checkpoint.epoch == 5
+    # Check SAFE_SLOTS_TO_UPDATE_JUSTIFIED
+    spec.on_tick(store, store.genesis_time + fork_2_state.slot * spec.config.SECONDS_PER_SLOT)
+    assert spec.compute_slots_since_epoch_start(spec.get_current_slot(store)) >= spec.SAFE_SLOTS_TO_UPDATE_JUSTIFIED
+    # Run on_block
+    yield from add_block(spec, store, signed_block, test_steps)
+    assert store.finalized_checkpoint.epoch == 0
+    assert store.justified_checkpoint.epoch == 3
+    assert store.best_justified_checkpoint.epoch == 5
+
+    # ------ fork_3_state: Create another chain to test the
+    # "Update justified if new justified is later than store justified" case
+    all_blocks = []
+    for _ in range(3):
+        next_epoch(spec, fork_3_state)
+
+    # epoch 3
+    _, signed_blocks, fork_3_state = next_epoch_with_attestations(spec, fork_3_state, True, True)
+    all_blocks += signed_blocks
+    assert fork_3_state.finalized_checkpoint.epoch == 0
+
+    # epoch 4, attest the first 5 blocks
+    _, blocks, fork_3_state = next_slots_with_attestations(spec, fork_3_state, 5, True, True)
+    all_blocks += blocks.copy()
+    assert fork_3_state.finalized_checkpoint.epoch == 0
+
+    # Propose a block at epoch 5, 5th slot
+    next_epoch(spec, fork_3_state)
+    next_slots(spec, fork_3_state, 4)
+    signed_block = state_transition_with_full_block(spec, fork_3_state, True, True)
+    all_blocks.append(signed_block.copy())
+    assert fork_3_state.finalized_checkpoint.epoch == 0
+
+    # Propose a block at epoch 6, 5th slot
+    next_epoch(spec, fork_3_state)
+    next_slots(spec, fork_3_state, 4)
+    signed_block = state_transition_with_full_block(spec, fork_3_state, True, True)
+    all_blocks.append(signed_block.copy())
+    assert fork_3_state.finalized_checkpoint.epoch == 3
+    assert fork_3_state.current_justified_checkpoint.epoch == 4
+
+    # FIXME: pending on the `on_block`, `on_attestation` fix
+    # # Apply blocks of `fork_3_state` to `store`
+    # for block in all_blocks:
+    #     if store.time < spec.compute_time_at_slot(fork_2_state, block.message.slot):
+    #         spec.on_tick(store, store.genesis_time + block.message.slot * spec.config.SECONDS_PER_SLOT)
+    #     # valid_attestations=False because the attestations are outdated (older than previous epoch)
+    #     yield from add_block(spec, store, block, test_steps, allow_invalid_attestations=False)
+
+    # assert store.finalized_checkpoint == fork_3_state.finalized_checkpoint
+    # assert (store.justified_checkpoint
+    #         == fork_3_state.current_justified_checkpoint
+    #         != store.best_justified_checkpoint)
+    # assert (store.best_justified_checkpoint
+    #         == fork_2_state.current_justified_checkpoint)
+
+    yield 'steps', test_steps
+
+
+@with_all_phases
+@spec_state_test
+def test_new_finalized_slot_is_not_justified_checkpoint_ancestor(spec, state):
+    """
+    J: Justified
+    F: Finalized
+    state (forked from genesis):
+        epoch
+        [0] <- [1] <- [2] <- [3] <- [4] <- [5]
+         F                    J
+
+    another_state (forked from epoch 0):
+         └──── [1] <- [2] <- [3] <- [4] <- [5]
+                       F      J
+    """
+    test_steps = []
+    # Initialization
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
+    current_time = state.slot * spec.config.SECONDS_PER_SLOT + store.genesis_time
+    on_tick_and_append_step(spec, store, current_time, test_steps)
+    assert store.time == current_time
+
+    # ----- Process state
+    # Goal: make `store.finalized_checkpoint.epoch == 0` and `store.justified_checkpoint.epoch == 3`
+    # Skip epoch 0
+    next_epoch(spec, state)
+
+    # Forking another_state
+    another_state = state.copy()
+
+    # Fill epoch 1 with previous epoch attestations
+    state, store, _ = yield from apply_next_epoch_with_attestations(
+        spec, state, store, False, True, test_steps=test_steps)
+    # Skip epoch 2
+    next_epoch(spec, state)
+    # Fill epoch 3 & 4 with previous epoch attestations
+    for _ in range(2):
+        state, store, _ = yield from apply_next_epoch_with_attestations(
+            spec, state, store, False, True, test_steps=test_steps)
+
+    assert state.finalized_checkpoint.epoch == store.finalized_checkpoint.epoch == 0
+    assert state.current_justified_checkpoint.epoch == store.justified_checkpoint.epoch == 3
+    assert store.justified_checkpoint == state.current_justified_checkpoint
+
+    # Create another chain
+    # Goal: make `another_state.finalized_checkpoint.epoch == 2` and `another_state.justified_checkpoint.epoch == 3`
+    all_blocks = []
+    # Fill epoch 1 & 2 with previous + current epoch attestations
+    for _ in range(3):
+        _, signed_blocks, another_state = next_epoch_with_attestations(spec, another_state, True, True)
+        all_blocks += signed_blocks
+
+    assert another_state.finalized_checkpoint.epoch == 2
+    assert another_state.current_justified_checkpoint.epoch == 3
+    assert state.finalized_checkpoint != another_state.finalized_checkpoint
+    assert state.current_justified_checkpoint != another_state.current_justified_checkpoint
+
+    # pre_store_justified_checkpoint_root = store.justified_checkpoint.root
+
+    # FIXME: pending on the `on_block`, `on_attestation` fix
+    # # Apply blocks of `another_state` to `store`
+    # for block in all_blocks:
+    #     # NOTE: Do not call `on_tick` here
+    #     yield from add_block(spec, store, block, test_steps, allow_invalid_attestations=True)
+
+    # finalized_slot = spec.compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)
+    # ancestor_at_finalized_slot = spec.get_ancestor(store, pre_store_justified_checkpoint_root, finalized_slot)
+    # assert ancestor_at_finalized_slot != store.finalized_checkpoint.root
+
+    # assert store.finalized_checkpoint == another_state.finalized_checkpoint
+    # assert store.justified_checkpoint == another_state.current_justified_checkpoint
+
+    yield 'steps', test_steps
+
+
+@with_all_phases
+@spec_state_test
+def test_new_finalized_slot_is_justified_checkpoint_ancestor(spec, state):
+    """
+    J: Justified
+    F: Finalized
+    state:
+        epoch
+        [0] <- [1] <- [2] <- [3] <- [4] <- [5]
+                       F             J
+
+    another_state (forked from state at epoch 3):
+                              └──── [4] <- [5]
+                              F      J
+    """
+    test_steps = []
+    # Initialization
+    store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state)
+    yield 'anchor_state', state
+    yield 'anchor_block', anchor_block
+    current_time = state.slot * spec.config.SECONDS_PER_SLOT + store.genesis_time
+    on_tick_and_append_step(spec, store, current_time, test_steps)
+    assert store.time == current_time
+
+    # Process state
+    next_epoch(spec, state)
+    spec.on_tick(store, store.genesis_time + state.slot * spec.config.SECONDS_PER_SLOT)
+
+    state, store, _ = yield from apply_next_epoch_with_attestations(
+        spec, state, store, False, True, test_steps=test_steps)
+
+    state, store, _ = yield from apply_next_epoch_with_attestations(
+        spec, state, store, True, False, test_steps=test_steps)
+    next_epoch(spec, state)
+
+    for _ in range(2):
+        state, store, _ = yield from apply_next_epoch_with_attestations(
+            spec, state, store, False, True, test_steps=test_steps)
+
+    assert state.finalized_checkpoint.epoch == store.finalized_checkpoint.epoch == 2
+    assert state.current_justified_checkpoint.epoch == store.justified_checkpoint.epoch == 4
+    assert store.justified_checkpoint == state.current_justified_checkpoint
+
+    # Create another chain
+    # Forking from epoch 3
+    all_blocks = []
+    slot = spec.compute_start_slot_at_epoch(3)
+    block_root = spec.get_block_root_at_slot(state, slot)
+    another_state = store.block_states[block_root].copy()
+    for _ in range(2):
+        _, signed_blocks, another_state = next_epoch_with_attestations(spec, another_state, True, True)
+        all_blocks += signed_blocks
+
+    assert another_state.finalized_checkpoint.epoch == 3
+    assert another_state.current_justified_checkpoint.epoch == 4
+
+    pre_store_justified_checkpoint_root = store.justified_checkpoint.root
+    for block in all_blocks:
+        # FIXME: Once `on_block` and `on_attestation` logic is fixed,
+        # fix test case and remove allow_invalid_attestations flag
+        yield from tick_and_add_block(spec, store, block, test_steps, allow_invalid_attestations=True)
+
+    finalized_slot = spec.compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)
+    ancestor_at_finalized_slot = spec.get_ancestor(store, pre_store_justified_checkpoint_root, finalized_slot)
+    assert ancestor_at_finalized_slot == store.finalized_checkpoint.root
+
+    assert store.finalized_checkpoint == another_state.finalized_checkpoint
+    assert store.justified_checkpoint != another_state.current_justified_checkpoint
+
+    yield 'steps', test_steps

tests/core/pyspec/eth2spec/test/phase0/unittests/fork_choice/test_on_block.py

@@ -1,239 +1,47 @@
 from copy import deepcopy
+
 from eth2spec.utils.ssz.ssz_impl import hash_tree_root
-
+from eth2spec.test.context import (
-from eth2spec.test.context import with_all_phases, spec_state_test
+    spec_state_test,
-from eth2spec.test.helpers.attestations import next_epoch_with_attestations
+    with_all_phases,
-from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block, transition_unsigned_block, \
-    build_empty_block
-from eth2spec.test.helpers.fork_choice import get_genesis_forkchoice_store
-from eth2spec.test.helpers.state import next_epoch, state_transition_and_sign_block, transition_to
-
-
-def run_on_block(spec, store, signed_block, valid=True):
-    if not valid:
-        try:
-            spec.on_block(store, signed_block)
-        except AssertionError:
-            return
-        else:
-            assert False
-
-    spec.on_block(store, signed_block)
-    assert store.blocks[hash_tree_root(signed_block.message)] == signed_block.message
-
-
-def apply_next_epoch_with_attestations(spec, state, store):
-    _, new_signed_blocks, post_state = next_epoch_with_attestations(spec, state, True, False)
-    for signed_block in new_signed_blocks:
-        block = signed_block.message
-        block_root = hash_tree_root(block)
-        store.blocks[block_root] = block
-        store.block_states[block_root] = post_state
-        last_signed_block = signed_block
-    spec.on_tick(store, store.time + state.slot * spec.config.SECONDS_PER_SLOT)
-    return post_state, store, last_signed_block
-
-
-@with_all_phases
-@spec_state_test
-def test_basic(spec, state):
-    # Initialization
-    store = get_genesis_forkchoice_store(spec, state)
-    time = 100
-    spec.on_tick(store, time)
-    assert store.time == time
-
-    # On receiving a block of `GENESIS_SLOT + 1` slot
-    block = build_empty_block_for_next_slot(spec, state)
-    signed_block = state_transition_and_sign_block(spec, state, block)
-    run_on_block(spec, store, signed_block)
-
-    # On receiving a block of next epoch
-    store.time = time + spec.config.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH
-    block = build_empty_block(spec, state, state.slot + spec.SLOTS_PER_EPOCH)
-    signed_block = state_transition_and_sign_block(spec, state, block)
-
-    run_on_block(spec, store, signed_block)
-
-    # TODO: add tests for justified_root and finalized_root
-
-
-@with_all_phases
-@spec_state_test
-def test_on_block_checkpoints(spec, state):
-    # Initialization
-    store = get_genesis_forkchoice_store(spec, state)
-    time = 100
-    spec.on_tick(store, time)
-
-    next_epoch(spec, state)
-    spec.on_tick(store, store.time + state.slot * spec.config.SECONDS_PER_SLOT)
-    state, store, last_signed_block = apply_next_epoch_with_attestations(spec, state, store)
-    next_epoch(spec, state)
-    spec.on_tick(store, store.time + state.slot * spec.config.SECONDS_PER_SLOT)
-    last_block_root = hash_tree_root(last_signed_block.message)
-
-    # Mock the finalized_checkpoint
-    fin_state = store.block_states[last_block_root]
-    fin_state.finalized_checkpoint = (
-        store.block_states[last_block_root].current_justified_checkpoint
 )
-
+from eth2spec.test.helpers.block import (
-    block = build_empty_block_for_next_slot(spec, fin_state)
+    build_empty_block_for_next_slot,
-    signed_block = state_transition_and_sign_block(spec, deepcopy(fin_state), block)
-    run_on_block(spec, store, signed_block)
-
-
-@with_all_phases
-@spec_state_test
-def test_on_block_future_block(spec, state):
-    # Initialization
-    store = get_genesis_forkchoice_store(spec, state)
-
-    # do not tick time
-
-    # Fail receiving block of `GENESIS_SLOT + 1` slot
-    block = build_empty_block_for_next_slot(spec, state)
-    signed_block = state_transition_and_sign_block(spec, state, block)
-    run_on_block(spec, store, signed_block, False)
-
-
-@with_all_phases
-@spec_state_test
-def test_on_block_bad_parent_root(spec, state):
-    # Initialization
-    store = get_genesis_forkchoice_store(spec, state)
-    time = 100
-    spec.on_tick(store, time)
-
-    # Fail receiving block of `GENESIS_SLOT + 1` slot
-    block = build_empty_block_for_next_slot(spec, state)
-    transition_unsigned_block(spec, state, block)
-    block.state_root = state.hash_tree_root()
-
-    block.parent_root = b'\x45' * 32
-
-    signed_block = sign_block(spec, state, block)
-
-    run_on_block(spec, store, signed_block, False)
-
-
-@with_all_phases
-@spec_state_test
-def test_on_block_before_finalized(spec, state):
-    # Initialization
-    store = get_genesis_forkchoice_store(spec, state)
-    time = 100
-    spec.on_tick(store, time)
-
-    store.finalized_checkpoint = spec.Checkpoint(
-        epoch=store.finalized_checkpoint.epoch + 2,
-        root=store.finalized_checkpoint.root
 )
-
+from eth2spec.test.helpers.fork_choice import (
-    # Fail receiving block of `GENESIS_SLOT + 1` slot
+    get_genesis_forkchoice_store,
-    block = build_empty_block_for_next_slot(spec, state)
+    run_on_block,
-    signed_block = state_transition_and_sign_block(spec, state, block)
+    apply_next_epoch_with_attestations,
-    run_on_block(spec, store, signed_block, False)
-
-
-@with_all_phases
-@spec_state_test
-def test_on_block_finalized_skip_slots(spec, state):
-    # Initialization
-    store = get_genesis_forkchoice_store(spec, state)
-    time = 100
-    spec.on_tick(store, time)
-
-    store.finalized_checkpoint = spec.Checkpoint(
-        epoch=store.finalized_checkpoint.epoch + 2,
-        root=store.finalized_checkpoint.root
 )
-
+from eth2spec.test.helpers.state import (
-    # Build block that includes the skipped slots up to finality in chain
+    next_epoch,
-    block = build_empty_block(spec, state, spec.compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + 2)
+    state_transition_and_sign_block,
-    signed_block = state_transition_and_sign_block(spec, state, block)
-    spec.on_tick(store, store.time + state.slot * spec.config.SECONDS_PER_SLOT)
-    run_on_block(spec, store, signed_block)
-
-
-@with_all_phases
-@spec_state_test
-def test_on_block_finalized_skip_slots_not_in_skip_chain(spec, state):
-    # Initialization
-    transition_to(spec, state, state.slot + spec.SLOTS_PER_EPOCH - 1)
-    block = build_empty_block_for_next_slot(spec, state)
-    transition_unsigned_block(spec, state, block)
-    block.state_root = state.hash_tree_root()
-    store = spec.get_forkchoice_store(state, block)
-    store.finalized_checkpoint = spec.Checkpoint(
-        epoch=store.finalized_checkpoint.epoch + 2,
-        root=store.finalized_checkpoint.root
 )
 
-    # First transition through the epoch to ensure no skipped slots
-    state, store, _ = apply_next_epoch_with_attestations(spec, state, store)
-
-    # Now build a block at later slot than finalized epoch
-    # Includes finalized block in chain, but not at appropriate skip slot
-    block = build_empty_block(spec, state, spec.compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + 2)
-    signed_block = state_transition_and_sign_block(spec, state, block)
-    spec.on_tick(store, store.time + state.slot * spec.config.SECONDS_PER_SLOT)
-    run_on_block(spec, store, signed_block, False)
-
-
-@with_all_phases
-@spec_state_test
-def test_on_block_update_justified_checkpoint_within_safe_slots(spec, state):
-    # Initialization
-    store = get_genesis_forkchoice_store(spec, state)
-    time = 0
-    spec.on_tick(store, time)
-
-    next_epoch(spec, state)
-    spec.on_tick(store, store.time + state.slot * spec.config.SECONDS_PER_SLOT)
-    state, store, last_signed_block = apply_next_epoch_with_attestations(spec, state, store)
-    next_epoch(spec, state)
-    spec.on_tick(store, store.time + state.slot * spec.config.SECONDS_PER_SLOT)
-    last_block_root = hash_tree_root(last_signed_block.message)
-
-    # Mock the justified checkpoint
-    just_state = store.block_states[last_block_root]
-    new_justified = spec.Checkpoint(
-        epoch=just_state.current_justified_checkpoint.epoch + 1,
-        root=b'\x77' * 32,
-    )
-    just_state.current_justified_checkpoint = new_justified
-
-    block = build_empty_block_for_next_slot(spec, just_state)
-    signed_block = state_transition_and_sign_block(spec, deepcopy(just_state), block)
-    assert spec.get_current_slot(store) % spec.SLOTS_PER_EPOCH < spec.SAFE_SLOTS_TO_UPDATE_JUSTIFIED
-    run_on_block(spec, store, signed_block)
-
-    assert store.justified_checkpoint == new_justified
-
 
 @with_all_phases
 @spec_state_test
 def test_on_block_outside_safe_slots_and_multiple_better_justified(spec, state):
+    """
+    NOTE: test_new_justified_is_later_than_store_justified also tests best_justified_checkpoint
+    """
     # Initialization
     store = get_genesis_forkchoice_store(spec, state)
-    time = 0
-    spec.on_tick(store, time)
 
     next_epoch(spec, state)
-    spec.on_tick(store, store.time + state.slot * spec.config.SECONDS_PER_SLOT)
+    spec.on_tick(store, store.genesis_time + state.slot * spec.config.SECONDS_PER_SLOT)
-    state, store, last_signed_block = apply_next_epoch_with_attestations(spec, state, store)
+    state, store, last_signed_block = yield from apply_next_epoch_with_attestations(
+        spec, state, store, True, False)
     last_block_root = hash_tree_root(last_signed_block.message)
 
-    # Mock fictitious justified checkpoint in store
+    # NOTE: Mock fictitious justified checkpoint in store
     store.justified_checkpoint = spec.Checkpoint(
         epoch=spec.compute_epoch_at_slot(last_signed_block.message.slot),
         root=spec.Root("0x4a55535449464945440000000000000000000000000000000000000000000000")
     )
 
     next_epoch(spec, state)
-    spec.on_tick(store, store.time + state.slot * spec.config.SECONDS_PER_SLOT)
+    spec.on_tick(store, store.genesis_time + state.slot * spec.config.SECONDS_PER_SLOT)
 
     # Create new higher justified checkpoint not in branch of store's justified checkpoint
     just_block = build_empty_block_for_next_slot(spec, state)
@@ -243,11 +51,13 @@ def test_on_block_outside_safe_slots_and_multiple_better_justified(spec, state):
     spec.on_tick(store, store.time + spec.SAFE_SLOTS_TO_UPDATE_JUSTIFIED * spec.config.SECONDS_PER_SLOT)
     assert spec.get_current_slot(store) % spec.SLOTS_PER_EPOCH >= spec.SAFE_SLOTS_TO_UPDATE_JUSTIFIED
 
+    previously_finalized = store.finalized_checkpoint
     previously_justified = store.justified_checkpoint
 
     # Add a series of new blocks with "better" justifications
     best_justified_checkpoint = spec.Checkpoint(epoch=0)
     for i in range(3, 0, -1):
+        # Mutate store
         just_state = store.block_states[last_block_root]
         new_justified = spec.Checkpoint(
             epoch=previously_justified.epoch + i,
@@ -261,63 +71,17 @@ def test_on_block_outside_safe_slots_and_multiple_better_justified(spec, state):
         block = build_empty_block_for_next_slot(spec, just_state)
         signed_block = state_transition_and_sign_block(spec, deepcopy(just_state), block)
 
+        # NOTE: Mock store so that the modified state could be accessed
+        parent_block = store.blocks[last_block_root].copy()
+        parent_block.state_root = just_state.hash_tree_root()
+        store.blocks[block.parent_root] = parent_block
+        store.block_states[block.parent_root] = just_state.copy()
+        assert block.parent_root in store.blocks.keys()
+        assert block.parent_root in store.block_states.keys()
+
         run_on_block(spec, store, signed_block)
 
+    assert store.finalized_checkpoint == previously_finalized
     assert store.justified_checkpoint == previously_justified
     # ensure the best from the series was stored
     assert store.best_justified_checkpoint == best_justified_checkpoint
-
-
-@with_all_phases
-@spec_state_test
-def test_on_block_outside_safe_slots_but_finality(spec, state):
-    # Initialization
-    store = get_genesis_forkchoice_store(spec, state)
-    time = 100
-    spec.on_tick(store, time)
-
-    next_epoch(spec, state)
-    spec.on_tick(store, store.time + state.slot * spec.config.SECONDS_PER_SLOT)
-    state, store, last_signed_block = apply_next_epoch_with_attestations(spec, state, store)
-    last_block_root = hash_tree_root(last_signed_block.message)
-
-    # Mock fictitious justified checkpoint in store
-    store.justified_checkpoint = spec.Checkpoint(
-        epoch=spec.compute_epoch_at_slot(last_signed_block.message.slot),
-        root=spec.Root("0x4a55535449464945440000000000000000000000000000000000000000000000")
-    )
-
-    next_epoch(spec, state)
-    spec.on_tick(store, store.time + state.slot * spec.config.SECONDS_PER_SLOT)
-
-    # Create new higher justified checkpoint not in branch of store's justified checkpoint
-    just_block = build_empty_block_for_next_slot(spec, state)
-    store.blocks[just_block.hash_tree_root()] = just_block
-
-    # Step time past safe slots
-    spec.on_tick(store, store.time + spec.SAFE_SLOTS_TO_UPDATE_JUSTIFIED * spec.config.SECONDS_PER_SLOT)
-    assert spec.get_current_slot(store) % spec.SLOTS_PER_EPOCH >= spec.SAFE_SLOTS_TO_UPDATE_JUSTIFIED
-
-    # Mock justified and finalized update in state
-    just_fin_state = store.block_states[last_block_root]
-    new_justified = spec.Checkpoint(
-        epoch=spec.compute_epoch_at_slot(just_block.slot) + 1,
-        root=just_block.hash_tree_root(),
-    )
-    assert new_justified.epoch > store.justified_checkpoint.epoch
-    new_finalized = spec.Checkpoint(
-        epoch=spec.compute_epoch_at_slot(just_block.slot),
-        root=just_block.parent_root,
-    )
-    assert new_finalized.epoch > store.finalized_checkpoint.epoch
-    just_fin_state.current_justified_checkpoint = new_justified
-    just_fin_state.finalized_checkpoint = new_finalized
-
-    # Build and add block that includes the new justified/finalized info
-    block = build_empty_block_for_next_slot(spec, just_fin_state)
-    signed_block = state_transition_and_sign_block(spec, deepcopy(just_fin_state), block)
-
-    run_on_block(spec, store, signed_block)
-
-    assert store.finalized_checkpoint == new_finalized
-    assert store.justified_checkpoint == new_justified

tests/formats/fork_choice/README.md

@@ -28,7 +28,11 @@ The steps to execute in sequence. There may be multiple items of the following t
 The parameter that is required for executing `on_tick(store, time)`.
 
 ```yaml
-{ tick: int }  --  to execute `on_tick(store, time)`
+{
+    tick: int       -- to execute `on_tick(store, time)`.
+    valid: bool     -- optional, default to `true`.
+                       If it's `false`, this execution step is expected to be invalid.
+}
 ```
 
 After this step, the `store` object may have been updated.
@@ -38,7 +42,12 @@ After this step, the `store` object may have been updated.
 The parameter that is required for executing `on_attestation(store, attestation)`.
 
 ```yaml
-{ attestation: string }  -- the name of the `attestation_<32-byte-root>.ssz_snappy` file. To execute `on_attestation(store, attestation)` with the given attestation.
+{
+    attestation: string  -- the name of the `attestation_<32-byte-root>.ssz_snappy` file.
+                            To execute `on_attestation(store, attestation)` with the given attestation.
+    valid: bool          -- optional, default to `true`.
+                            If it's `false`, this execution step is expected to be invalid.
+}
 ```
 The file is located in the same folder (see below).
 
@@ -49,7 +58,12 @@ After this step, the `store` object may have been updated.
 The parameter that is required for executing `on_block(store, block)`.
 
 ```yaml
-{ block: string }  -- the name of the `block_<32-byte-root>.ssz_snappy` file. To execute `on_block(store, block)` with the given attestation.
+{
+    block: string  -- the name of the `block_<32-byte-root>.ssz_snappy` file.
+                      To execute `on_block(store, block)` with the given attestation.
+    valid: bool    -- optional, default to `true`.
+                      If it's `false`, this execution step is expected to be invalid.
+}  
 ```
 The file is located in the same folder (see below).
 

tests/generators/fork_choice/main.py

@@ -5,6 +5,7 @@ from eth2spec.test.helpers.constants import PHASE0, ALTAIR, MERGE
 if __name__ == "__main__":
     phase_0_mods = {key: 'eth2spec.test.phase0.fork_choice.test_' + key for key in [
         'get_head',
+        'on_block',
     ]}
     # No additional Altair specific finality tests, yet.
     altair_mods = phase_0_mods