Merge pull request #1773 from ethereum/hwwhww/shard_fork_choice

Shard fork choice rule

setup.py

@@ -394,6 +394,7 @@ class PySpecCommand(Command):
                     specs/phase1/shard-transition.md
                     specs/phase1/fork-choice.md
                     specs/phase1/phase1-fork.md
+                    specs/phase1/shard-fork-choice.md
                 """
             else:
                 raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork)

specs/phase1/fork-choice.md

@@ -10,6 +10,9 @@
 
 - [Introduction](#introduction)
 - [Fork choice](#fork-choice)
+  - [Helpers](#helpers)
+    - [Extended `LatestMessage`](#extended-latestmessage)
+    - [Updated `update_latest_messages`](#updated-update_latest_messages)
   - [Handlers](#handlers)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -25,6 +28,33 @@ Due to the changes in the structure of `IndexedAttestation` in Phase 1, `on_atte
 
 The rest of the fork choice remains stable.
 
+### Helpers
+
+#### Extended `LatestMessage`
+
+```python
+@dataclass(eq=True, frozen=True)
+class LatestMessage(object):
+    epoch: Epoch
+    root: Root
+    shard: Shard
+    shard_root: Root
+```
+
+#### Updated `update_latest_messages`
+
+```python
+def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None:
+    target = attestation.data.target
+    beacon_block_root = attestation.data.beacon_block_root
+    shard = get_shard(store.block_states[beacon_block_root], attestation)
+    for i in attesting_indices:
+        if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch:
+            store.latest_messages[i] = LatestMessage(
+                epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.shard_head_root
+            )
+```
+
 ### Handlers
 
 ```python
@@ -49,4 +79,4 @@ def on_attestation(store: Store, attestation: Attestation) -> None:
         if attestation.aggregation_bits[i]
     ]
     update_latest_messages(store, attesting_indices, attestation)
-```
+```

specs/phase1/shard-fork-choice.md

@@ -0,0 +1,182 @@
+# Ethereum 2.0 Phase 1 -- Beacon Chain + Shard Chain Fork Choice
+
+**Notice**: This document is a work-in-progress for researchers and implementers.
+
+## Table of contents
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+
+- [Introduction](#introduction)
+- [Fork choice](#fork-choice)
+  - [Helpers](#helpers)
+    - [`ShardStore`](#shardstore)
+    - [`get_forkchoice_shard_store`](#get_forkchoice_shard_store)
+    - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance)
+    - [`get_shard_head`](#get_shard_head)
+    - [`get_shard_ancestor`](#get_shard_ancestor)
+    - [`get_pending_shard_blocks`](#get_pending_shard_blocks)
+  - [Handlers](#handlers)
+    - [`on_shard_block`](#on_shard_block)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+## Introduction
+
+This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. It assumes the [beacon chain fork choice spec](./fork-choice.md).
+
+## Fork choice
+
+### Helpers
+
+#### `ShardStore`
+
+```python
+@dataclass
+class ShardStore:
+    shard: Shard
+    blocks: Dict[Root, ShardBlock] = field(default_factory=dict)
+    block_states: Dict[Root, ShardState] = field(default_factory=dict)
+```
+
+#### `get_forkchoice_shard_store`
+
+```python
+def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore:
+    return ShardStore(
+        shard=shard,
+        blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot, shard=shard)},
+        block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]},
+    )
+```
+
+#### `get_shard_latest_attesting_balance`
+
+```python
+def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, root: Root) -> Gwei:
+    state = store.checkpoint_states[store.justified_checkpoint]
+    active_indices = get_active_validator_indices(state, get_current_epoch(state))
+    return Gwei(sum(
+        state.validators[i].effective_balance for i in active_indices
+        if (
+            i in store.latest_messages
+            # TODO: check the latest message logic: currently, validator's previous vote of another shard
+            # would be ignored once their newer vote is accepted. Check if it makes sense.
+            and store.latest_messages[i].shard == shard_store.shard
+            and get_shard_ancestor(
+                store, shard_store, store.latest_messages[i].shard_root, shard_store.blocks[root].slot
+            ) == root
+        )
+    ))
+```
+
+#### `get_shard_head`
+
+```python
+def get_shard_head(store: Store, shard_store: ShardStore) -> Root:
+    # Execute the LMD-GHOST fork choice
+    beacon_head_root = get_head(store)
+    shard_head_state = store.block_states[beacon_head_root].shard_states[shard_store.shard]
+    shard_head_root = shard_head_state.latest_block_root
+    shard_blocks = {
+        root: shard_block for root, shard_block in shard_store.blocks.items()
+        if shard_block.slot > shard_head_state.slot
+    }
+    while True:
+        # Find the valid child block roots
+        children = [
+            root for root, shard_block in shard_blocks.items()
+            if shard_block.shard_parent_root == shard_head_root
+        ]
+        if len(children) == 0:
+            return shard_head_root
+        # Sort by latest attesting balance with ties broken lexicographically
+        shard_head_root = max(
+            children, key=lambda root: (get_shard_latest_attesting_balance(store, shard_store, root), root)
+        )
+```
+
+#### `get_shard_ancestor`
+
+```python
+def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: Slot) -> Root:
+    block = shard_store.blocks[root]
+    if block.slot > slot:
+        return get_shard_ancestor(store, shard_store, block.shard_parent_root, slot)
+    elif block.slot == slot:
+        return root
+    else:
+        # root is older than queried slot, thus a skip slot. Return most recent root prior to slot
+        return root
+```
+
+#### `get_pending_shard_blocks`
+
+```python
+def get_pending_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]:
+    """
+    Return the canonical shard block branch that has not yet been crosslinked.
+    """
+    shard = shard_store.shard
+
+    beacon_head_root = get_head(store)
+    beacon_head_state = store.block_states[beacon_head_root]
+    latest_shard_block_root = beacon_head_state.shard_states[shard].latest_block_root
+
+    shard_head_root = get_shard_head(store, shard_store)
+    root = shard_head_root
+    shard_blocks = []
+    while root != latest_shard_block_root:
+        shard_block = shard_store.blocks[root]
+        shard_blocks.append(shard_block)
+        root = shard_block.shard_parent_root
+
+    shard_blocks.reverse()
+    return shard_blocks
+```
+
+### Handlers
+
+#### `on_shard_block`
+
+```python
+def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: SignedShardBlock) -> None:
+    shard_block = signed_shard_block.message
+    shard = shard_store.shard
+
+    # Check shard
+    # TODO: check it in networking spec
+    assert shard_block.shard == shard
+
+    # Check shard parent exists
+    assert shard_block.shard_parent_root in shard_store.block_states
+    shard_parent_state = shard_store.block_states[shard_block.shard_parent_root]
+
+    # Check beacon parent exists
+    assert shard_block.beacon_parent_root in store.block_states
+    beacon_parent_state = store.block_states[shard_block.beacon_parent_root]
+
+    # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor)
+    finalized_beacon_state = store.block_states[store.finalized_checkpoint.root]
+    finalized_shard_state = finalized_beacon_state.shard_states[shard]
+    assert shard_block.slot > finalized_shard_state.slot
+
+    # Check block is a descendant of the finalized block at the checkpoint finalized slot
+    finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)
+    assert (
+        get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root
+    )
+
+    # Check the block is valid and compute the post-state
+    assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block)
+    assert verify_shard_block_signature(beacon_parent_state, signed_shard_block)
+
+    post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block)
+
+    # Add new block to the store
+    shard_store.blocks[hash_tree_root(shard_block)] = shard_block
+
+    # Add new state for this block to the store
+    shard_store.block_states[hash_tree_root(shard_block)] = post_state
+```

specs/phase1/shard-transition.md

@@ -30,7 +30,7 @@ This document describes the shard transition function and fraud proofs as part o
 ### Misc
 
 ```python
-def compute_shard_transition_digest(beacon_state: BeaconState,
+def compute_shard_transition_digest(beacon_parent_state: BeaconState,
                                     shard_state: ShardState,
                                     beacon_parent_root: Root,
                                     shard_body_root: Root) -> Bytes32:
@@ -43,15 +43,27 @@ def compute_shard_transition_digest(beacon_state: BeaconState,
 ### Shard block verification functions
 
 ```python
-def verify_shard_block_message(beacon_state: BeaconState,
+def verify_shard_block_message(beacon_parent_state: BeaconState,
-                               shard_state: ShardState,
+                               shard_parent_state: ShardState,
-                               block: ShardBlock,
+                               block: ShardBlock) -> bool:
-                               slot: Slot,
+    # Check `shard_parent_root` field
-                               shard: Shard) -> bool:
+    assert block.shard_parent_root == shard_parent_state.latest_block_root
-    assert block.shard_parent_root == shard_state.latest_block_root
+    # Check `beacon_parent_root` field
-    assert block.slot == slot
+    beacon_parent_block_header = beacon_parent_state.latest_block_header.copy()
+    if beacon_parent_block_header.state_root == Root():
+        beacon_parent_block_header.state_root = hash_tree_root(beacon_parent_state)
+    beacon_parent_root = hash_tree_root(beacon_parent_block_header)
+    assert block.beacon_parent_root == beacon_parent_root
+    # Check `slot` field
+    shard = block.shard
+    next_slot = Slot(block.slot + 1)
+    offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_parent_state, shard), next_slot)
+    assert block.slot in offset_slots
+    # Check `shard` field
     assert block.shard == shard
-    assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard)
+    # Check `proposer_index` field
+    assert block.proposer_index == get_shard_proposer_index(beacon_parent_state, block.slot, shard)
+    # Check `body` field
     assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE
     return True
 ```
@@ -172,38 +184,9 @@ def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[
     return [hash_tree_root(proposal.message.body) for proposal in proposals]
 ```
 
-```python
-def get_proposal_choices_at_slot(beacon_state: BeaconState,
-                                 shard_state: ShardState,
-                                 slot: Slot,
-                                 shard: Shard,
-                                 shard_blocks: Sequence[SignedShardBlock],
-                                 validate_signature: bool=True) -> Sequence[SignedShardBlock]:
-    """
-    Return the valid shard blocks at the given ``slot``.
-    Note that this function doesn't change the state.
-    """
-    choices = []
-    shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot]
-    for block in shard_blocks_at_slot:
-        try:
-            # Verify block message and signature
-            # TODO these validations should have been checked upon receiving shard blocks.
-            assert verify_shard_block_message(beacon_state, shard_state, block.message, slot, shard)
-            if validate_signature:
-                assert verify_shard_block_signature(beacon_state, block)
-
-            shard_state = get_post_shard_state(beacon_state, shard_state, block.message)
-        except Exception:
-            pass  # TODO: throw error in the test helper
-        else:
-            choices.append(block)
-    return choices
-```
-
 ```python
 def get_proposal_at_slot(beacon_state: BeaconState,
-                         shard_state: ShardState,
+                         shard_parent_state: ShardState,
                          slot: Shard,
                          shard: Shard,
                          shard_blocks: Sequence[SignedShardBlock],
@@ -212,24 +195,17 @@ def get_proposal_at_slot(beacon_state: BeaconState,
     Return ``proposal``, ``shard_state`` of the given ``slot``.
     Note that this function doesn't change the state.
     """
-    choices = get_proposal_choices_at_slot(
+    shard_blocks = [block for block in shard_blocks if block.message.slot == slot]
-        beacon_state=beacon_state,
+    if len(shard_blocks) == 0:
-        shard_state=shard_state,
+        block = ShardBlock(slot=slot, shard=shard)
-        slot=slot,
-        shard=shard,
-        shard_blocks=shard_blocks,
-        validate_signature=validate_signature,
-    )
-    if len(choices) == 0:
-        block = ShardBlock(slot=slot)
         proposal = SignedShardBlock(message=block)
-    elif len(choices) == 1:
+    elif len(shard_blocks) == 1:
-        proposal = choices[0]
+        proposal = shard_blocks[0]
     else:
-        proposal = get_winning_proposal(beacon_state, choices)
+        proposal = get_winning_proposal(beacon_state, shard_blocks)
 
     # Apply state transition
-    shard_state = get_post_shard_state(beacon_state, shard_state, proposal.message)
+    shard_state = get_post_shard_state(beacon_state, shard_parent_state, proposal.message)
 
     return proposal, shard_state
 ```
@@ -244,10 +220,11 @@ def get_shard_state_transition_result(
     proposals = []
     shard_states = []
     shard_state = beacon_state.shard_states[shard]
-    for slot in get_offset_slots(beacon_state, shard):
+    offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), Slot(beacon_state.slot + 1))
+    for slot in offset_slots:
         proposal, shard_state = get_proposal_at_slot(
             beacon_state=beacon_state,
-            shard_state=shard_state,
+            shard_parent_state=shard_state,
             slot=slot,
             shard=shard,
             shard_blocks=shard_blocks,
@@ -269,7 +246,7 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y
 def get_shard_transition(beacon_state: BeaconState,
                          shard: Shard,
                          shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition:
-    offset_slots = get_offset_slots(beacon_state, shard)
+    offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), Slot(beacon_state.slot + 1))
     proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks)
 
     shard_block_lengths = []

tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py

@@ -1,41 +1,13 @@
 from eth2spec.test.context import with_all_phases, spec_state_test
 from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations
 from eth2spec.test.helpers.block import build_empty_block_for_next_slot
+from eth2spec.test.helpers.fork_choice import add_attestation_to_store, add_block_to_store, get_anchor_root
 from eth2spec.test.helpers.state import (
     next_epoch,
     state_transition_and_sign_block,
 )
 
 
-def add_block_to_store(spec, store, signed_block):
-    pre_state = store.block_states[signed_block.message.parent_root]
-    block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT
-
-    if store.time < block_time:
-        spec.on_tick(store, block_time)
-
-    spec.on_block(store, signed_block)
-
-
-def add_attestation_to_store(spec, store, attestation):
-    parent_block = store.blocks[attestation.data.beacon_block_root]
-    pre_state = store.block_states[spec.hash_tree_root(parent_block)]
-    block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT
-    next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT
-
-    if store.time < next_epoch_time:
-        spec.on_tick(store, next_epoch_time)
-
-    spec.on_attestation(store, attestation)
-
-
-def get_anchor_root(spec, state):
-    anchor_block_header = state.latest_block_header.copy()
-    if anchor_block_header.state_root == spec.Bytes32():
-        anchor_block_header.state_root = spec.hash_tree_root(state)
-    return spec.hash_tree_root(anchor_block_header)
-
-
 @with_all_phases
 @spec_state_test
 def test_genesis(spec, state):

tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py

@@ -18,18 +18,25 @@ def run_on_attestation(spec, state, store, attestation, valid=True):
 
     if spec.fork == PHASE0:
         sample_index = indexed_attestation.attesting_indices[0]
+        latest_message = spec.LatestMessage(
+            epoch=attestation.data.target.epoch,
+            root=attestation.data.beacon_block_root,
+        )
     else:
         attesting_indices = [
             index for i, index in enumerate(indexed_attestation.committee)
             if attestation.aggregation_bits[i]
         ]
         sample_index = attesting_indices[0]
-    assert (
+        latest_message = spec.LatestMessage(
-        store.latest_messages[sample_index] ==
-        spec.LatestMessage(
             epoch=attestation.data.target.epoch,
             root=attestation.data.beacon_block_root,
+            shard=spec.get_shard(state, attestation),
+            shard_root=attestation.data.shard_head_root,
         )
+
+    assert (
+        store.latest_messages[sample_index] == latest_message
     )
 
 

tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py

@@ -0,0 +1,132 @@
+from eth2spec.utils.ssz.ssz_impl import hash_tree_root
+
+from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls
+from eth2spec.test.helpers.attestations import get_valid_on_time_attestation
+from eth2spec.test.helpers.shard_block import (
+    build_shard_block,
+    get_shard_transitions,
+    get_committee_index_of_shard,
+)
+from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root
+from eth2spec.test.helpers.state import state_transition_and_sign_block
+from eth2spec.test.helpers.block import build_empty_block
+
+
+def run_on_shard_block(spec, store, shard_store, signed_block, valid=True):
+    if not valid:
+        try:
+            spec.on_shard_block(store, shard_store, signed_block)
+        except AssertionError:
+            return
+        else:
+            assert False
+
+    spec.on_shard_block(store, shard_store, signed_block)
+    assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message
+
+
+def apply_shard_block(spec, store, shard_store, beacon_parent_state, shard_blocks_buffer):
+    shard = shard_store.shard
+    body = b'\x56' * 4
+    shard_head_root = spec.get_shard_head(store, shard_store)
+    shard_parent_state = shard_store.block_states[shard_head_root]
+    assert shard_parent_state.slot != beacon_parent_state.slot
+    shard_block = build_shard_block(
+        spec, beacon_parent_state, shard,
+        shard_parent_state=shard_parent_state, slot=beacon_parent_state.slot, body=body, signed=True
+    )
+    shard_blocks_buffer.append(shard_block)
+    run_on_shard_block(spec, store, shard_store, shard_block)
+    assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root()
+
+
+def check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer):
+    pending_shard_blocks = [
+        spec.SignedShardBlock(message=b)
+        for b in spec.get_pending_shard_blocks(store, shard_store)
+    ]
+    assert pending_shard_blocks == shard_blocks_buffer
+
+
+def is_in_offset_sets(spec, beacon_head_state, shard):
+    offset_slots = spec.compute_offset_slots(
+        beacon_head_state.shard_states[shard].slot, beacon_head_state.slot + 1
+    )
+    return beacon_head_state.slot in offset_slots
+
+
+def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer):
+    store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH
+
+    shard = shard_store.shard
+    committee_index = get_committee_index_of_shard(spec, state, state.slot, shard)
+    has_shard_committee = committee_index is not None  # has committee of `shard` at this slot
+
+    beacon_block = build_empty_block(spec, state, slot=state.slot + 1)
+
+    # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block
+    if has_shard_committee and len(shard_blocks_buffer) > 0:
+        # Sanity check `get_pending_shard_blocks` function
+        check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer)
+        # Use temporary next state to get ShardTransition of shard block
+        shard_transitions = get_shard_transitions(
+            spec,
+            state,
+            shard_blocks={shard: shard_blocks_buffer},
+        )
+        shard_transition = shard_transitions[shard]
+        attestation = get_valid_on_time_attestation(
+            spec,
+            state,
+            index=committee_index,
+            shard_transition=shard_transition,
+            signed=False,
+        )
+        assert spec.get_shard(state, attestation) == shard
+        beacon_block.body.attestations = [attestation]
+        beacon_block.body.shard_transitions = shard_transitions
+
+        # Clear buffer
+        shard_blocks_buffer.clear()
+
+    signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block)  # transition!
+    add_block_to_store(spec, store, signed_beacon_block)
+    assert spec.get_head(store) == beacon_block.hash_tree_root()
+
+    # On shard block at transitioned `state.slot`
+    if is_in_offset_sets(spec, state, shard):
+        # The created shard block would be appended to `shard_blocks_buffer`
+        apply_shard_block(spec, store, shard_store, state, shard_blocks_buffer)
+
+    return has_shard_committee
+
+
+@with_all_phases_except([PHASE0])
+@spec_state_test
+@never_bls  # Set to never_bls for testing `check_pending_shard_blocks`
+def test_basic(spec, state):
+    spec.PHASE_1_GENESIS_SLOT = 0  # NOTE: mock genesis slot here
+    state = spec.upgrade_to_phase1(state)
+    shard = spec.Shard(1)
+
+    # Initialization
+    store = spec.get_forkchoice_store(state)
+    anchor_root = get_anchor_root(spec, state)
+    assert spec.get_head(store) == anchor_root
+
+    shard_store = spec.get_forkchoice_shard_store(state, shard)
+    shard_head_root = spec.get_shard_head(store, shard_store)
+    assert shard_head_root == state.shard_states[shard].latest_block_root
+    assert shard_store.block_states[shard_head_root].slot == 1
+    assert shard_store.block_states[shard_head_root] == state.shard_states[shard]
+
+    # For mainnet config, it's possible that only one committee of `shard` per epoch.
+    # we set this counter to test more rounds.
+    shard_committee_counter = 2
+    shard_blocks_buffer = []
+    while shard_committee_counter > 0:
+        has_shard_committee = apply_shard_and_beacon(
+            spec, state, store, shard_store, shard_blocks_buffer
+        )
+        if has_shard_committee:
+            shard_committee_counter -= 1

tests/core/pyspec/eth2spec/test/helpers/attestations.py

@@ -85,9 +85,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t
             # No shard transition -> no shard block
             shard = spec.get_shard(state, spec.Attestation(data=attestation_data))
             if on_time:
-                temp_state = state.copy()
+                shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[])
-                next_slot(spec, temp_state)
-                shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[])
                 lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1
                 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index]
                 attestation_data.shard_transition_root = shard_transition.hash_tree_root()
@@ -320,9 +318,7 @@ def next_epoch_with_attestations(spec,
                 for index in range(committees_per_slot):
                     if spec.fork == PHASE1:
                         shard = spec.compute_shard_from_committee_index(post_state, index, slot_to_attest)
-                        shard_transition = get_shard_transition_of_committee(
+                        shard_transition = get_shard_transition_of_committee(spec, post_state, index)
-                            spec, post_state, index, slot=slot_to_attest
-                        )
                         block.body.shard_transitions[shard] = shard_transition
                     else:
                         shard_transition = None

tests/core/pyspec/eth2spec/test/helpers/fork_choice.py

@@ -0,0 +1,27 @@
+def get_anchor_root(spec, state):
+    anchor_block_header = state.latest_block_header.copy()
+    if anchor_block_header.state_root == spec.Bytes32():
+        anchor_block_header.state_root = spec.hash_tree_root(state)
+    return spec.hash_tree_root(anchor_block_header)
+
+
+def add_block_to_store(spec, store, signed_block):
+    pre_state = store.block_states[signed_block.message.parent_root]
+    block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT
+
+    if store.time < block_time:
+        spec.on_tick(store, block_time)
+
+    spec.on_block(store, signed_block)
+
+
+def add_attestation_to_store(spec, store, attestation):
+    parent_block = store.blocks[attestation.data.beacon_block_root]
+    pre_state = store.block_states[spec.hash_tree_root(parent_block)]
+    block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT
+    next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT
+
+    if store.time < next_epoch_time:
+        spec.on_tick(store, next_epoch_time)
+
+    spec.on_attestation(store, attestation)

tests/core/pyspec/eth2spec/test/helpers/shard_block.py

@@ -1,6 +1,4 @@
-from eth2spec.test.helpers.attestations import get_valid_on_time_attestation
 from eth2spec.test.helpers.block import get_state_and_beacon_parent_root_at_slot
-from eth2spec.test.helpers.state import transition_to
 from eth2spec.test.helpers.keys import privkeys
 from eth2spec.utils import bls
 from eth2spec.utils.bls import only_with_bls
@@ -23,19 +21,21 @@ def build_shard_block(spec,
                       shard,
                       slot=None,
                       body=None,
+                      shard_parent_state=None,
                       signed=False):
-    shard_state = beacon_state.shard_states[shard]
+    if shard_parent_state is None:
+        shard_parent_state = beacon_state.shard_states[shard]
+
     if slot is None:
-        slot = shard_state.slot + 1
+        slot = shard_parent_state.slot + 1
 
     if body is None:
         body = b'\x56' * 128
 
-    proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard)
     beacon_state, beacon_parent_root = get_state_and_beacon_parent_root_at_slot(spec, beacon_state, slot)
-
+    proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard)
     block = spec.ShardBlock(
-        shard_parent_root=shard_state.latest_block_root,
+        shard_parent_root=shard_parent_state.latest_block_root,
         beacon_parent_root=beacon_parent_root,
         slot=slot,
         shard=shard,
@@ -52,15 +52,17 @@ def build_shard_block(spec,
     return signed_block
 
 
-def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot):
+def get_shard_transitions(spec, parent_beacon_state, shard_blocks):
-    temp_state = state.copy()
-    transition_to(spec, temp_state, on_time_slot)
     shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS
+    on_time_slot = parent_beacon_state.slot + 1
     for shard, blocks in shard_blocks.items():
-        offset_slots = spec.get_offset_slots(temp_state, shard)
+        offset_slots = spec.compute_offset_slots(
+            spec.get_latest_slot_for_shard(parent_beacon_state, shard),
+            on_time_slot,
+        )
         len_offset_slots = len(offset_slots)
-        assert len_offset_slots == on_time_slot - state.shard_states[shard].slot - 1
+        shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks)
-        shard_transition = spec.get_shard_transition(temp_state, shard, blocks)
+
         if len(blocks) > 0:
             shard_block_root = blocks[-1].message.hash_tree_root()
             assert shard_transition.shard_states[len_offset_slots - 1].latest_block_root == shard_block_root
@@ -70,17 +72,11 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot):
     return shard_transitions
 
 
-def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition):
+def get_committee_index_of_shard(spec, state, slot, shard):  # Optional[CommitteeIndex]
-    temp_state = state.copy()
+    active_shard_count = spec.get_active_shard_count(state)
-    transition_to(spec, temp_state, on_time_slot - 1)
+    committee_count = spec.get_committee_count_at_slot(state, slot)
-    attestation = get_valid_on_time_attestation(
+    start_shard = spec.get_start_shard(state, slot)
-        spec,
+    for committee_index in range(committee_count):
-        temp_state,
+        if (start_shard + committee_index) % active_shard_count == shard:
-        index=index,
+            return committee_index
-        shard_transition=shard_transition,
+    return None
-        signed=True,
-    )
-    assert attestation.data.slot == temp_state.slot
-    if shard_transition is not None:
-        assert attestation.data.shard_transition_root == shard_transition.hash_tree_root()
-    return attestation

tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py

@@ -1,5 +1,4 @@
 from eth2spec.test.context import expect_assertion_error
-from eth2spec.test.helpers.state import transition_to
 
 
 def run_shard_transitions_processing(spec, state, shard_transitions, attestations, valid=True):
@@ -29,15 +28,10 @@ def run_shard_transitions_processing(spec, state, shard_transitions, attestation
     yield 'post', state
 
 
-def get_shard_transition_of_committee(spec, state, committee_index, slot=None, shard_blocks=None):
+def get_shard_transition_of_committee(spec, state, committee_index, shard_blocks=None):
     if shard_blocks is None:
         shard_blocks = []
 
-    if slot is None:
-        slot = state.slot
-
     shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot)
-    temp_state = state.copy()
+    shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks)
-    transition_to(spec, temp_state, slot + 1)
-    shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=shard_blocks)
     return shard_transition

tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py

@@ -2,71 +2,64 @@ from eth2spec.test.context import (
     PHASE0,
     with_all_phases_except,
     spec_state_test,
-    always_bls,
 )
+from eth2spec.test.helpers.attestations import get_valid_on_time_attestation
 from eth2spec.test.helpers.shard_transitions import run_shard_transitions_processing
 from eth2spec.test.helpers.shard_block import (
-    build_attestation_with_shard_transition,
     build_shard_block,
-    build_shard_transitions_till_slot,
+    get_shard_transitions,
 )
-from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot
+from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot, next_slot
 
 
 def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True):
     state = transition_to_valid_shard_slot(spec, state)
-    init_slot = state.slot
     committee_index = spec.CommitteeIndex(0)
     shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1)
     assert state.shard_states[shard].slot == state.slot - 1
+    transition_to(spec, state, state.slot + target_len_offset_slot)
+    assert state.shard_states[shard].slot == state.slot - target_len_offset_slot - 1
 
     # Create SignedShardBlock
     body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE
-    shard_block = build_shard_block(spec, state, shard, body=body, signed=True)
+    shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True)
     shard_blocks = [shard_block]
-    # Create a shard_transitions that would be included at beacon block `state.slot + target_len_offset_slot`
+    shard_transitions = get_shard_transitions(
-    shard_transitions = build_shard_transitions_till_slot(
         spec,
         state,
         shard_blocks={shard: shard_blocks},
-        on_time_slot=state.slot + target_len_offset_slot,
     )
     shard_transition = shard_transitions[shard]
-    # Create an attestation that would be included at beacon block `state.slot + target_len_offset_slot`
+    attestation = get_valid_on_time_attestation(
-    attestation = build_attestation_with_shard_transition(
         spec,
         state,
         index=committee_index,
-        on_time_slot=state.slot + target_len_offset_slot,
         shard_transition=shard_transition,
+        signed=False,
     )
+    next_slot(spec, state)
     pre_gasprice = state.shard_states[shard].gasprice
-
-    transition_to(spec, state, state.slot + target_len_offset_slot)
     pre_shard_state = state.shard_states[shard]
-
     yield from run_shard_transitions_processing(spec, state, shard_transitions, [attestation], valid=valid)
 
     if valid:
-        # After state transition,
-        assert state.slot == init_slot + target_len_offset_slot
         shard_state = state.shard_states[shard]
         assert shard_state != pre_shard_state
         assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1]
-
+        assert shard_state.latest_block_root == shard_block.message.hash_tree_root()
         if target_len_offset_slot == 1:
             assert shard_state.gasprice > pre_gasprice
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
-@always_bls
 def test_basic_crosslinks(spec, state):
+    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
     yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=1, valid=True)
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
-@always_bls
 def test_multiple_offset_slots(spec, state):
-    yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=3, valid=True)
+    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
+    yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=2, valid=True)

tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py

@@ -4,37 +4,40 @@ from eth2spec.test.context import (
     PHASE0,
     with_all_phases_except,
     spec_state_test,
-    always_bls,
 )
+from eth2spec.test.helpers.attestations import get_valid_on_time_attestation
 from eth2spec.test.helpers.block import build_empty_block
 from eth2spec.test.helpers.shard_block import (
-    build_attestation_with_shard_transition,
     build_shard_block,
-    build_shard_transitions_till_slot,
+    get_shard_transitions,
 )
-from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot
+from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to
 
 
-def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index, valid=True):
+def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard, valid=True):
-    shard_transitions = build_shard_transitions_till_slot(
+    transition_to(spec, state, state.slot + target_len_offset_slot)
-        spec, state, shard_blocks, on_time_slot=state.slot + target_len_offset_slot
+
-    )
+    body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE
+    shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True)
+    shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]}
+
+    shard_transitions = get_shard_transitions(spec, state, shard_blocks)
     attestations = [
-        build_attestation_with_shard_transition(
+        get_valid_on_time_attestation(
             spec,
             state,
-            on_time_slot=state.slot + target_len_offset_slot,
             index=committee_index,
             shard_transition=shard_transitions[shard],
+            signed=True,
         )
         for shard in shard_blocks.keys()
     ]
 
-    # Propose beacon block at slot `x + 1`
+    beacon_block = build_empty_block(spec, state, slot=state.slot + 1)
-    beacon_block = build_empty_block(spec, state, slot=state.slot + target_len_offset_slot)
     beacon_block.body.attestations = attestations
     beacon_block.body.shard_transitions = shard_transitions
 
+    pre_gasprice = state.shard_states[shard].gasprice
     pre_shard_states = state.shard_states.copy()
     yield 'pre', state.copy()
     yield 'block', beacon_block
@@ -52,17 +55,18 @@ def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_off
             assert post_shard_state == shard_transitions[shard].shard_states[
                 len(shard_transitions[shard].shard_states) - 1
             ]
-            assert beacon_block.slot == shard_transitions[shard].shard_states[0].slot + target_len_offset_slot
             assert post_shard_state.slot == state.slot - 1
             if len(shard_blocks[shard]) == 0:
                 # `latest_block_root` is the same
                 assert post_shard_state.latest_block_root == pre_shard_states[shard].latest_block_root
+            if target_len_offset_slot == 1 and len(shard_blocks) > 0:
+                assert post_shard_state.gasprice > pre_gasprice
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
-@always_bls
 def test_process_beacon_block_with_normal_shard_transition(spec, state):
+    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
     state = transition_to_valid_shard_slot(spec, state)
 
     target_len_offset_slot = 1
@@ -70,25 +74,13 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state):
     shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1)
     assert state.shard_states[shard].slot == state.slot - 1
 
-    pre_gasprice = state.shard_states[shard].gasprice
+    yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard)
-
-    # Create SignedShardBlock at slot `shard_state.slot + 1`
-    body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE
-    shard_block = build_shard_block(spec, state, shard, body=body, signed=True)
-    shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]}
-
-    yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index)
-
-    shard_state = state.shard_states[shard]
-
-    if target_len_offset_slot == 1 and len(shard_blocks) > 0:
-        assert shard_state.gasprice > pre_gasprice
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
-@always_bls
 def test_process_beacon_block_with_empty_proposal_transition(spec, state):
+    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
     state = transition_to_valid_shard_slot(spec, state)
 
     target_len_offset_slot = 1
@@ -96,12 +88,4 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state):
     shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1)
     assert state.shard_states[shard].slot == state.slot - 1
 
-    # No new shard block
+    yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard)
-    shard_blocks = {}
-
-    pre_gasprice = state.shard_states[shard].gasprice
-
-    yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index)
-
-    if target_len_offset_slot == 1 and len(shard_blocks) > 0:
-        assert state.shard_states[shard].gasprice > pre_gasprice