Merge branch 'dev' into phase1-validator

setup.py

@@ -394,6 +394,7 @@ class PySpecCommand(Command):
                     specs/phase1/shard-transition.md
                     specs/phase1/fork-choice.md
                     specs/phase1/phase1-fork.md
+                    specs/phase1/shard-fork-choice.md
                     specs/phase1/validator.md
                 """
             else:

specs/phase0/p2p-interface.md

@@ -263,7 +263,7 @@ Additional global topics are used to propagate lower frequency validator message
     - _[IGNORE]_ The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`.
     - _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation.
 - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. The following validations MUST pass before forwarding the `proposer_slashing` on to the network
-    - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.index`.
+    - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.signed_header_1.message.proposer_index`.
     - _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation.
 - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network.
     - _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`).
@@ -275,7 +275,7 @@ Additional global topics are used to propagate lower frequency validator message
 Attestation subnets are used to propagate unaggregated attestations to subsections of the network. Their `Name`s are:
 
 - `beacon_attestation_{subnet_id}` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet.
-    - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation) == subnet_id`).
+    - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.index) == subnet_id`).
     - _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot).
     - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len(get_attesting_indices(state, attestation.data, attestation.aggregation_bits)) == 1`).
     - _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index.
@@ -286,7 +286,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio
 
 Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. The correct subnet for an attestation can be calculated with `compute_subnet_for_attestation`. `beacon_attestation_{subnet_id}` topics, are rotated through throughout the epoch in a similar fashion to rotating through shards in committees in Phase 1.
 
-Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` as `Attestation`s.
+Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.index)}` as `Attestation`s.
 
 Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `AggregateAndProof`s.
 

specs/phase0/validator.md

@@ -199,8 +199,8 @@ The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahe
 
 Specifically a validator should:
 * Call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments.
-* Find peers of the pubsub topic `committee_index{committee_index % ATTESTATION_SUBNET_COUNT}_beacon_attestation`.
+* Find peers of the pubsub topic `beacon_attestation_{compute_subnet_for_attestation(state, slot, committee_index)}`.
-    * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][committee_index % ATTESTATION_SUBNET_COUNT] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field.
+    * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][compute_subnet_for_attestation(state, slot, committee_index)] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field.
     * If the validator is assigned to be an aggregator for the slot (see `is_aggregator()`), then subscribe to the topic.
 
 *Note*: If the validator is _not_ assigned to be an aggregator, the validator only needs sufficient number of peers on the topic to be able to publish messages. The validator does not need to _subscribe_ and listen to all messages on the topic.
@@ -425,18 +425,18 @@ def get_attestation_signature(state: BeaconState, attestation_data: AttestationD
 
 #### Broadcast attestation
 
-Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` pubsub topic.
+Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.committee_index)}` pubsub topic.
 
 ```python
-def compute_subnet_for_attestation(state: BeaconState, attestation: Attestation) -> uint64:
+def compute_subnet_for_attestation(state: BeaconState, slot: Slot, committee_index: CommitteeIndex) -> uint64:
     """
     Compute the correct subnet for an attestation for Phase 0.
     Note, this mimics expected Phase 1 behavior where attestations will be mapped to their shard subnet.
     """
-    slots_since_epoch_start = attestation.data.slot % SLOTS_PER_EPOCH
+    slots_since_epoch_start = slot % SLOTS_PER_EPOCH
-    committees_since_epoch_start = get_committee_count_at_slot(state, attestation.data.slot) * slots_since_epoch_start
+    committees_since_epoch_start = get_committee_count_at_slot(state, slot) * slots_since_epoch_start
 
-    return (committees_since_epoch_start + attestation.data.index) % ATTESTATION_SUBNET_COUNT
+    return (committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT
 ```
 
 ### Attestation aggregation

specs/phase1/beacon-chain.md

@@ -545,8 +545,13 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque
 
 ```python
 def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex:
-    committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard)
+    """
-    r = bytes_to_int(get_seed(beacon_state, get_current_epoch(beacon_state), DOMAIN_SHARD_COMMITTEE)[:8])
+    Return the proposer's index of shard block at ``slot``.
+    """
+    epoch = compute_epoch_at_slot(slot)
+    committee = get_shard_committee(beacon_state, epoch, shard)
+    seed = hash(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE) + int_to_bytes(slot, length=8))
+    r = bytes_to_int(seed[:8])
     return committee[r % len(committee)]
 ```
 

specs/phase1/fork-choice.md

@@ -10,6 +10,9 @@
 
 - [Introduction](#introduction)
 - [Fork choice](#fork-choice)
+  - [Helpers](#helpers)
+    - [Extended `LatestMessage`](#extended-latestmessage)
+    - [Updated `update_latest_messages`](#updated-update_latest_messages)
   - [Handlers](#handlers)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -25,6 +28,33 @@ Due to the changes in the structure of `IndexedAttestation` in Phase 1, `on_atte
 
 The rest of the fork choice remains stable.
 
+### Helpers
+
+#### Extended `LatestMessage`
+
+```python
+@dataclass(eq=True, frozen=True)
+class LatestMessage(object):
+    epoch: Epoch
+    root: Root
+    shard: Shard
+    shard_root: Root
+```
+
+#### Updated `update_latest_messages`
+
+```python
+def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None:
+    target = attestation.data.target
+    beacon_block_root = attestation.data.beacon_block_root
+    shard = get_shard(store.block_states[beacon_block_root], attestation)
+    for i in attesting_indices:
+        if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch:
+            store.latest_messages[i] = LatestMessage(
+                epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.shard_head_root
+            )
+```
+
 ### Handlers
 
 ```python
@@ -49,4 +79,4 @@ def on_attestation(store: Store, attestation: Attestation) -> None:
         if attestation.aggregation_bits[i]
     ]
     update_latest_messages(store, attesting_indices, attestation)
-```
+```

specs/phase1/shard-fork-choice.md

@@ -0,0 +1,182 @@
+# Ethereum 2.0 Phase 1 -- Beacon Chain + Shard Chain Fork Choice
+
+**Notice**: This document is a work-in-progress for researchers and implementers.
+
+## Table of contents
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+
+- [Introduction](#introduction)
+- [Fork choice](#fork-choice)
+  - [Helpers](#helpers)
+    - [`ShardStore`](#shardstore)
+    - [`get_forkchoice_shard_store`](#get_forkchoice_shard_store)
+    - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance)
+    - [`get_shard_head`](#get_shard_head)
+    - [`get_shard_ancestor`](#get_shard_ancestor)
+    - [`get_pending_shard_blocks`](#get_pending_shard_blocks)
+  - [Handlers](#handlers)
+    - [`on_shard_block`](#on_shard_block)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+## Introduction
+
+This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. It assumes the [beacon chain fork choice spec](./fork-choice.md).
+
+## Fork choice
+
+### Helpers
+
+#### `ShardStore`
+
+```python
+@dataclass
+class ShardStore:
+    shard: Shard
+    blocks: Dict[Root, ShardBlock] = field(default_factory=dict)
+    block_states: Dict[Root, ShardState] = field(default_factory=dict)
+```
+
+#### `get_forkchoice_shard_store`
+
+```python
+def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore:
+    return ShardStore(
+        shard=shard,
+        blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot, shard=shard)},
+        block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]},
+    )
+```
+
+#### `get_shard_latest_attesting_balance`
+
+```python
+def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, root: Root) -> Gwei:
+    state = store.checkpoint_states[store.justified_checkpoint]
+    active_indices = get_active_validator_indices(state, get_current_epoch(state))
+    return Gwei(sum(
+        state.validators[i].effective_balance for i in active_indices
+        if (
+            i in store.latest_messages
+            # TODO: check the latest message logic: currently, validator's previous vote of another shard
+            # would be ignored once their newer vote is accepted. Check if it makes sense.
+            and store.latest_messages[i].shard == shard_store.shard
+            and get_shard_ancestor(
+                store, shard_store, store.latest_messages[i].shard_root, shard_store.blocks[root].slot
+            ) == root
+        )
+    ))
+```
+
+#### `get_shard_head`
+
+```python
+def get_shard_head(store: Store, shard_store: ShardStore) -> Root:
+    # Execute the LMD-GHOST fork choice
+    beacon_head_root = get_head(store)
+    shard_head_state = store.block_states[beacon_head_root].shard_states[shard_store.shard]
+    shard_head_root = shard_head_state.latest_block_root
+    shard_blocks = {
+        root: shard_block for root, shard_block in shard_store.blocks.items()
+        if shard_block.slot > shard_head_state.slot
+    }
+    while True:
+        # Find the valid child block roots
+        children = [
+            root for root, shard_block in shard_blocks.items()
+            if shard_block.shard_parent_root == shard_head_root
+        ]
+        if len(children) == 0:
+            return shard_head_root
+        # Sort by latest attesting balance with ties broken lexicographically
+        shard_head_root = max(
+            children, key=lambda root: (get_shard_latest_attesting_balance(store, shard_store, root), root)
+        )
+```
+
+#### `get_shard_ancestor`
+
+```python
+def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: Slot) -> Root:
+    block = shard_store.blocks[root]
+    if block.slot > slot:
+        return get_shard_ancestor(store, shard_store, block.shard_parent_root, slot)
+    elif block.slot == slot:
+        return root
+    else:
+        # root is older than queried slot, thus a skip slot. Return most recent root prior to slot
+        return root
+```
+
+#### `get_pending_shard_blocks`
+
+```python
+def get_pending_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]:
+    """
+    Return the canonical shard block branch that has not yet been crosslinked.
+    """
+    shard = shard_store.shard
+
+    beacon_head_root = get_head(store)
+    beacon_head_state = store.block_states[beacon_head_root]
+    latest_shard_block_root = beacon_head_state.shard_states[shard].latest_block_root
+
+    shard_head_root = get_shard_head(store, shard_store)
+    root = shard_head_root
+    shard_blocks = []
+    while root != latest_shard_block_root:
+        shard_block = shard_store.blocks[root]
+        shard_blocks.append(shard_block)
+        root = shard_block.shard_parent_root
+
+    shard_blocks.reverse()
+    return shard_blocks
+```
+
+### Handlers
+
+#### `on_shard_block`
+
+```python
+def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: SignedShardBlock) -> None:
+    shard_block = signed_shard_block.message
+    shard = shard_store.shard
+
+    # Check shard
+    # TODO: check it in networking spec
+    assert shard_block.shard == shard
+
+    # Check shard parent exists
+    assert shard_block.shard_parent_root in shard_store.block_states
+    shard_parent_state = shard_store.block_states[shard_block.shard_parent_root]
+
+    # Check beacon parent exists
+    assert shard_block.beacon_parent_root in store.block_states
+    beacon_parent_state = store.block_states[shard_block.beacon_parent_root]
+
+    # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor)
+    finalized_beacon_state = store.block_states[store.finalized_checkpoint.root]
+    finalized_shard_state = finalized_beacon_state.shard_states[shard]
+    assert shard_block.slot > finalized_shard_state.slot
+
+    # Check block is a descendant of the finalized block at the checkpoint finalized slot
+    finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)
+    assert (
+        get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root
+    )
+
+    # Check the block is valid and compute the post-state
+    assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block)
+    assert verify_shard_block_signature(beacon_parent_state, signed_shard_block)
+
+    post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block)
+
+    # Add new block to the store
+    shard_store.blocks[hash_tree_root(shard_block)] = shard_block
+
+    # Add new state for this block to the store
+    shard_store.block_states[hash_tree_root(shard_block)] = post_state
+```

specs/phase1/shard-transition.md

@@ -27,7 +27,7 @@ This document describes the shard transition function and fraud proofs as part o
 ### Misc
 
 ```python
-def compute_shard_transition_digest(beacon_state: BeaconState,
+def compute_shard_transition_digest(beacon_parent_state: BeaconState,
                                     shard_state: ShardState,
                                     beacon_parent_root: Root,
                                     shard_body_root: Root) -> Bytes32:
@@ -40,15 +40,27 @@ def compute_shard_transition_digest(beacon_state: BeaconState,
 ### Shard block verification functions
 
 ```python
-def verify_shard_block_message(beacon_state: BeaconState,
+def verify_shard_block_message(beacon_parent_state: BeaconState,
-                               shard_state: ShardState,
+                               shard_parent_state: ShardState,
-                               block: ShardBlock,
+                               block: ShardBlock) -> bool:
-                               slot: Slot,
+    # Check `shard_parent_root` field
-                               shard: Shard) -> bool:
+    assert block.shard_parent_root == shard_parent_state.latest_block_root
-    assert block.shard_parent_root == shard_state.latest_block_root
+    # Check `beacon_parent_root` field
-    assert block.slot == slot
+    beacon_parent_block_header = beacon_parent_state.latest_block_header.copy()
+    if beacon_parent_block_header.state_root == Root():
+        beacon_parent_block_header.state_root = hash_tree_root(beacon_parent_state)
+    beacon_parent_root = hash_tree_root(beacon_parent_block_header)
+    assert block.beacon_parent_root == beacon_parent_root
+    # Check `slot` field
+    shard = block.shard
+    next_slot = Slot(block.slot + 1)
+    offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_parent_state, shard), next_slot)
+    assert block.slot in offset_slots
+    # Check `shard` field
     assert block.shard == shard
-    assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard)
+    # Check `proposer_index` field
+    assert block.proposer_index == get_shard_proposer_index(beacon_parent_state, block.slot, shard)
+    # Check `body` field
     assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE
     return True
 ```

tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py

@@ -1,41 +1,13 @@
 from eth2spec.test.context import with_all_phases, spec_state_test
 from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations
 from eth2spec.test.helpers.block import build_empty_block_for_next_slot
+from eth2spec.test.helpers.fork_choice import add_attestation_to_store, add_block_to_store, get_anchor_root
 from eth2spec.test.helpers.state import (
     next_epoch,
     state_transition_and_sign_block,
 )
 
 
-def add_block_to_store(spec, store, signed_block):
-    pre_state = store.block_states[signed_block.message.parent_root]
-    block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT
-
-    if store.time < block_time:
-        spec.on_tick(store, block_time)
-
-    spec.on_block(store, signed_block)
-
-
-def add_attestation_to_store(spec, store, attestation):
-    parent_block = store.blocks[attestation.data.beacon_block_root]
-    pre_state = store.block_states[spec.hash_tree_root(parent_block)]
-    block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT
-    next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT
-
-    if store.time < next_epoch_time:
-        spec.on_tick(store, next_epoch_time)
-
-    spec.on_attestation(store, attestation)
-
-
-def get_anchor_root(spec, state):
-    anchor_block_header = state.latest_block_header.copy()
-    if anchor_block_header.state_root == spec.Bytes32():
-        anchor_block_header.state_root = spec.hash_tree_root(state)
-    return spec.hash_tree_root(anchor_block_header)
-
-
 @with_all_phases
 @spec_state_test
 def test_genesis(spec, state):

tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py

@@ -18,18 +18,25 @@ def run_on_attestation(spec, state, store, attestation, valid=True):
 
     if spec.fork == PHASE0:
         sample_index = indexed_attestation.attesting_indices[0]
+        latest_message = spec.LatestMessage(
+            epoch=attestation.data.target.epoch,
+            root=attestation.data.beacon_block_root,
+        )
     else:
         attesting_indices = [
             index for i, index in enumerate(indexed_attestation.committee)
             if attestation.aggregation_bits[i]
         ]
         sample_index = attesting_indices[0]
-    assert (
+        latest_message = spec.LatestMessage(
-        store.latest_messages[sample_index] ==
-        spec.LatestMessage(
             epoch=attestation.data.target.epoch,
             root=attestation.data.beacon_block_root,
+            shard=spec.get_shard(state, attestation),
+            shard_root=attestation.data.shard_head_root,
         )
+
+    assert (
+        store.latest_messages[sample_index] == latest_message
     )
 
 

tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py

@@ -0,0 +1,132 @@
+from eth2spec.utils.ssz.ssz_impl import hash_tree_root
+
+from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls
+from eth2spec.test.helpers.attestations import get_valid_on_time_attestation
+from eth2spec.test.helpers.shard_block import (
+    build_shard_block,
+    get_shard_transitions,
+    get_committee_index_of_shard,
+)
+from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root
+from eth2spec.test.helpers.state import state_transition_and_sign_block
+from eth2spec.test.helpers.block import build_empty_block
+
+
+def run_on_shard_block(spec, store, shard_store, signed_block, valid=True):
+    if not valid:
+        try:
+            spec.on_shard_block(store, shard_store, signed_block)
+        except AssertionError:
+            return
+        else:
+            assert False
+
+    spec.on_shard_block(store, shard_store, signed_block)
+    assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message
+
+
+def apply_shard_block(spec, store, shard_store, beacon_parent_state, shard_blocks_buffer):
+    shard = shard_store.shard
+    body = b'\x56' * 4
+    shard_head_root = spec.get_shard_head(store, shard_store)
+    shard_parent_state = shard_store.block_states[shard_head_root]
+    assert shard_parent_state.slot != beacon_parent_state.slot
+    shard_block = build_shard_block(
+        spec, beacon_parent_state, shard,
+        shard_parent_state=shard_parent_state, slot=beacon_parent_state.slot, body=body, signed=True
+    )
+    shard_blocks_buffer.append(shard_block)
+    run_on_shard_block(spec, store, shard_store, shard_block)
+    assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root()
+
+
+def check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer):
+    pending_shard_blocks = [
+        spec.SignedShardBlock(message=b)
+        for b in spec.get_pending_shard_blocks(store, shard_store)
+    ]
+    assert pending_shard_blocks == shard_blocks_buffer
+
+
+def is_in_offset_sets(spec, beacon_head_state, shard):
+    offset_slots = spec.compute_offset_slots(
+        beacon_head_state.shard_states[shard].slot, beacon_head_state.slot + 1
+    )
+    return beacon_head_state.slot in offset_slots
+
+
+def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer):
+    store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH
+
+    shard = shard_store.shard
+    committee_index = get_committee_index_of_shard(spec, state, state.slot, shard)
+    has_shard_committee = committee_index is not None  # has committee of `shard` at this slot
+
+    beacon_block = build_empty_block(spec, state, slot=state.slot + 1)
+
+    # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block
+    if has_shard_committee and len(shard_blocks_buffer) > 0:
+        # Sanity check `get_pending_shard_blocks` function
+        check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer)
+        # Use temporary next state to get ShardTransition of shard block
+        shard_transitions = get_shard_transitions(
+            spec,
+            state,
+            shard_blocks={shard: shard_blocks_buffer},
+        )
+        shard_transition = shard_transitions[shard]
+        attestation = get_valid_on_time_attestation(
+            spec,
+            state,
+            index=committee_index,
+            shard_transition=shard_transition,
+            signed=False,
+        )
+        assert spec.get_shard(state, attestation) == shard
+        beacon_block.body.attestations = [attestation]
+        beacon_block.body.shard_transitions = shard_transitions
+
+        # Clear buffer
+        shard_blocks_buffer.clear()
+
+    signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block)  # transition!
+    add_block_to_store(spec, store, signed_beacon_block)
+    assert spec.get_head(store) == beacon_block.hash_tree_root()
+
+    # On shard block at transitioned `state.slot`
+    if is_in_offset_sets(spec, state, shard):
+        # The created shard block would be appended to `shard_blocks_buffer`
+        apply_shard_block(spec, store, shard_store, state, shard_blocks_buffer)
+
+    return has_shard_committee
+
+
+@with_all_phases_except([PHASE0])
+@spec_state_test
+@never_bls  # Set to never_bls for testing `check_pending_shard_blocks`
+def test_basic(spec, state):
+    spec.PHASE_1_GENESIS_SLOT = 0  # NOTE: mock genesis slot here
+    state = spec.upgrade_to_phase1(state)
+    shard = spec.Shard(1)
+
+    # Initialization
+    store = spec.get_forkchoice_store(state)
+    anchor_root = get_anchor_root(spec, state)
+    assert spec.get_head(store) == anchor_root
+
+    shard_store = spec.get_forkchoice_shard_store(state, shard)
+    shard_head_root = spec.get_shard_head(store, shard_store)
+    assert shard_head_root == state.shard_states[shard].latest_block_root
+    assert shard_store.block_states[shard_head_root].slot == 1
+    assert shard_store.block_states[shard_head_root] == state.shard_states[shard]
+
+    # For mainnet config, it's possible that only one committee of `shard` per epoch.
+    # we set this counter to test more rounds.
+    shard_committee_counter = 2
+    shard_blocks_buffer = []
+    while shard_committee_counter > 0:
+        has_shard_committee = apply_shard_and_beacon(
+            spec, state, store, shard_store, shard_blocks_buffer
+        )
+        if has_shard_committee:
+            shard_committee_counter -= 1

tests/core/pyspec/eth2spec/test/helpers/attestations.py

@@ -318,9 +318,7 @@ def next_epoch_with_attestations(spec,
                 for index in range(committees_per_slot):
                     if spec.fork == PHASE1:
                         shard = spec.compute_shard_from_committee_index(post_state, index, slot_to_attest)
-                        shard_transition = get_shard_transition_of_committee(
+                        shard_transition = get_shard_transition_of_committee(spec, post_state, index)
-                            spec, post_state, index, slot=slot_to_attest
-                        )
                         block.body.shard_transitions[shard] = shard_transition
                     else:
                         shard_transition = None

tests/core/pyspec/eth2spec/test/helpers/fork_choice.py

@@ -0,0 +1,27 @@
+def get_anchor_root(spec, state):
+    anchor_block_header = state.latest_block_header.copy()
+    if anchor_block_header.state_root == spec.Bytes32():
+        anchor_block_header.state_root = spec.hash_tree_root(state)
+    return spec.hash_tree_root(anchor_block_header)
+
+
+def add_block_to_store(spec, store, signed_block):
+    pre_state = store.block_states[signed_block.message.parent_root]
+    block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT
+
+    if store.time < block_time:
+        spec.on_tick(store, block_time)
+
+    spec.on_block(store, signed_block)
+
+
+def add_attestation_to_store(spec, store, attestation):
+    parent_block = store.blocks[attestation.data.beacon_block_root]
+    pre_state = store.block_states[spec.hash_tree_root(parent_block)]
+    block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT
+    next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT
+
+    if store.time < next_epoch_time:
+        spec.on_tick(store, next_epoch_time)
+
+    spec.on_attestation(store, attestation)

tests/core/pyspec/eth2spec/test/helpers/shard_block.py

@@ -1,4 +1,3 @@
-from eth2spec.test.helpers.attestations import get_valid_on_time_attestation
 from eth2spec.test.helpers.block import get_state_and_beacon_parent_root_at_slot
 from eth2spec.test.helpers.keys import privkeys
 from eth2spec.utils import bls
@@ -22,19 +21,21 @@ def build_shard_block(spec,
                       shard,
                       slot=None,
                       body=None,
+                      shard_parent_state=None,
                       signed=False):
-    shard_state = beacon_state.shard_states[shard]
+    if shard_parent_state is None:
+        shard_parent_state = beacon_state.shard_states[shard]
+
     if slot is None:
-        slot = shard_state.slot + 1
+        slot = shard_parent_state.slot + 1
 
     if body is None:
         body = b'\x56' * 128
 
-    proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard)
     beacon_state, beacon_parent_root = get_state_and_beacon_parent_root_at_slot(spec, beacon_state, slot)
-
+    proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard)
     block = spec.ShardBlock(
-        shard_parent_root=shard_state.latest_block_root,
+        shard_parent_root=shard_parent_state.latest_block_root,
         beacon_parent_root=beacon_parent_root,
         slot=slot,
         shard=shard,
@@ -51,14 +52,20 @@ def build_shard_block(spec,
     return signed_block
 
 
-def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot):
+def get_shard_transitions(spec, parent_beacon_state, shard_blocks):
     shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS
+    on_time_slot = parent_beacon_state.slot + 1
     for shard, blocks in shard_blocks.items():
-        offset_slots = spec.compute_offset_slots(spec.get_latest_slot_for_shard(state, shard), on_time_slot)
+        shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks)
+        offset_slots = spec.compute_offset_slots(
+            spec.get_latest_slot_for_shard(parent_beacon_state, shard),
+            on_time_slot,
+        )
         len_offset_slots = len(offset_slots)
         # TODO this is actually unsafe for long offset_slots
-        assert len_offset_slots == on_time_slot - state.shard_states[shard].slot - 1
+        assert len_offset_slots == on_time_slot - parent_beacon_state.shard_states[shard].slot - 1
-        shard_transition = spec.get_shard_transition(state, shard, blocks)
+        shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks)
+
         if len(blocks) > 0:
             shard_block_root = blocks[-1].message.hash_tree_root()
             assert shard_transition.shard_states[len_offset_slots - 1].latest_block_root == shard_block_root
@@ -68,14 +75,11 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot):
     return shard_transitions
 
 
-def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition):
+def get_committee_index_of_shard(spec, state, slot, shard):  # Optional[CommitteeIndex]
-    attestation = get_valid_on_time_attestation(
+    active_shard_count = spec.get_active_shard_count(state)
-        spec,
+    committee_count = spec.get_committee_count_at_slot(state, slot)
-        state,
+    start_shard = spec.get_start_shard(state, slot)
-        index=index,
+    for committee_index in range(committee_count):
-        shard_transition=shard_transition,
+        if (start_shard + committee_index) % active_shard_count == shard:
-        signed=True,
+            return committee_index
-    )
+    return None
-    if shard_transition is not None:
-        assert attestation.data.shard_transition_root == shard_transition.hash_tree_root()
-    return attestation

tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py

@@ -28,13 +28,10 @@ def run_shard_transitions_processing(spec, state, shard_transitions, attestation
     yield 'post', state
 
 
-def get_shard_transition_of_committee(spec, state, committee_index, slot=None, shard_blocks=None):
+def get_shard_transition_of_committee(spec, state, committee_index, shard_blocks=None):
     if shard_blocks is None:
         shard_blocks = []
 
-    if slot is None:
-        slot = state.slot
-
     shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot)
     shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks)
     return shard_transition

tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py

@@ -2,75 +2,70 @@ from eth2spec.test.context import (
     PHASE0,
     with_all_phases_except,
     spec_state_test,
-    always_bls,
 )
+from eth2spec.test.helpers.attestations import get_valid_on_time_attestation
 from eth2spec.test.helpers.shard_transitions import run_shard_transitions_processing
 from eth2spec.test.helpers.shard_block import (
-    build_attestation_with_shard_transition,
     build_shard_block,
-    build_shard_transitions_till_slot,
+    get_shard_transitions,
 )
-from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot
+from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot, next_slot
 
 
 def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True):
     state = transition_to_valid_shard_slot(spec, state)
-    init_slot = state.slot
     committee_index = spec.CommitteeIndex(0)
+    init_slot = state.slot
     shard_slot = state.slot + target_len_offset_slot - 1
     shard = spec.compute_shard_from_committee_index(state, committee_index, shard_slot)
     assert state.shard_states[shard].slot == init_slot - 1
 
     # Create SignedShardBlock
     body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE
-    shard_block = build_shard_block(spec, state, shard, body=body, signed=True)
+    shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True)
     shard_blocks = [shard_block]
 
     # Transition state latest shard slot
     transition_to(spec, state, shard_slot)
     # Create a shard_transitions that would be included at beacon block `state.slot + target_len_offset_slot`
-    shard_transitions = build_shard_transitions_till_slot(
+    shard_transitions = get_shard_transitions(
         spec,
         state,
         shard_blocks={shard: shard_blocks},
-        on_time_slot=init_slot + target_len_offset_slot,
     )
     shard_transition = shard_transitions[shard]
-    # Create an attestation that would be included at beacon block `state.slot + target_len_offset_slot`
+    attestation = get_valid_on_time_attestation(
-    attestation = build_attestation_with_shard_transition(
         spec,
         state,
         index=committee_index,
-        on_time_slot=init_slot + target_len_offset_slot,
         shard_transition=shard_transition,
+        signed=False,
     )
+    next_slot(spec, state)
     pre_gasprice = state.shard_states[shard].gasprice
 
     transition_to(spec, state, init_slot + target_len_offset_slot)
     pre_shard_state = state.shard_states[shard]
-
     yield from run_shard_transitions_processing(spec, state, shard_transitions, [attestation], valid=valid)
 
     if valid:
-        # After state transition,
-        assert state.slot == init_slot + target_len_offset_slot
         shard_state = state.shard_states[shard]
         assert shard_state != pre_shard_state
         assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1]
-
+        assert shard_state.latest_block_root == shard_block.message.hash_tree_root()
         if target_len_offset_slot == 1:
             assert shard_state.gasprice > pre_gasprice
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
-@always_bls
 def test_basic_crosslinks(spec, state):
+    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
     yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=1, valid=True)
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
-@always_bls
 def test_multiple_offset_slots(spec, state):
-    yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=3, valid=True)
+    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
+    yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=2, valid=True)

tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py

@@ -4,37 +4,40 @@ from eth2spec.test.context import (
     PHASE0,
     with_all_phases_except,
     spec_state_test,
-    always_bls,
 )
+from eth2spec.test.helpers.attestations import get_valid_on_time_attestation
 from eth2spec.test.helpers.block import build_empty_block
 from eth2spec.test.helpers.shard_block import (
-    build_attestation_with_shard_transition,
     build_shard_block,
-    build_shard_transitions_till_slot,
+    get_shard_transitions,
 )
-from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot
+from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to
 
 
-def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index, valid=True):
+def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard, valid=True):
-    shard_transitions = build_shard_transitions_till_slot(
+    transition_to(spec, state, state.slot + target_len_offset_slot)
-        spec, state, shard_blocks, on_time_slot=state.slot + target_len_offset_slot
+
-    )
+    body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE
+    shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True)
+    shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]}
+
+    shard_transitions = get_shard_transitions(spec, state, shard_blocks)
     attestations = [
-        build_attestation_with_shard_transition(
+        get_valid_on_time_attestation(
             spec,
             state,
-            on_time_slot=state.slot + target_len_offset_slot,
             index=committee_index,
             shard_transition=shard_transitions[shard],
+            signed=True,
         )
         for shard in shard_blocks.keys()
     ]
 
-    # Propose beacon block at slot `x + 1`
+    beacon_block = build_empty_block(spec, state, slot=state.slot + 1)
-    beacon_block = build_empty_block(spec, state, slot=state.slot + target_len_offset_slot)
     beacon_block.body.attestations = attestations
     beacon_block.body.shard_transitions = shard_transitions
 
+    pre_gasprice = state.shard_states[shard].gasprice
     pre_shard_states = state.shard_states.copy()
     yield 'pre', state.copy()
     yield 'block', beacon_block
@@ -52,17 +55,18 @@ def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_off
             assert post_shard_state == shard_transitions[shard].shard_states[
                 len(shard_transitions[shard].shard_states) - 1
             ]
-            assert beacon_block.slot == shard_transitions[shard].shard_states[0].slot + target_len_offset_slot
             assert post_shard_state.slot == state.slot - 1
             if len(shard_blocks[shard]) == 0:
                 # `latest_block_root` is the same
                 assert post_shard_state.latest_block_root == pre_shard_states[shard].latest_block_root
+            if target_len_offset_slot == 1 and len(shard_blocks) > 0:
+                assert post_shard_state.gasprice > pre_gasprice
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
-@always_bls
 def test_process_beacon_block_with_normal_shard_transition(spec, state):
+    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
     state = transition_to_valid_shard_slot(spec, state)
 
     target_len_offset_slot = 1
@@ -70,25 +74,13 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state):
     shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1)
     assert state.shard_states[shard].slot == state.slot - 1
 
-    pre_gasprice = state.shard_states[shard].gasprice
+    yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard)
-
-    # Create SignedShardBlock at slot `shard_state.slot + 1`
-    body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE
-    shard_block = build_shard_block(spec, state, shard, body=body, signed=True)
-    shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]}
-
-    yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index)
-
-    shard_state = state.shard_states[shard]
-
-    if target_len_offset_slot == 1 and len(shard_blocks) > 0:
-        assert shard_state.gasprice > pre_gasprice
 
 
 @with_all_phases_except([PHASE0])
 @spec_state_test
-@always_bls
 def test_process_beacon_block_with_empty_proposal_transition(spec, state):
+    # NOTE: this test is only for full crosslink (minimal config), not for mainnet
     state = transition_to_valid_shard_slot(spec, state)
 
     target_len_offset_slot = 1
@@ -96,12 +88,4 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state):
     shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1)
     assert state.shard_states[shard].slot == state.slot - 1
 
-    # No new shard block
+    yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard)
-    shard_blocks = {}
-
-    pre_gasprice = state.shard_states[shard].gasprice
-
-    yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index)
-
-    if target_len_offset_slot == 1 and len(shard_blocks) > 0:
-        assert state.shard_states[shard].gasprice > pre_gasprice