Merge branch 'dev' into vbuterin-patch-13

configs/constant_presets/minimal.yaml

@@ -129,3 +129,13 @@ DOMAIN_TRANSFER: 0x05000000
 DOMAIN_CUSTODY_BIT_CHALLENGE: 0x06000000
 DOMAIN_SHARD_PROPOSER: 0x80000000
 DOMAIN_SHARD_ATTESTER: 0x81000000
+
+
+# Phase 1
+# ---------------------------------------------------------------
+SHARD_SLOTS_PER_BEACON_SLOT: 2
+EPOCHS_PER_SHARD_PERIOD: 4
+# PHASE_1_FORK_EPOCH >= EPOCHS_PER_SHARD_PERIOD * 2
+PHASE_1_FORK_EPOCH: 8
+# PHASE_1_FORK_SLOT = PHASE_1_FORK_EPOCH * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH
+PHASE_1_FORK_SLOT: 128

scripts/build_spec.py

@@ -37,7 +37,7 @@ from eth2spec.utils.bls import (
 from eth2spec.utils.hash_function import hash
 '''
 PHASE1_IMPORTS = '''from typing import (
-    Any, Dict, Optional, Set, Sequence, MutableSequence, Tuple,
+    Any, Dict, Optional, Set, Sequence, MutableSequence, Tuple, Union,
 )
 
 from dataclasses import (

specs/core/1_shard-data-chains.md

@@ -14,6 +14,8 @@
         - [Misc](#misc)
         - [Initial values](#initial-values)
         - [Time parameters](#time-parameters)
+        - [State list lengths](#state-list-lengths)
+        - [Rewards and penalties](#rewards-and-penalties)
         - [Signature domain types](#signature-domain-types)
         - [TODO PLACEHOLDER](#todo-placeholder)
     - [Data structures](#data-structures)
@@ -22,20 +24,24 @@
         - [`ShardBlockSignatures`](#shardblocksignatures)
         - [`ShardBlockCore`](#shardblockcore)
         - [`ExtendedShardBlockCore`](#extendedshardblockcore)
+        - [`ShardState`](#shardstate)
+        - [`ShardReceiptDelta`](#shardreceiptdelta)
     - [Helper functions](#helper-functions)
-        - [`compute_epoch_of_shard_slot`](#compute_epoch_of_shard_slot)
         - [`compute_slot_of_shard_slot`](#compute_slot_of_shard_slot)
+        - [`compute_epoch_of_shard_slot`](#compute_epoch_of_shard_slot)
         - [`get_shard_period_start_epoch`](#get_shard_period_start_epoch)
         - [`get_period_committee`](#get_period_committee)
         - [`get_persistent_committee`](#get_persistent_committee)
         - [`get_shard_block_proposer_index`](#get_shard_block_proposer_index)
-        - [`get_shard_block_attester_committee`](#get_shard_block_attester_committee)
         - [`get_shard_header`](#get_shard_header)
         - [`pad`](#pad)
         - [`flatten_shard_header`](#flatten_shard_header)
         - [`compute_crosslink_data_root`](#compute_crosslink_data_root)
+        - [`get_default_shard_state`](#get_default_shard_state)
     - [Object validity](#object-validity)
-        - [Shard blocks](#shard-blocks)
+        - [Shard block validation: preliminary](#shard-block-validation-preliminary)
+        - [Shard state transition function helpers](#shard-state-transition-function-helpers)
+        - [Shard state transition function](#shard-state-transition-function)
         - [Beacon attestations](#beacon-attestations)
     - [Shard fork choice rule](#shard-fork-choice-rule)
 
@@ -59,10 +65,11 @@ We define the following Python custom types for type hinting and readability:
 
 | Name | Value |
 | - | - |
-| `SHARD_HEADER_SIZE` | `2**9` (= 512) |
-| `SHARD_BLOCK_SIZE_LIMIT` | `2**16` (= 65,536) |
 | `SHARD_SLOTS_PER_BEACON_SLOT` | `2**1` (= 2) |
 | `MAX_PERSISTENT_COMMITTEE_SIZE` | `2**7` (= 128) |
+| `SHARD_HEADER_SIZE` | `2**9` (= 512) |
+| `SHARD_BLOCK_SIZE_TARGET` | `2**14` (= 16,384) |
+| `SHARD_BLOCK_SIZE_LIMIT` | `2**16` (= 65,536) |
 
 ### Initial values
 
@@ -70,7 +77,6 @@ We define the following Python custom types for type hinting and readability:
 | - | - |
 | `PHASE_1_FORK_EPOCH` | **TBD** |
 | `PHASE_1_FORK_SLOT` | **TBD** |
-| `GENESIS_SHARD_SLOT` | 0 |
 
 ### Time parameters
 
@@ -79,6 +85,19 @@ We define the following Python custom types for type hinting and readability:
 | `CROSSLINK_LOOKBACK` | `2**0` (= 1) | epochs | 6.4 minutes |
 | `EPOCHS_PER_SHARD_PERIOD` | `2**8` (= 256) | epochs | ~27 hours |
 
+### State list lengths
+
+| Name | Value | Unit |
+| - | - | :-: |
+| `HISTORY_ACCUMULATOR_VECTOR` | `2**6` (= 64) | state tree maximum depth |
+
+### Rewards and penalties
+
+| Name | Value |
+| - | - |
+| `BASEFEE_ADJUSTMENT_FACTOR` | `2**3` (= 8) |
+| `REWARD_COEFFICIENT_BASE` | `2**20` ( = 1,048,576) |
+
 ### Signature domain types
 
 The following types are defined, mapping into `DomainType` (little endian):
@@ -148,6 +167,32 @@ class ExtendedShardBlockCore(Container):
     attester_bitfield: Bitvector[MAX_PERSISTENT_COMMITTEE_SIZE * 2]
 ```
 
+### `ShardState`
+
+```python
+class ShardState(Container):
+    history_accumulator: Vector[Hash, HISTORY_ACCUMULATOR_VECTOR]
+    earlier_committee_rewards: List[uint64, MAX_PERSISTENT_COMMITTEE_SIZE]
+    later_committee_rewards: List[uint64, MAX_PERSISTENT_COMMITTEE_SIZE]
+    earlier_committee_fees: List[Gwei, MAX_PERSISTENT_COMMITTEE_SIZE]
+    later_committee_fees: List[Gwei, MAX_PERSISTENT_COMMITTEE_SIZE]
+    basefee: Gwei
+    slot: ShardSlot
+    shard: Shard
+    most_recent_block_core: ShardBlockCore
+    receipt_root: Hash
+    total_bytes: uint64
+```
+
+### `ShardReceiptDelta`
+
+```python
+class ShardReceiptDelta(Container):
+    index: ValidatorIndex
+    reward_coefficient: uint64
+    block_fee: Gwei
+```
+
 ## Helper functions
 
 ### `compute_slot_of_shard_slot`
@@ -167,16 +212,14 @@ def compute_epoch_of_shard_slot(slot: ShardSlot) -> Epoch:
 ### `get_shard_period_start_epoch`
 
 ```python
-def get_shard_period_start_epoch(epoch: Epoch, lookback: Epoch=Epoch(0)) -> Epoch:
+def get_shard_period_start_epoch(epoch: Epoch, lookback: int=0) -> Epoch:
     return Epoch(epoch - (epoch % EPOCHS_PER_SHARD_PERIOD) - lookback * EPOCHS_PER_SHARD_PERIOD)
 ```
 
 ### `get_period_committee`
 
 ```python
-def get_period_committee(state: BeaconState,
+def get_period_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]:
-                         epoch: Epoch,
-                         shard: Shard) -> List[ValidatorIndex, MAX_PERSISTENT_COMMITTEE_SIZE]:
     """
     Return committee for a period. Used to construct persistent committees.
     """
@@ -201,8 +244,8 @@ def get_persistent_committee(state: BeaconState,
     """
     epoch = compute_epoch_of_shard_slot(slot)
 
-    earlier_committee = get_period_committee(state, get_shard_period_start_epoch(epoch, lookback=Epoch(2)), shard)
+    earlier_committee = get_period_committee(state, get_shard_period_start_epoch(epoch, lookback=2), shard)
-    later_committee = get_period_committee(state, get_shard_period_start_epoch(epoch, lookback=Epoch(1)), shard)
+    later_committee = get_period_committee(state, get_shard_period_start_epoch(epoch, lookback=1), shard)
 
     # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from
     # later committee; return a sorted list of the union of the two, deduplicated
@@ -250,9 +293,9 @@ def get_shard_header(block: ShardBlock) -> ShardBlockHeader:
             data_root=hash_tree_root(block.core.data),
             state_root=block.core.state_root,
             total_bytes=block.core.total_bytes,
-            attester_bitfield=block.core.attester_bitfield
+            attester_bitfield=block.core.attester_bitfield,
         ),
-        signatures=block.signatures
+        signatures=block.signatures,
     )
 ```
 
@@ -299,93 +342,228 @@ def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Hash:
     return hash_tree_root(BytesN[MAX_SIZE](pad(header + footer, MAX_SIZE)))
 ```
 
-## Object validity
+### `get_default_shard_state`
-
-### Shard blocks
-
-Let:
-
-- `beacon_blocks` be the `BeaconBlock` list such that `beacon_blocks[slot]` is the canonical `BeaconBlock` at slot `slot`
-- `beacon_state` be the canonical `BeaconState` after processing `beacon_blocks[-1]`
-- `shard` is the shard ID
-- `valid_shard_blocks` be the list of valid `ShardBlock`, recursively defined
-- `candidate` be a candidate `ShardBlock` for which validity is to be determined by running `is_valid_shard_block`
 
 ```python
-def is_valid_shard_block(beacon_state: BeaconState,
+def get_default_shard_state(beacon_state: BeaconState, shard: Shard) -> ShardState:
-                         beacon_blocks: Sequence[BeaconBlock],
+    earlier_committee = get_period_committee(
-                         shard: Shard,
+        beacon_state,
-                         valid_shard_blocks: Sequence[ShardBlock],
+        Epoch(PHASE_1_FORK_EPOCH - EPOCHS_PER_SHARD_PERIOD * 2),
-                         candidate: ShardBlock) -> bool:
+        shard,
-    # Check if block is already determined valid
+    )
-    for _, block in enumerate(valid_shard_blocks):
+    later_committee = get_period_committee(
-        if candidate == block:
+        beacon_state,
-            return True
+        Epoch(PHASE_1_FORK_EPOCH - EPOCHS_PER_SHARD_PERIOD),
+        shard,
+    )
+    return ShardState(
+        basefee=1,
+        shard=shard,
+        slot=PHASE_1_FORK_SLOT,
+        earlier_committee_rewards=[REWARD_COEFFICIENT_BASE for _ in range(len(earlier_committee))],
+        later_committee_rewards=[REWARD_COEFFICIENT_BASE for _ in range(len(later_committee))],
+        earlier_committee_fees=[Gwei(0) for _ in range(len(earlier_committee))],
+        later_committee_fees=[Gwei(0) for _ in range(len(later_committee))],
+    )
+```
 
+## Object validity
+
+### Shard block validation: preliminary
+
+Accept a shard block `block` only if all of the following are correct:
+
+* Either `block.core.parent_root == Hash()` or a block `parent` such that `hash_tree_root(parent.core) == block.core.parent_root` has already been accepted.
+* `block.core.beacon_chain_root == get_block_root(head_beacon_state, compute_epoch_of_shard_slot(parent.core.slot))` where `head_beacon_state` is the current beacon chain head state. Alternatively phrased, a beacon chain block `beacon_ref` such that `signing_root(beacon_ref) == block.core.beacon_chain_root` has already been accepted and is part of the canonical chain, and no block with slot `beacon_ref.slot < slot <= compute_start_slot_of_epoch(compute_epoch_of_shard_slot(parent.core.slot))` is part of the canonical chain.
+* Let `beacon_state` be the state where `beacon_ref.state_root == hash_tree_root(beacon_state)`. Let `prev_state` be the post-state of the `parent` if the `parent` exists, otherwise let it be `get_default_shard_state(beacon_state, shard)` (defined below). `block.core.state_root` must equal the `hash_tree_root` of the state after applying `shard_state_transition(prev_state, beacon_state, block)`.
+
+Note that these acceptance conditions depend on the canonical beacon chain; when the canonical beacon chain reorganizes, the eligibility of shard blocks should be re-evaluated.
+
+### Shard state transition function helpers
+
+```python
+def add_reward(state: ShardState, beacon_state: BeaconState, index: ValidatorIndex, delta: int) -> None:
+    epoch = compute_epoch_of_shard_slot(state.slot)
+    earlier_committee = get_period_committee(
+        beacon_state,
+        get_shard_period_start_epoch(epoch, lookback=2),
+        state.shard,
+    )
+    later_committee = get_period_committee(beacon_state, get_shard_period_start_epoch(epoch, lookback=1), state.shard)
+    if index in earlier_committee:
+        state.earlier_committee_rewards[earlier_committee.index(index)] += delta
+    elif index in later_committee:
+        state.later_committee_rewards[later_committee.index(index)] += delta
+    else:
+        raise Exception("Should never be here")
+```
+
+```python
+def add_fee(state: ShardState, beacon_state: BeaconState, index: ValidatorIndex, delta: int) -> None:
+    epoch = compute_epoch_of_shard_slot(state.slot)
+    earlier_committee = get_period_committee(beacon_state, get_shard_period_start_epoch(epoch, lookback=2), state.shard)
+    later_committee = get_period_committee(beacon_state, get_shard_period_start_epoch(epoch, lookback=1), state.shard)
+    if index in earlier_committee:
+        state.earlier_committee_fees[earlier_committee.index(index)] += delta
+    elif index in later_committee:
+        state.later_committee_fees[later_committee.index(index)] += delta
+    else:
+        raise Exception("Should never be here")
+```
+
+### Shard state transition function
+
+```python
+def shard_state_transition(state: ShardState,
+                           beacon_state: BeaconState,
+                           block: ShardBlock,
+                           validate_state_root: bool=False) -> None:
+    assert block.core.slot > state.slot
+    for slot in range(state.slot, block.core.slot):
+        shard_slot_transition(state, beacon_state)
+    shard_block_transition(state, beacon_state, block, validate_state_root=validate_state_root)
+```
+
+```python
+def shard_slot_transition(state: ShardState, beacon_state: BeaconState) -> None:
+    # Correct saved state root
+    if state.most_recent_block_core.state_root == Hash():
+        state.most_recent_block_core.state_root = hash_tree_root(state)
+
+    # Save states in history accumulator
+    depth = 0
+    h = hash_tree_root(state)
+    while state.slot % 2**depth == 0 and depth < HISTORY_ACCUMULATOR_VECTOR:
+        state.history_accumulator[depth] = h
+        depth += 1
+
+    # Period transitions
+    if (state.slot + 1) % (SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD) == 0:
+        epoch = compute_epoch_of_shard_slot(state.slot)
+        earlier_committee = get_period_committee(
+            beacon_state,
+            get_shard_period_start_epoch(epoch, lookback=2),
+            state.shard,
+        )
+        later_committee = get_period_committee(
+            beacon_state,
+            get_shard_period_start_epoch(epoch, lookback=1),
+            state.shard,
+        )
+        state.receipt_root = hash_tree_root(List[ShardReceiptDelta, PLACEHOLDER]([
+            ShardReceiptDelta(
+                index=validator_index,
+                reward_coefficient=state.earlier_committee_rewards[i],
+                block_fee=state.earlier_committee_fees[i],
+            )
+            for i, validator_index in enumerate(earlier_committee)
+        ]))
+        state.earlier_committee_rewards = state.later_committee_rewards
+        state.earlier_committee_fees = state.later_committee_fees
+        state.later_committee_rewards = [REWARD_COEFFICIENT_BASE for _ in range(len(later_committee))],
+        state.later_committee_fees = [Gwei(0) for _ in range(len(later_committee))],
+    else:
+        state.receipt_root = Hash()
+    state.slot += ShardSlot(1)
+```
+
+```python
+def shard_block_transition(state: ShardState,
+                           beacon_state: BeaconState,
+                           block: ShardBlock,
+                           validate_state_root: bool) -> None:
     # Check slot number
-    assert compute_slot_of_shard_slot(candidate.core.slot) >= PHASE_1_FORK_SLOT
+    assert block.core.slot == state.slot
-
-    # Check beacon block
-    beacon_block_slot = compute_start_slot_of_epoch(compute_epoch_of_shard_slot(candidate.core.slot))
-    beacon_block = beacon_blocks[beacon_block_slot]
-    assert candidate.core.beacon_block_root == signing_root(beacon_block)
-    assert beacon_block.slot <= candidate.core.slot
-
-    # Check state root
-    assert candidate.core.state_root == Hash()  # [to be removed in phase 2]
 
     # Check parent block
-    if candidate.core.parent_root != Hash():
+    if block.core.parent_root != Hash():
-        parent_block = next(
+        assert block.core.parent_root == hash_tree_root(state.most_recent_block_core)
-            (block for block in valid_shard_blocks if hash_tree_root(block.core) == candidate.core.parent_root),
-            None
-        )
-        assert parent_block is not None
-        assert parent_block.core.slot < candidate.core.slot
-        parent_beacon_block_slot = compute_start_slot_of_epoch(compute_epoch_of_shard_slot(parent_block.core.slot))
-        assert signing_root(beacon_blocks[parent_beacon_block_slot]) == parent_block.core.beacon_chain_root
 
+    # Calculate base reward
+    total_balance = get_total_active_balance(beacon_state)
+    base_reward = (
+        REWARD_COEFFICIENT_BASE * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH
+    )
     # Check attestations
-    attester_committee = get_persistent_committee(beacon_state, shard, block.core.slot)
+    attester_committee = get_persistent_committee(beacon_state, state.shard, block.core.slot)
     pubkeys = []
-    for i, index in enumerate(attester_committee):
+    attestations = 0
+
+    for i, validator_index in enumerate(attester_committee):
         if block.core.attester_bitfield[i]:
-            pubkeys.append(beacon_state.validators[index].pubkey)
+            pubkeys.append(beacon_state.validators[validator_index].pubkey)
-    for i in range(len(attester_committee), MAX_PERSISTENT_COMMITTEE_SIZE * 2):
+            add_reward(state, beacon_state, validator_index, base_reward)
-        assert block.attester_bitfield[i] is False
+            attestations += 1
+
+    for i in range(len(attester_committee), MAX_PERSISTENT_COMMITTEE_SIZE):
+        assert block.core.attester_bitfield[i] is False or block.core.attester_bitfield[i] == 0  # TODO: FIX Bitvector
+
     assert bls_verify(
         pubkey=bls_aggregate_pubkeys(pubkeys),
-        message_hash=candidate.core.parent_root,
+        message_hash=block.core.parent_root,
-        signature=candidate.signatures.attestation_signature,
+        signature=block.signatures.attestation_signature,
-        domain=get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(candidate.core.slot))
+        domain=get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(block.core.slot))
     )
 
     # Check proposer
-    proposer_index = get_shard_block_proposer_index(beacon_state, shard, candidate.core.slot)
+    proposer_index = get_shard_block_proposer_index(beacon_state, state.shard, block.core.slot)
     assert proposer_index is not None
+    add_reward(state, beacon_state, proposer_index, attestations * base_reward // PROPOSER_REWARD_QUOTIENT)
     assert bls_verify(
         pubkey=beacon_state.validators[proposer_index].pubkey,
-        message_hash=hash_tree_root(candidate.core),
+        message_hash=hash_tree_root(block.core),
-        signature=candidate.signatures.proposer_signature,
+        signature=block.signatures.proposer_signature,
-        domain=get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(candidate.core.slot)),
+        domain=get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.core.slot)),
     )
 
-    return True
+    # Process and update block data fees
+    add_fee(state, beacon_state, proposer_index, state.basefee * len(block.core.data) // SHARD_BLOCK_SIZE_LIMIT)
+    QUOTIENT = SHARD_BLOCK_SIZE_LIMIT * BASEFEE_ADJUSTMENT_FACTOR
+    if len(block.core.data) > SHARD_BLOCK_SIZE_TARGET:
+        state.basefee += Gwei(max(1, state.basefee * (len(block.core.data) - SHARD_BLOCK_SIZE_TARGET) // QUOTIENT))
+    elif len(block.core.data) < SHARD_BLOCK_SIZE_TARGET:
+        state.basefee -= Gwei(max(1, state.basefee * (len(block.core.data) - SHARD_BLOCK_SIZE_TARGET) // QUOTIENT))
+    state.basefee = Gwei(max(
+        1,
+        min(
+            EFFECTIVE_BALANCE_INCREMENT // EPOCHS_PER_SHARD_PERIOD // SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH,
+            state.basefee,
+        )
+    ))
+
+    # Check total bytes
+    state.total_bytes += len(block.core.data)
+    assert block.core.total_bytes == state.total_bytes
+
+    # Update in-state block header
+    state.most_recent_block_core = ShardBlockCore(
+        slot=block.core.slot,
+        beacon_chain_root=block.core.beacon_chain_root,
+        parent_root=block.core.parent_root,
+        data_root=hash_tree_root(block.core.data),
+        state_root=Hash(),
+        total_bytes=block.core.total_bytes,
+        attester_bitfield=block.core.attester_bitfield,
+    )
+
+    # Check state root
+    if validate_state_root:
+        assert block.core.state_root == hash_tree_root(state)
 ```
 
 ### Beacon attestations
 
 Let:
 
-- `shard` be a valid `Shard`
+- `pre_state` is the `ShardState` before processing any blocks
-- `shard_blocks` be the `ShardBlock` list such that `shard_blocks[slot]` is the canonical `ShardBlock` for shard `shard` at slot `slot`
+- `shard_blocks_or_state_roots` be the `Union[ShardBlock, Hash]` list such that `shard_blocks[slot]` is the canonical `ShardBlock` for shard `pre_state.shard` at slot `slot` if a block exists, or the post-state-root of processing state up to and including that slot if a block does not exist.
 - `beacon_state` be the canonical `BeaconState`
 - `valid_attestations` be the set of valid `Attestation` objects, recursively defined
 - `candidate` be a candidate `Attestation` which is valid under Phase 0 rules, and for which validity is to be determined under Phase 1 rules by running `is_valid_beacon_attestation`
 
 ```python
-def is_valid_beacon_attestation(shard: Shard,
+def is_valid_beacon_attestation(pre_state: ShardState,
-                                shard_blocks: Sequence[ShardBlock],
+                                shard_blocks_or_state_roots: Sequence[Union[ShardBlock, Hash]],
                                 beacon_state: BeaconState,
                                 valid_attestations: Set[Attestation],
                                 candidate: Attestation) -> bool:
@@ -407,12 +585,22 @@ def is_valid_beacon_attestation(shard: Shard,
         assert candidate.data.previous_attestation.epoch < compute_epoch_of_slot(candidate.data.slot)
 
     # Check crosslink data root
-    start_epoch = beacon_state.crosslinks[shard].epoch
+    start_epoch = beacon_state.crosslinks[pre_state.shard].epoch
     end_epoch = min(compute_epoch_of_slot(candidate.data.slot) - CROSSLINK_LOOKBACK,
                     start_epoch + MAX_EPOCHS_PER_CROSSLINK)
     blocks = []
     for slot in range(start_epoch * SLOTS_PER_EPOCH, end_epoch * SLOTS_PER_EPOCH):
-        blocks.append(shard_blocks[slot])
+        if isinstance(shard_blocks_or_state_roots[slot], ShardBlock):
+            blocks.append(shard_blocks_or_state_roots[slot])
+        else:
+            blocks.append(ShardBlock(
+                core=ExtendedShardBlockCore(
+                    slot=slot,
+                    state_root=shard_blocks_or_state_roots[slot],
+                    total_bytes=pre_state.total_bytes,
+                ),
+                signatures=ShardBlockSignatures(),
+            ))
     assert candidate.data.crosslink.data_root == compute_crosslink_data_root(blocks)
 
     return True

test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py

@@ -0,0 +1,37 @@
+from eth2spec.test.helpers.keys import privkeys
+from eth2spec.utils.bls import (
+    bls_aggregate_signatures,
+    bls_sign,
+)
+
+
+def sign_shard_attestation(spec, shard_state, beacon_state, block, participants):
+    signatures = []
+    message_hash = block.core.parent_root
+    block_epoch = spec.compute_epoch_of_shard_slot(block.core.slot)
+    for validator_index in participants:
+        privkey = privkeys[validator_index]
+        signatures.append(
+            get_attestation_signature(
+                spec,
+                shard_state,
+                beacon_state,
+                message_hash,
+                block_epoch,
+                privkey,
+            )
+        )
+
+    return bls_aggregate_signatures(signatures)
+
+
+def get_attestation_signature(spec, shard_state, beacon_state, message_hash, block_epoch, privkey):
+    return bls_sign(
+        message_hash=message_hash,
+        privkey=privkey,
+        domain=spec.get_domain(
+            state=beacon_state,
+            domain_type=spec.DOMAIN_SHARD_ATTESTER,
+            message_epoch=block_epoch,
+        )
+    )

test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py

@@ -7,6 +7,10 @@ from eth2spec.utils.ssz.ssz_impl import (
     signing_root,
 )
 
+from .attestations import (
+    sign_shard_attestation,
+)
+
 
 @only_with_bls()
 def sign_shard_block(spec, state, block, shard, proposer_index=None):
@@ -26,22 +30,52 @@ def sign_shard_block(spec, state, block, shard, proposer_index=None):
     )
 
 
-def build_empty_shard_block(spec, state, slot, shard, parent_root, signed=False):
+def build_empty_shard_block(spec,
+                            shard_state,
+                            beacon_state,
+                            slot,
+                            parent_root,
+                            signed=False,
+                            full_attestation=False):
     if slot is None:
-        slot = state.slot
+        slot = shard_state.slot
+
     block = spec.ShardBlock(
         core=spec.ExtendedShardBlockCore(
             slot=slot,
-            beacon_chain_root=state.block_roots[state.slot % spec.SLOTS_PER_HISTORICAL_ROOT],
+            beacon_chain_root=beacon_state.block_roots[beacon_state.slot % spec.SLOTS_PER_HISTORICAL_ROOT],
             parent_root=parent_root,
         ),
         signatures=spec.ShardBlockSignatures(
-            attestation_signature=b'\x12' * 96,
+            attestation_signature=b'\x00' * 96,
             proposer_signature=b'\x25' * 96,
         )
     )
 
+    # attestation
+    if full_attestation:
+        attester_committee = spec.get_persistent_committee(beacon_state, shard_state.shard, block.core.slot)
+        block.core.attester_bitfield = list(
+            (True,) * len(attester_committee) +
+            (False,) * (spec.MAX_PERSISTENT_COMMITTEE_SIZE * 2 - len(attester_committee))
+        )
+        block.signatures.attestation_signature = sign_shard_attestation(
+            spec,
+            shard_state,
+            beacon_state,
+            block,
+            participants=attester_committee,
+        )
+    else:
+        block.signatures.attestation_signature = sign_shard_attestation(
+            spec,
+            shard_state,
+            beacon_state,
+            block,
+            participants=(),
+        )
+
     if signed:
-        sign_shard_block(spec, state, block, shard)
+        sign_shard_block(spec, beacon_state, block, shard_state.shard)
 
     return block

test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_beacon_attestation.py

@@ -0,0 +1,48 @@
+from eth2spec.test.context import (
+    with_all_phases_except,
+    spec_state_test,
+    always_bls,
+)
+from eth2spec.test.helpers.phase1.shard_block import (
+    build_empty_shard_block,
+)
+from eth2spec.test.helpers.attestations import get_valid_attestation
+
+
+@with_all_phases_except(['phase0'])
+@always_bls
+@spec_state_test
+def test_process_empty_shard_block(spec, state):
+    beacon_state = state
+
+    shard_slot = spec.PHASE_1_FORK_SLOT
+    beacon_state.slot = spec.Slot(spec.PHASE_1_FORK_EPOCH * spec.SLOTS_PER_EPOCH)
+    shard_state = spec.get_default_shard_state(beacon_state, shard=spec.Shard(0))
+    shard_state.slot = shard_slot
+
+    block = build_empty_shard_block(
+        spec,
+        shard_state,
+        beacon_state,
+        slot=shard_slot + 1,
+        parent_root=spec.Hash(),
+        signed=True,
+        full_attestation=True,
+    )
+
+    yield 'pre', shard_state
+    yield 'beacon_state', beacon_state
+    yield 'block', block
+
+    beacon_attestation = get_valid_attestation(spec, beacon_state, signed=True)
+    yield 'beacon_attestation', beacon_attestation
+
+    is_valid_beacon_attestation = spec.is_valid_beacon_attestation(
+        pre_state=shard_state,
+        shard_blocks_or_state_roots=(block,),
+        beacon_state=beacon_state,
+        valid_attestations=set([beacon_attestation]),
+        candidate=beacon_attestation,
+    )
+    assert is_valid_beacon_attestation
+    yield 'is_valid_beacon_attestation', is_valid_beacon_attestation

test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py

@@ -11,16 +11,58 @@ from eth2spec.test.context import (
 @with_all_phases_except(['phase0'])
 @always_bls
 @spec_state_test
-def test_is_valid_shard_block(spec, state):
+def test_process_empty_shard_block(spec, state):
+    beacon_state = state
+
+    shard_slot = spec.PHASE_1_FORK_SLOT
+    beacon_state.slot = spec.Slot(spec.PHASE_1_FORK_EPOCH * spec.SLOTS_PER_EPOCH)
+    shard_state = spec.get_default_shard_state(beacon_state, shard=spec.Shard(0))
+    shard_state.slot = shard_slot
+
     block = build_empty_shard_block(
         spec,
-        state,
+        shard_state,
-        slot=spec.Slot(spec.PERSISTENT_COMMITTEE_PERIOD * 100),
+        beacon_state,
-        shard=spec.Shard(1),
+        slot=shard_slot + 1,
         parent_root=spec.Hash(),
         signed=True,
+        full_attestation=False,
     )
 
-    # TODO: test `is_valid_shard_block`
+    yield 'pre', shard_state
+    yield 'beacon_state', beacon_state
+    yield 'block', block
 
-    yield 'blocks', (block,)
+    spec.shard_state_transition(shard_state, beacon_state, block)
+
+    yield 'post', shard_state
+
+
+@with_all_phases_except(['phase0'])
+@always_bls
+@spec_state_test
+def test_process_full_attestation_shard_block(spec, state):
+    beacon_state = state
+
+    shard_slot = spec.PHASE_1_FORK_SLOT
+    beacon_state.slot = spec.Slot(spec.PHASE_1_FORK_EPOCH * spec.SLOTS_PER_EPOCH)
+    shard_state = spec.get_default_shard_state(beacon_state, shard=spec.Shard(0))
+    shard_state.slot = shard_slot
+
+    block = build_empty_shard_block(
+        spec,
+        shard_state,
+        beacon_state,
+        slot=shard_slot + 1,
+        parent_root=spec.Hash(),
+        signed=True,
+        full_attestation=True,
+    )
+
+    yield 'pre', shard_state
+    yield 'beacon_state', beacon_state
+    yield 'block', block
+
+    spec.shard_state_transition(shard_state, beacon_state, block)
+
+    yield 'post', shard_state