[squashed] shard chain updates wip

PR feedback from Danny and some refactor

1. Add stub `PHASE_1_GENESIS_SLOT`
2. Rename `get_updated_gasprice`  to `compute_updated_gasprice`
3. Rename `compute_shard_data_roots` to `compute_shard_body_roots`

Apply shard transition for the skipped slots

Refactor `shard_state_transition`

Get `beacon_parent_root` from offset slot

Add more test

Add `verify_shard_block_message`

Add `> 0`

Keep `beacon_parent_block` unchanged in `is_valid_fraud_proof`

Remove some lines

Fix type

Refactor + simplify skipped slot processing

configs/mainnet.yaml

@@ -161,6 +161,8 @@ DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000
 # Phase 1: Upgrade from Phase 0
 # ---------------------------------------------------------------
 PHASE_1_FORK_VERSION: 0x01000000
+# [STUB]
+PHASE_1_GENESIS_SLOT: 32
 INITIAL_ACTIVE_SHARDS: 64
 
 # Phase 1: General

configs/minimal.yaml

@@ -162,6 +162,8 @@ DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000
 # ---------------------------------------------------------------
 # [customized] for testnet distinction
 PHASE_1_FORK_VERSION: 0x01000001
+# [customized] for testing
+PHASE_1_GENESIS_SLOT: 8
 # [customized] reduced for testing
 INITIAL_ACTIVE_SHARDS: 4
 

specs/phase1/beacon-chain.md

@@ -39,6 +39,7 @@
     - [`committee_to_compact_committee`](#committee_to_compact_committee)
     - [`compute_shard_from_committee_index`](#compute_shard_from_committee_index)
     - [`compute_offset_slots`](#compute_offset_slots)
+    - [`compute_updated_gasprice`](#compute_updated_gasprice)
   - [Beacon state accessors](#beacon-state-accessors)
     - [`get_active_shard_count`](#get_active_shard_count)
     - [`get_online_validator_indices`](#get_online_validator_indices)
@@ -46,7 +47,6 @@
     - [`get_light_client_committee`](#get_light_client_committee)
     - [`get_shard_proposer_index`](#get_shard_proposer_index)
     - [`get_indexed_attestation`](#get_indexed_attestation)
-    - [`get_updated_gasprice`](#get_updated_gasprice)
     - [`get_start_shard`](#get_start_shard)
     - [`get_shard`](#get_shard)
     - [`get_latest_slot_for_shard`](#get_latest_slot_for_shard)
@@ -439,6 +439,20 @@ def compute_offset_slots(start_slot: Slot, end_slot: Slot) -> Sequence[Slot]:
     return [Slot(start_slot + x) for x in SHARD_BLOCK_OFFSETS if start_slot + x < end_slot]
 ```
 
+#### `compute_updated_gasprice`
+
+```python
+def compute_updated_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei:
+    if length > TARGET_SHARD_BLOCK_SIZE:
+        delta = (prev_gasprice * (length - TARGET_SHARD_BLOCK_SIZE)
+                 // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT)
+        return min(prev_gasprice + delta, MAX_GASPRICE)
+    else:
+        delta = (prev_gasprice * (TARGET_SHARD_BLOCK_SIZE - length)
+                 // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT)
+        return max(prev_gasprice, MIN_GASPRICE + delta) - delta
+```
+
 ### Beacon state accessors
 
 #### `get_active_shard_count`
@@ -512,20 +526,6 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation)
     )
 ```
 
-#### `get_updated_gasprice`
-
-```python
-def get_updated_gasprice(prev_gasprice: Gwei, length: uint8) -> Gwei:
-    if length > TARGET_SHARD_BLOCK_SIZE:
-        delta = (prev_gasprice * (length - TARGET_SHARD_BLOCK_SIZE) 
-                 // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT)
-        return min(prev_gasprice + delta, MAX_GASPRICE)
-    else:
-        delta = (prev_gasprice * (TARGET_SHARD_BLOCK_SIZE - length)
-                 // TARGET_SHARD_BLOCK_SIZE // GASPRICE_ADJUSTMENT_COEFFICIENT)
-        return max(prev_gasprice, MIN_GASPRICE + delta) - delta
-```
-
 #### `get_start_shard`
 
 ```python
@@ -617,7 +617,6 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe
         return bls.AggregateVerify(zip(all_pubkeys, all_signing_roots), signature=attestation.signature)
 ```
 
-
 ### Block processing
 
 ```python
@@ -630,7 +629,6 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None:
     process_operations(state, block.body)
 ```
 
-
 #### Operations
 
 ```python
@@ -714,34 +712,31 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr
     assert transition.start_slot == offset_slots[0]
 
     headers = []
-    header = ShardBlockHeader()
     proposers = []
     prev_gasprice = state.shard_states[shard].gasprice
     shard_parent_root = state.shard_states[shard].latest_block_root
-    beacon_parent_root = get_block_root_at_slot(state, get_previous_slot(state.slot))
     for i in range(len(offset_slots)):
         shard_block_length = transition.shard_block_lengths[i]
-        is_empty_proposal = (shard_block_length == 0)
+        is_empty_proposal = shard_block_length == 0
         shard_state = transition.shard_states[i]
-        proposal_index = get_shard_proposer_index(state, offset_slots[i], shard)
-        # Reconstruct shard headers
-        header = ShardBlockHeader(
-            shard_parent_root=shard_parent_root,
-            beacon_parent_root=beacon_parent_root,
-            proposer_index=proposal_index,
-            slot=offset_slots[i],
-            body_root=transition.shard_data_roots[i]
-        )
-        shard_parent_root = hash_tree_root(header)
-
         if not is_empty_proposal:
-            # Only add non-empty signature
+            proposal_index = get_shard_proposer_index(state, offset_slots[i], shard)
+            # Reconstruct shard headers
+            header = ShardBlockHeader(
+                shard_parent_root=shard_parent_root,
+                beacon_parent_root=get_block_root_at_slot(state, offset_slots[i]),
+                proposer_index=proposal_index,
+                slot=offset_slots[i],
+                body_root=transition.shard_data_roots[i]
+            )
+            shard_parent_root = hash_tree_root(header)
             headers.append(header)
             proposers.append(proposal_index)
-            # Verify correct calculation of gas prices and slots
+
-            assert shard_state.gasprice == get_updated_gasprice(prev_gasprice, shard_block_length)
+        # Verify correct calculation of gas prices and slots
-            assert shard_state.slot == offset_slots[i]
+        assert shard_state.gasprice == compute_updated_gasprice(prev_gasprice, shard_block_length)
-            prev_gasprice = shard_state.gasprice
+        assert shard_state.slot == offset_slots[i]
+        prev_gasprice = shard_state.gasprice
 
     pubkeys = [state.validators[proposer].pubkey for proposer in proposers]
     signing_roots = [
@@ -753,7 +748,6 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr
 
     # Save updated state
     state.shard_states[shard] = transition.shard_states[len(transition.shard_states) - 1]
-    assert state.slot > 0
     state.shard_states[shard].slot = state.slot - 1
 ```
 

specs/phase1/fraud-proofs.md

@@ -36,17 +36,48 @@ This document describes the shard transition function and fraud proofs as part o
 
 ```python
 def shard_state_transition(beacon_state: BeaconState,
-                           shard: Shard,
-                           slot: Slot,
                            shard_state: ShardState,
-                           beacon_parent_root: Root,
                            signed_block: SignedShardBlock) -> None:
     # Update shard state
-    shard_state.data = hash(
+    prev_gasprice = shard_state.gasprice
-        hash_tree_root(shard_state) + hash_tree_root(beacon_parent_root) + hash_tree_root(signed_block.message.body)
+    if len(signed_block.message.body) == 0:
+        latest_block_root = shard_state.latest_block_root
+    else:
+        latest_block_root = hash_tree_root(signed_block.message)
+
+    shard_state.data = compute_shard_transition_data(
+        beacon_state,
+        shard_state,
+        signed_block.message.beacon_parent_root,
+        signed_block.message.body,
     )
-    shard_state.slot = slot
+    shard_state.gasprice = compute_updated_gasprice(prev_gasprice, len(signed_block.message.body))
-    shard_state.latest_block_root = hash_tree_root(signed_block.message)
+    shard_state.slot = signed_block.message.slot
+    shard_state.latest_block_root = latest_block_root
+```
+
+```python
+def compute_shard_transition_data(beacon_state: BeaconState,
+                                  shard_state: ShardState,
+                                  beacon_parent_root: Root,
+                                  shard_body_root: Root) -> Bytes32:
+    return hash(
+        hash_tree_root(shard_state) + beacon_parent_root + shard_body_root
+    )
+```
+
+```python
+def verify_shard_block_message(beacon_state: BeaconState,
+                               shard_state: ShardState,
+                               block: ShardBlock,
+                               slot: Slot,
+                               shard: Shard) -> bool:
+    assert block.shard_parent_root == shard_state.latest_block_root
+    assert block.beacon_parent_root == get_block_root_at_slot(beacon_state, slot)
+    assert block.slot == slot
+    assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard)
+    assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE
+    return True
 ```
 
 ```python
@@ -67,20 +98,20 @@ TODO. The intent is to have a single universal fraud proof type, which contains
 3. The `transition: ShardTransition` itself
 4. The full body of the shard block `shard_block`
 5. A Merkle proof to the `shard_states` in the parent block the attestation is referencing
+6. The `subkey` to generate the custody bit
 
 Call the following function to verify the proof:
 
 ```python
-def verify_fraud_proof(beacon_state: BeaconState,
+def is_valid_fraud_proof(beacon_state: BeaconState,
-                       attestation: Attestation,
+                         attestation: Attestation,
-                       offset_index: uint64,
+                         offset_index: uint64,
-                       transition: ShardTransition,
+                         transition: ShardTransition,
-                       signed_block: SignedShardBlock,
+                         signed_block: SignedShardBlock,
-                       subkey: BLSPubkey,
+                         subkey: BLSPubkey,
-                       beacon_parent_block: BeaconBlock) -> bool:
+                         beacon_parent_block: BeaconBlock) -> bool:
     # 1. Check if `custody_bits[offset_index][j] != generate_custody_bit(subkey, block_contents)` for any `j`.
     shard = get_shard(beacon_state, attestation)
-    slot = attestation.data.slot
     custody_bits = attestation.custody_bits_blocks
     for j in range(custody_bits[offset_index]):
         if custody_bits[offset_index][j] != generate_custody_bit(subkey, signed_block):
@@ -89,19 +120,12 @@ def verify_fraud_proof(beacon_state: BeaconState,
     # 2. Check if the shard state transition result is wrong between
     # `transition.shard_states[offset_index - 1]` to `transition.shard_states[offset_index]`.
     if offset_index == 0:
-        shard_state = beacon_parent_block.shard_transitions[shard].shard_states[-1]
+        shard_state = beacon_parent_block.shard_transitions[shard].shard_states[-1].copy()
     else:
         shard_state = transition.shard_states[offset_index - 1].copy()  # Not doing the actual state updates here.
 
-    shard_state_transition(
+    shard_state_transition(beacon_state, shard_state, signed_block)
-        beacon_state=beacon_state,
+    if shard_state.data != transition.shard_states[offset_index].data:
-        shard=shard,
-        slot=slot,
-        shard_state=shard_state,
-        beacon_parent_root=hash_tree_root(beacon_parent_block),
-        signed_block=signed_block,
-    )
-    if shard_state.latest_block_root != transition.shard_states[offset_index].data:
         return True
 
     return False
@@ -126,26 +150,7 @@ def get_winning_proposal(beacon_state: BeaconState, proposals: Sequence[SignedSh
 ```
 
 ```python
-def get_empty_body_block(shard_parent_root: Root,
+def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[Root]:
-                         beacon_parent_root: Root,
-                         slot: Slot,
-                         proposer_index: ValidatorIndex) -> ShardBlock:
-    return ShardBlock(
-        shard_parent_root=shard_parent_root,
-        beacon_parent_root=beacon_parent_root,
-        slot=slot,
-        proposer_index=proposer_index,
-    )
-```
-
-```python
-def is_empty_body(proposal: ShardBlock) -> bool:
-    # TODO
-    return len(proposal.body) == 0
-```
-
-```python
-def compute_shard_data_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[Root]:
     return [hash_tree_root(proposal.message.body) for proposal in proposals]
 ```
 
@@ -155,28 +160,23 @@ def get_proposal_choices_at_slot(beacon_state: BeaconState,
                                  slot: Slot,
                                  shard: Shard,
                                  shard_blocks: Sequence[SignedShardBlock],
-                                 validate_result: bool=True) -> Sequence[SignedShardBlock]:
+                                 validate_signature: bool=True) -> Sequence[SignedShardBlock]:
+    """
+    Return the valid shard blocks at the given ``slot``.
+    Note that this function doesn't change the state.
+    """
     choices = []
-    beacon_parent_root = get_block_root_at_slot(beacon_state, get_previous_slot(beacon_state.slot))
-    proposer_index = get_shard_proposer_index(beacon_state, slot, shard)
     shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot]
     for block in shard_blocks_at_slot:
         temp_shard_state = shard_state.copy()  # Not doing the actual state updates here.
         # Try to apply state transition to temp_shard_state.
         try:
-            # Verify the proposer_index and signature
+            # Verify block message and signature
-            assert block.message.proposer_index == proposer_index
+            assert verify_shard_block_message(beacon_state, temp_shard_state, block.message, slot, shard)
-            if validate_result:
+            if validate_signature:
                 assert verify_shard_block_signature(beacon_state, block)
 
-            shard_state_transition(
+            shard_state_transition(beacon_state, temp_shard_state, block)
-                beacon_state=beacon_state,
-                shard=shard,
-                slot=slot,
-                shard_state=temp_shard_state,
-                beacon_parent_root=beacon_parent_root,
-                signed_block=block,
-            )
         except Exception:
             pass  # TODO: throw error in the test helper
         else:
@@ -189,11 +189,12 @@ def get_proposal_at_slot(beacon_state: BeaconState,
                          shard_state: ShardState,
                          slot: Shard,
                          shard: Shard,
-                         shard_parent_root: Root,
                          shard_blocks: Sequence[SignedShardBlock],
-                         validate_result: bool=True) -> Tuple[SignedShardBlock, ShardState, Root]:
+                         validate_signature: bool=True) -> Tuple[SignedShardBlock, ShardState]:
-    beacon_parent_root = get_block_root_at_slot(beacon_state, get_previous_slot(beacon_state.slot))
+    """
-    proposer_index = get_shard_proposer_index(beacon_state, slot, shard)
+    Return ``proposal``, ``shard_state`` of the given ``slot``.
+    Note that this function doesn't change the state.
+    """
     shard_state = shard_state.copy()  # Don't update the given shard_state
     choices = get_proposal_choices_at_slot(
         beacon_state=beacon_state,
@@ -201,35 +202,20 @@ def get_proposal_at_slot(beacon_state: BeaconState,
         slot=slot,
         shard=shard,
         shard_blocks=shard_blocks,
-        validate_result=validate_result,
+        validate_signature=validate_signature,
     )
     if len(choices) == 0:
-        block_header = get_empty_body_block(
+        block = ShardBlock(slot=slot)
-            shard_parent_root=shard_parent_root,
+        proposal = SignedShardBlock(message=block)
-            beacon_parent_root=beacon_parent_root,
-            slot=slot,
-            proposer_index=proposer_index,
-        )
-        proposal = SignedShardBlock(message=block_header)
     elif len(choices) == 1:
         proposal = choices[0]
     else:
         proposal = get_winning_proposal(beacon_state, choices)
 
-    shard_parent_root = hash_tree_root(proposal.message)
+    # Apply state transition
+    shard_state_transition(beacon_state, shard_state, proposal)
 
-    if not is_empty_body(proposal.message):
+    return proposal, shard_state
-        # Apply state transition to shard_state.
-        shard_state_transition(
-            beacon_state=beacon_state,
-            shard=shard,
-            slot=slot,
-            shard_state=shard_state,
-            beacon_parent_root=beacon_parent_root,
-            signed_block=proposal,
-        )
-
-    return proposal, shard_state, shard_parent_root
 ```
 
 ```python
@@ -237,26 +223,24 @@ def get_shard_state_transition_result(
     beacon_state: BeaconState,
     shard: Shard,
     shard_blocks: Sequence[SignedShardBlock],
-    validate_result: bool=True,
+    validate_signature: bool=True,
 ) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]:
     proposals = []
     shard_states = []
     shard_state = beacon_state.shard_states[shard].copy()
-    shard_parent_root = beacon_state.shard_states[shard].latest_block_root
     for slot in get_offset_slots(beacon_state, shard):
-        proposal, shard_state, shard_parent_root = get_proposal_at_slot(
+        proposal, shard_state = get_proposal_at_slot(
             beacon_state=beacon_state,
             shard_state=shard_state,
             slot=slot,
             shard=shard,
-            shard_parent_root=shard_parent_root,
             shard_blocks=shard_blocks,
-            validate_result=validate_result,
+            validate_signature=validate_signature,
         )
         shard_states.append(shard_state)
         proposals.append(proposal)
 
-    shard_data_roots = compute_shard_data_roots(proposals)
+    shard_data_roots = compute_shard_body_roots(proposals)
 
     return proposals, shard_states, shard_data_roots
 ```

specs/phase1/phase1-fork.md

@@ -7,7 +7,7 @@
   - [Introduction](#introduction)
   - [Configuration](#configuration)
   - [Fork to Phase 1](#fork-to-phase-1)
-    - [Fork trigger.](#fork-trigger)
+    - [Fork trigger](#fork-trigger)
     - [Upgrading the state](#upgrading-the-state)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -35,17 +35,18 @@ Warning: this configuration is not definitive.
 | Name | Value |
 | - | - |
 | `PHASE_1_FORK_VERSION` | `Version('0x01000000')` |
+| `PHASE_1_GENESIS_SLOT` | `2**5` **TBD** |
 | `INITIAL_ACTIVE_SHARDS` | `2**6` (= 64) |
 
 ## Fork to Phase 1
 
-### Fork trigger.
+### Fork trigger
 
-TBD. Social consensus, along with state conditions such as epoch boundary, finality, deposits, active validator count, etc. may be part of the decision process to trigger the fork.
+TBD. Social consensus, along with state conditions such as epoch boundary, finality, deposits, active validator count, etc. may be part of the decision process to trigger the fork. For now we assume the condition will be triggered at slot `PHASE_1_GENESIS_SLOT`, where `PHASE_1_GENESIS_SLOT % SLOTS_PER_EPOCH == 0`.
 
 ### Upgrading the state
 
-After `process_slots` of Phase 0 finishes, but before the first Phase 1 block is processed, an irregular state change is made to upgrade to Phase 1.
+After `process_slots` of Phase 0 finishes, if `state.slot == PHASE_1_GENESIS_SLOT`, an irregular state change is made to upgrade to Phase 1.
 
 ```python
 def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState:

tests/core/pyspec/eth2spec/test/helpers/shard_block.py

@@ -32,7 +32,7 @@ def build_shard_block(spec,
 
     block = spec.ShardBlock(
         shard_parent_root=shard_state.latest_block_root,
-        beacon_parent_root=spec.get_block_root_at_slot(beacon_state, spec.get_previous_slot(beacon_state.slot)),
+        beacon_parent_root=spec.get_block_root_at_slot(beacon_state, spec.get_previous_slot(slot)),
         slot=slot,
         proposer_index=proposer_index,
         body=body,

tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py

@@ -10,7 +10,7 @@ from eth2spec.test.helpers.crosslinks import (
     run_crosslinks_processing,
 )
 from eth2spec.test.helpers.shard_block import build_shard_block
-from eth2spec.test.helpers.state import next_epoch, next_slot
+from eth2spec.test.helpers.state import next_epoch, next_slot, next_slots
 
 
 @with_all_phases_except(['phase0'])
@@ -24,7 +24,8 @@ def test_basic_crosslinks(spec, state):
 
     committee_index = spec.CommitteeIndex(0)
     shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot)
-    shard_block = build_shard_block(spec, state, shard, body=b'\x12' * 10, slot=state.slot, signed=True)
+    body = b'1' * spec.MAX_SHARD_BLOCK_SIZE
+    shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True)
     shard_blocks = [shard_block]
 
     next_slot(spec, state)
@@ -43,10 +44,58 @@ def test_basic_crosslinks(spec, state):
     )
     attestations = [attestation]
 
+    pre_gasprice = state.shard_states[shard].gasprice
     offset_slots = spec.get_offset_slots(state, shard)
+    assert len(offset_slots) == 1
 
     yield from run_crosslinks_processing(spec, state, shard_transitions, attestations)
 
     shard_state = state.shard_states[shard]
     assert shard_state.slot == offset_slots[-1]
     assert shard_state.latest_block_root == shard_block.message.hash_tree_root()
+    assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1]
+    assert shard_state.gasprice > pre_gasprice
+
+
+@with_all_phases_except(['phase0'])
+@spec_state_test
+@always_bls
+def test_multiple_offset_slots(spec, state):
+    next_epoch(spec, state)
+    next_epoch(spec, state)
+    state = spec.upgrade_to_phase1(state)
+    next_slot(spec, state)
+
+    committee_index = spec.CommitteeIndex(0)
+    shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot)
+    body = b'1' * spec.MAX_SHARD_BLOCK_SIZE
+    shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True)
+    shard_blocks = [shard_block]
+
+    next_slots(spec, state, 3)
+
+    shard_transition = spec.get_shard_transition(state, shard, shard_blocks)
+    shard_transitions = [spec.ShardTransition()] * len(state.shard_states)
+    shard_transitions[shard] = shard_transition
+
+    attestation = get_valid_on_time_attestation(
+        spec,
+        state,
+        slot=state.slot,
+        index=committee_index,
+        shard_transition=shard_transition,
+        signed=True,
+    )
+    attestations = [attestation]
+
+    pre_gasprice = state.shard_states[shard].gasprice
+    offset_slots = spec.get_offset_slots(state, shard)
+    assert len(offset_slots) == 3
+
+    yield from run_crosslinks_processing(spec, state, shard_transitions, attestations)
+
+    shard_state = state.shard_states[shard]
+    assert shard_state.slot == offset_slots[-1]
+    assert shard_state.latest_block_root == shard_block.message.hash_tree_root()
+    assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1]
+    assert shard_state.gasprice > pre_gasprice