Merge pull request #1858 from ethereum/hwwhww/get_start_shard

`get_start_shard` proposal

configs/minimal.yaml

@@ -165,7 +165,7 @@ PHASE_1_FORK_VERSION: 0x01000001
 # [customized] for testing
 PHASE_1_GENESIS_SLOT: 8
 # [customized] reduced for testing
-INITIAL_ACTIVE_SHARDS: 4
+INITIAL_ACTIVE_SHARDS: 2
 
 
 # Phase 1: General

setup.py

@@ -140,7 +140,7 @@ SUNDRY_CONSTANTS_FUNCTIONS = '''
 def ceillog2(x: uint64) -> int:
     return (x - 1).bit_length()
 '''
-SUNDRY_FUNCTIONS = '''
+PHASE0_SUNDRY_FUNCTIONS = '''
 # Monkey patch hash cache
 _hash = hash
 hash_cache: Dict[bytes, Bytes32] = {}
@@ -220,6 +220,13 @@ get_attesting_indices = cache_this(
     _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)'''
 
 
+PHASE1_SUNDRY_FUNCTIONS = '''
+_get_start_shard = get_start_shard
+get_start_shard = cache_this(
+    lambda state, slot: (state.validators.hash_tree_root(), slot),
+    _get_start_shard, lru_size=SLOTS_PER_EPOCH * 3)'''
+
+
 def objects_to_spec(spec_object: SpecObject, imports: str, fork: str) -> str:
     """
     Given all the objects that constitute a spec, combine them into a single pyfile.
@@ -250,9 +257,11 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str) -> str:
             + '\n\n' + CONFIG_LOADER
             + '\n\n' + ssz_objects_instantiation_spec
             + '\n\n' + functions_spec
-            + '\n' + SUNDRY_FUNCTIONS
+            + '\n' + PHASE0_SUNDRY_FUNCTIONS
-            + '\n'
     )
+    if fork == 'phase1':
+        spec += '\n' + PHASE1_SUNDRY_FUNCTIONS
+    spec += '\n'
     return spec
 
 

specs/phase1/beacon-chain.md

@@ -48,6 +48,7 @@
     - [`get_light_client_committee`](#get_light_client_committee)
     - [`get_shard_proposer_index`](#get_shard_proposer_index)
     - [`get_indexed_attestation`](#get_indexed_attestation)
+    - [`get_committee_count_delta`](#get_committee_count_delta)
     - [`get_start_shard`](#get_start_shard)
     - [`get_shard`](#get_shard)
     - [`get_latest_slot_for_shard`](#get_latest_slot_for_shard)
@@ -73,6 +74,7 @@
       - [New Attester slashing processing](#new-attester-slashing-processing)
     - [Light client processing](#light-client-processing)
   - [Epoch transition](#epoch-transition)
+    - [Phase 1 final updates](#phase-1-final-updates)
     - [Custody game updates](#custody-game-updates)
     - [Online-tracking](#online-tracking)
     - [Light client committee updates](#light-client-committee-updates)
@@ -285,6 +287,7 @@ class BeaconState(Container):
     current_justified_checkpoint: Checkpoint
     finalized_checkpoint: Checkpoint
     # Phase 1
+    current_epoch_start_shard: Shard
     shard_states: List[ShardState, MAX_SHARDS]
     online_countdown: List[OnlineEpochs, VALIDATOR_REGISTRY_LIMIT]  # not a raw byte array, considered its large size.
     current_light_committee: CompactCommittee
@@ -550,18 +553,49 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation)
     )
 ```
 
+#### `get_committee_count_delta`
+
+```python
+def get_committee_count_delta(state: BeaconState, start_slot: Slot, stop_slot: Slot) -> uint64:
+    """
+    Return the sum of committee counts in range ``[start_slot, stop_slot)``.
+    """
+    return sum(get_committee_count_at_slot(state, Slot(slot)) for slot in range(start_slot, stop_slot))
+```
+
 #### `get_start_shard`
 
 ```python
 def get_start_shard(state: BeaconState, slot: Slot) -> Shard:
-    # TODO: implement start shard logic
+    """
-    return Shard(0)
+    Return the start shard at ``slot``.
+    """
+    current_epoch_start_slot = compute_start_slot_at_epoch(get_current_epoch(state))
+    active_shard_count = get_active_shard_count(state)
+    if current_epoch_start_slot == slot:
+        return state.current_epoch_start_shard
+    elif slot > current_epoch_start_slot:
+        # Current epoch or the next epoch lookahead
+        shard_delta = get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot)
+        return Shard((state.current_epoch_start_shard + shard_delta) % active_shard_count)
+    else:
+        # Previous epoch
+        shard_delta = get_committee_count_delta(state, start_slot=slot, stop_slot=current_epoch_start_slot)
+        max_committees_per_epoch = MAX_COMMITTEES_PER_SLOT * SLOTS_PER_EPOCH
+        return Shard(
+            # Ensure positive
+            (state.current_epoch_start_shard + max_committees_per_epoch * active_shard_count - shard_delta)
+            % active_shard_count
+        )
 ```
 
 #### `get_shard`
 
 ```python
 def get_shard(state: BeaconState, attestation: Attestation) -> Shard:
+    """
+    Return the shard that the given ``attestation`` is attesting.
+    """
     return compute_shard_from_committee_index(state, attestation.data.index, attestation.data.slot)
 ```
 
@@ -569,6 +603,9 @@ def get_shard(state: BeaconState, attestation: Attestation) -> Shard:
 
 ```python
 def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot:
+    """
+    Return the latest slot number of the given ``shard``.
+    """
     return state.shard_states[shard].slot
 ```
 
@@ -577,7 +614,8 @@ def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot:
 ```python
 def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]:
     """
-    Return the offset slots of the given ``shard`` between that latest included slot and current slot.
+    Return the offset slots of the given ``shard``.
+    The offset slot are after the latest slot and before current slot. 
     """
     return compute_offset_slots(get_latest_slot_for_shard(state, shard), state.slot)
 ```
@@ -651,8 +689,7 @@ def is_on_time_attestation(state: BeaconState,
     """
     Check if the given attestation is on-time.
     """
-    # TODO: MIN_ATTESTATION_INCLUSION_DELAY should always be 1
+    return attestation.data.slot == compute_previous_slot(state.slot)
-    return attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot
 ```
 
 #### `is_winning_attestation`
@@ -761,20 +798,19 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None:
     else:
         assert attestation.data.source == state.previous_justified_checkpoint
 
-    shard = get_shard(state, attestation)
-
     # Type 1: on-time attestations, the custody bits should be non-empty.
     if attestation.custody_bits_blocks != []:
         # Ensure on-time attestation
         assert is_on_time_attestation(state, attestation)
         # Correct data root count
+        shard = get_shard(state, attestation)
         assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, shard))
         # Correct parent block root
         assert data.beacon_block_root == get_block_root_at_slot(state, compute_previous_slot(state.slot))
     # Type 2: no shard transition, no custody bits
     else:
         # Ensure delayed attestation
-        assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY < state.slot  
+        assert data.slot < compute_previous_slot(state.slot)
         # Late attestations cannot have a shard transition root
         assert data.shard_transition_root == Root()
 
@@ -869,9 +905,10 @@ def process_crosslink_for_shard(state: BeaconState,
                                 committee_index: CommitteeIndex,
                                 shard_transition: ShardTransition,
                                 attestations: Sequence[Attestation]) -> Root:
-    committee = get_beacon_committee(state, state.slot, committee_index)
+    on_time_attestation_slot = compute_previous_slot(state.slot)
+    committee = get_beacon_committee(state, on_time_attestation_slot, committee_index)
     online_indices = get_online_validator_indices(state)
-    shard = compute_shard_from_committee_index(state, committee_index, state.slot)
+    shard = compute_shard_from_committee_index(state, committee_index, on_time_attestation_slot)
 
     # Loop over all shard transition roots
     shard_transition_roots = set([a.data.shard_transition_root for a in attestations])
@@ -926,15 +963,15 @@ def process_crosslink_for_shard(state: BeaconState,
 def process_crosslinks(state: BeaconState,
                        shard_transitions: Sequence[ShardTransition],
                        attestations: Sequence[Attestation]) -> None:
-    committee_count = get_committee_count_at_slot(state, state.slot)
+    on_time_attestation_slot = compute_previous_slot(state.slot)
+    committee_count = get_committee_count_at_slot(state, on_time_attestation_slot)
     for committee_index in map(CommitteeIndex, range(committee_count)):
-        shard = compute_shard_from_committee_index(state, committee_index, state.slot)
         # All attestations in the block for this committee/shard and current slot
         shard_attestations = [
             attestation for attestation in attestations
             if is_on_time_attestation(state, attestation) and attestation.data.index == committee_index
         ]
-
+        shard = compute_shard_from_committee_index(state, committee_index, on_time_attestation_slot)
         winning_root = process_crosslink_for_shard(state, committee_index, shard_transitions[shard], shard_attestations)
         if winning_root != Root():
             # Mark relevant pending attestations as creating a successful crosslink
@@ -1041,10 +1078,20 @@ def process_epoch(state: BeaconState) -> None:
     process_registry_updates(state)
     process_reveal_deadlines(state)
     process_slashings(state)
-    process_final_updates(state)
+    process_final_updates(state)  # phase 0 final updates
+    process_phase_1_final_updates(state)
+```
+
+#### Phase 1 final updates
+
+```python
+def process_phase_1_final_updates(state: BeaconState) -> None:
     process_custody_final_updates(state)
     process_online_tracking(state)
     process_light_client_committee_updates(state)
+
+    # Update current_epoch_start_shard
+    state.current_epoch_start_shard = get_start_shard(state, Slot(state.slot + 1))
 ```
 
 #### Custody game updates

specs/phase1/phase1-fork.md

@@ -99,6 +99,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState:
         current_justified_checkpoint=pre.current_justified_checkpoint,
         finalized_checkpoint=pre.finalized_checkpoint,
         # Phase 1
+        current_epoch_start_shard=Shard(0),
         shard_states=List[ShardState, MAX_SHARDS](
             ShardState(
                 slot=pre.slot,

tests/core/pyspec/eth2spec/test/helpers/attestations.py

@@ -1,8 +1,9 @@
 from typing import List
 
 from eth2spec.test.context import expect_assertion_error, PHASE0, PHASE1
-from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot, transition_to
+from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot
 from eth2spec.test.helpers.block import build_empty_block_for_next_slot
+from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee
 from eth2spec.test.helpers.keys import privkeys
 from eth2spec.utils import bls
 from eth2spec.utils.ssz.ssz_typing import Bitlist
@@ -81,12 +82,12 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t
             attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index]
             attestation_data.shard_transition_root = shard_transition.hash_tree_root()
         else:
-            # No shard transition
+            # No shard transition -> no shard block
             shard = spec.get_shard(state, spec.Attestation(data=attestation_data))
             if on_time:
                 temp_state = state.copy()
                 next_slot(spec, temp_state)
-                shard_transition = spec.get_shard_transition(temp_state, shard, [])
+                shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[])
                 lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1
                 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index]
                 attestation_data.shard_transition_root = shard_transition.hash_tree_root()
@@ -180,7 +181,7 @@ def get_valid_attestation(spec,
     fill_aggregate_attestation(spec, state, attestation, signed=signed, filter_participant_set=filter_participant_set)
 
     if spec.fork == PHASE1 and on_time:
-        attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed)
+        attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed=signed)
 
     return attestation
 
@@ -317,7 +318,19 @@ def next_epoch_with_attestations(spec,
             committees_per_slot = spec.get_committee_count_at_slot(state, slot_to_attest)
             if slot_to_attest >= spec.compute_start_slot_at_epoch(spec.get_current_epoch(post_state)):
                 for index in range(committees_per_slot):
-                    cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest, index=index, signed=True)
+                    if spec.fork == PHASE1:
+                        shard = spec.compute_shard_from_committee_index(post_state, index, slot_to_attest)
+                        shard_transition = get_shard_transition_of_committee(
+                            spec, post_state, index, slot=slot_to_attest
+                        )
+                        block.body.shard_transitions[shard] = shard_transition
+                    else:
+                        shard_transition = None
+
+                    cur_attestation = get_valid_attestation(
+                        spec, post_state, slot_to_attest,
+                        shard_transition=shard_transition, index=index, signed=True, on_time=True
+                    )
                     block.body.attestations.append(cur_attestation)
 
         if fill_prev_epoch:
@@ -328,9 +341,6 @@ def next_epoch_with_attestations(spec,
                     spec, post_state, slot_to_attest, index=index, signed=True, on_time=False)
                 block.body.attestations.append(prev_attestation)
 
-        if spec.fork == PHASE1:
-            fill_block_shard_transitions_by_attestations(spec, post_state, block)
-
         signed_block = state_transition_and_sign_block(spec, post_state, block)
         signed_blocks.append(signed_block)
 
@@ -396,14 +406,3 @@ def cached_prepare_state_with_attestations(spec, state):
 
     # Put the LRU cache result into the state view, as if we transitioned the original view
     state.set_backing(_prep_state_cache_dict[key])
-
-
-def fill_block_shard_transitions_by_attestations(spec, state, block):
-    block.body.shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS
-    for attestation in block.body.attestations:
-        shard = spec.get_shard(state, attestation)
-        if attestation.data.slot == state.slot:
-            temp_state = state.copy()
-            transition_to(spec, temp_state, slot=block.slot)
-            shard_transition = spec.get_shard_transition(temp_state, shard, [])
-            block.body.shard_transitions[shard] = shard_transition

tests/core/pyspec/eth2spec/test/helpers/shard_block.py

@@ -70,7 +70,7 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot):
     return shard_transitions
 
 
-def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition=None):
+def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition):
     temp_state = state.copy()
     transition_to(spec, temp_state, on_time_slot - 1)
     attestation = get_valid_on_time_attestation(

tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py

@@ -1,4 +1,5 @@
 from eth2spec.test.context import expect_assertion_error
+from eth2spec.test.helpers.state import transition_to
 
 
 def run_shard_transitions_processing(spec, state, shard_transitions, attestations, valid=True):
@@ -26,3 +27,17 @@ def run_shard_transitions_processing(spec, state, shard_transitions, attestation
 
     # yield post-state
     yield 'post', state
+
+
+def get_shard_transition_of_committee(spec, state, committee_index, slot=None, shard_blocks=None):
+    if shard_blocks is None:
+        shard_blocks = []
+
+    if slot is None:
+        slot = state.slot
+
+    shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot)
+    temp_state = state.copy()
+    transition_to(spec, temp_state, slot + 1)
+    shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=shard_blocks)
+    return shard_transition

tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py

@@ -16,8 +16,9 @@ from eth2spec.test.helpers.attester_slashings import (
     get_indexed_attestation_participants,
 )
 from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing, check_proposer_slashing_effect
-from eth2spec.test.helpers.attestations import get_valid_attestation, fill_block_shard_transitions_by_attestations
+from eth2spec.test.helpers.attestations import get_valid_attestation
 from eth2spec.test.helpers.deposits import prepare_state_and_deposit
+from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee
 
 from eth2spec.test.context import (
     spec_state_test, with_all_phases, expect_assertion_error, always_bls, with_phases,
@@ -687,14 +688,23 @@ def test_attestation(spec, state):
 
     yield 'pre', state
 
-    attestation = get_valid_attestation(spec, state, signed=True, on_time=True)
+    attestation_block = build_empty_block(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY)
+
+    index = 0
+    if spec.fork == PHASE1:
+        shard = spec.compute_shard_from_committee_index(state, index, state.slot)
+        shard_transition = get_shard_transition_of_committee(spec, state, index)
+        attestation_block.body.shard_transitions[shard] = shard_transition
+    else:
+        shard_transition = None
+
+    attestation = get_valid_attestation(
+        spec, state, shard_transition=shard_transition, index=index, signed=True, on_time=True
+    )
 
     # Add to state via block transition
     pre_current_attestations_len = len(state.current_epoch_attestations)
-    attestation_block = build_empty_block(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY)
     attestation_block.body.attestations.append(attestation)
-    if spec.fork == PHASE1:
-        fill_block_shard_transitions_by_attestations(spec, state, attestation_block)
     signed_attestation_block = state_transition_and_sign_block(spec, state, attestation_block)
 
     assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1

tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py

@@ -15,11 +15,10 @@ from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard
 
 def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True):
     state = transition_to_valid_shard_slot(spec, state)
-    # At the beginning, let `x = state.slot`, `state.shard_states[shard].slot == x - 1`
+    init_slot = state.slot
-    slot_x = state.slot
     committee_index = spec.CommitteeIndex(0)
-    shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot)
+    shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1)
-    assert state.shard_states[shard].slot == slot_x - 1
+    assert state.shard_states[shard].slot == state.slot - 1
 
     # Create SignedShardBlock
     body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE
@@ -50,7 +49,7 @@ def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True):
 
     if valid:
         # After state transition,
-        assert state.slot == slot_x + target_len_offset_slot
+        assert state.slot == init_slot + target_len_offset_slot
         shard_state = state.shard_states[shard]
         assert shard_state != pre_shard_state
         assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1]

tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py

@@ -67,7 +67,7 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state):
 
     target_len_offset_slot = 1
     committee_index = spec.CommitteeIndex(0)
-    shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot)
+    shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1)
     assert state.shard_states[shard].slot == state.slot - 1
 
     pre_gasprice = state.shard_states[shard].gasprice
@@ -93,7 +93,7 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state):
 
     target_len_offset_slot = 1
     committee_index = spec.CommitteeIndex(0)
-    shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot)
+    shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1)
     assert state.shard_states[shard].slot == state.slot - 1
 
     # No new shard block

tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py

@@ -0,0 +1,71 @@
+from eth2spec.test.context import (
+    PHASE0,
+    with_all_phases_except,
+    spec_state_test,
+)
+from eth2spec.test.helpers.state import next_epoch
+
+
+@with_all_phases_except([PHASE0])
+@spec_state_test
+def test_get_committee_count_delta(spec, state):
+    assert spec.get_committee_count_delta(state, 0, 0) == 0
+    assert spec.get_committee_count_at_slot(state, 0) != 0
+    assert spec.get_committee_count_delta(state, 0, 1) == spec.get_committee_count_at_slot(state, 0)
+    assert spec.get_committee_count_delta(state, 1, 2) == spec.get_committee_count_at_slot(state, 1)
+    assert spec.get_committee_count_delta(state, 0, 2) == (
+        spec.get_committee_count_at_slot(state, 0) + spec.get_committee_count_at_slot(state, 1)
+    )
+
+
+@with_all_phases_except([PHASE0])
+@spec_state_test
+def test_get_start_shard_current_epoch_start(spec, state):
+    assert state.current_epoch_start_shard == 0
+    next_epoch(spec, state)
+    active_shard_count = spec.get_active_shard_count(state)
+    assert state.current_epoch_start_shard == (
+        spec.get_committee_count_delta(state, 0, spec.SLOTS_PER_EPOCH) % active_shard_count
+    )
+    current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state))
+
+    slot = current_epoch_start_slot
+    start_shard = spec.get_start_shard(state, slot)
+    assert start_shard == state.current_epoch_start_shard
+
+
+@with_all_phases_except([PHASE0])
+@spec_state_test
+def test_get_start_shard_next_slot(spec, state):
+    next_epoch(spec, state)
+    active_shard_count = spec.get_active_shard_count(state)
+    current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state))
+
+    slot = current_epoch_start_slot + 1
+    start_shard = spec.get_start_shard(state, slot)
+
+    current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state))
+    expected_start_shard = (
+        state.current_epoch_start_shard
+        + spec.get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot)
+    ) % active_shard_count
+    assert start_shard == expected_start_shard
+
+
+@with_all_phases_except([PHASE0])
+@spec_state_test
+def test_get_start_shard_previous_slot(spec, state):
+    next_epoch(spec, state)
+    active_shard_count = spec.get_active_shard_count(state)
+    current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state))
+
+    slot = current_epoch_start_slot - 1
+    start_shard = spec.get_start_shard(state, slot)
+
+    current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state))
+    expected_start_shard = (
+        state.current_epoch_start_shard
+        + spec.MAX_COMMITTEES_PER_SLOT * spec.SLOTS_PER_EPOCH * active_shard_count
+        - spec.get_committee_count_delta(state, start_slot=slot, stop_slot=current_epoch_start_slot)
+    ) % active_shard_count
+    assert start_shard == expected_start_shard