From dab5a936c4b1fdfae0cc9c9b5c98ac0643097bd7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 Apr 2020 23:55:46 +0800 Subject: [PATCH 01/30] wip shard fork choice rule --- setup.py | 1 + specs/phase1/shard-fork-choice.md | 209 ++++++++++++++++++ .../test/fork_choice/test_get_head.py | 30 +-- .../test/fork_choice/test_on_shard_head.py | 97 ++++++++ .../eth2spec/test/helpers/fork_choice.py | 27 +++ 5 files changed, 335 insertions(+), 29 deletions(-) create mode 100644 specs/phase1/shard-fork-choice.md create mode 100644 tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py create mode 100644 tests/core/pyspec/eth2spec/test/helpers/fork_choice.py diff --git a/setup.py b/setup.py index e0d6561dd..316a7d32a 100644 --- a/setup.py +++ b/setup.py @@ -378,6 +378,7 @@ class PySpecCommand(Command): specs/phase1/shard-transition.md specs/phase1/fork-choice.md specs/phase1/phase1-fork.md + specs/phase1/shard-fork-choice.md """ else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md new file mode 100644 index 000000000..46e467e12 --- /dev/null +++ b/specs/phase1/shard-fork-choice.md @@ -0,0 +1,209 @@ +# Ethereum 2.0 Phase 1 -- Beacon Chain + Shard Chain Fork Choice + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [Introduction](#introduction) +- [Fork choice](#fork-choice) + - [Helpers](#helpers) + - [Extended `Store`](#extended-store) + - [Updated `get_forkchoice_store`](#updated-get_forkchoice_store) + - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) + - [`get_shard_head`](#get_shard_head) + - [`get_shard_ancestor`](#get_shard_ancestor) + - [`filter_shard_block_tree`](#filter_shard_block_tree) + - [`get_filtered_block_tree`](#get_filtered_block_tree) + - [Handlers](#handlers) + - [`on_shard_block`](#on_shard_block) + + + +## Introduction + +This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. + +## Fork choice + +### Helpers + +#### Extended `Store` + +```python +@dataclass +class Store(object): + time: uint64 + genesis_time: uint64 + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint + best_justified_checkpoint: Checkpoint + blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) + block_states: Dict[Root, BeaconState] = field(default_factory=dict) + checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) + latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) + # shard chain + shard_init_slots: Dict[Shard, Slot] = field(default_factory=dict) + shard_blocks: Dict[Shard, Dict[Root, ShardBlock]] = field(default_factory=dict) + shard_block_states: Dict[Shard, Dict[Root, ShardState]] = field(default_factory=dict) +``` + +#### Updated `get_forkchoice_store` + +```python +def get_forkchoice_store(anchor_state: BeaconState, + shard_init_slots: Dict[Shard, Slot], + anchor_state_shard_blocks: Dict[Shard, Dict[Root, ShardBlock]]) -> Store: + shard_count = len(anchor_state.shard_states) + anchor_block_header = anchor_state.latest_block_header.copy() + if anchor_block_header.state_root == Bytes32(): + anchor_block_header.state_root = hash_tree_root(anchor_state) + anchor_root = hash_tree_root(anchor_block_header) + anchor_epoch = get_current_epoch(anchor_state) + justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + return Store( + time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, + genesis_time=anchor_state.genesis_time, + justified_checkpoint=justified_checkpoint, + finalized_checkpoint=finalized_checkpoint, + best_justified_checkpoint=justified_checkpoint, + blocks={anchor_root: anchor_block_header}, + block_states={anchor_root: anchor_state.copy()}, + checkpoint_states={justified_checkpoint: anchor_state.copy()}, + # shard chain + shard_init_slots=shard_init_slots, + shard_blocks=anchor_state_shard_blocks, + shard_block_states={ + shard: { + anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard] + } + for shard in map(Shard, range(shard_count)) + }, + ) +``` + +#### `get_shard_latest_attesting_balance` + +```python +def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) -> Gwei: + state = store.checkpoint_states[store.justified_checkpoint] + active_indices = get_active_validator_indices(state, get_current_epoch(state)) + return Gwei(sum( + state.validators[i].effective_balance for i in active_indices + if ( + i in store.latest_messages and get_shard_ancestor( + store, shard, store.latest_messages[i].root, store.shard_blocks[shard][root].slot + ) == root + ) + )) +``` + +#### `get_shard_head` + +```python +def get_shard_head(store: Store, shard: Shard) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_shard_block_tree(store, shard) + + # Execute the LMD-GHOST fork choice + head_beacon_root = get_head(store) + head_shard_root = store.block_states[head_beacon_root].shard_states[shard].latest_block_root + while True: + children = [ + root for root in blocks.keys() + if blocks[root].shard_parent_root == head_shard_root + ] + if len(children) == 0: + return head_shard_root + # Sort by latest attesting balance with ties broken lexicographically + head_shard_root = max(children, key=lambda root: (get_shard_latest_attesting_balance(store, shard, root), root)) +``` + +#### `get_shard_ancestor` + +```python +def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: + block = store.shard_blocks[shard][root] + if block.slot > slot: + return get_shard_ancestor(store, shard, block.shard_parent_root, slot) + elif block.slot == slot: + return root + else: + # root is older than queried slot, thus a skip slot. Return earliest root prior to slot + return root +``` + +#### `filter_shard_block_tree` + +```python +def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: + block = store.shard_blocks[shard][block_root] + children = [ + root for root in store.shard_blocks[shard].keys() + if ( + store.shard_blocks[shard][root].shard_parent_root == block_root + and store.shard_blocks[shard][root].slot != store.shard_init_slots[shard] + ) + ] + + if any(children): + filter_block_tree_result = [filter_shard_block_tree(store, shard, child, blocks) for child in children] + if any(filter_block_tree_result): + blocks[block_root] = block + return True + return False + + return False +``` + +#### `get_filtered_block_tree` + +```python +def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, ShardBlock]: + base_beacon_block_root = get_head(store) + base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root + blocks: Dict[Root, ShardBlock] = {} + filter_shard_block_tree(store, shard, base_shard_block_root, blocks) + return blocks +``` + +### Handlers + +#### `on_shard_block` + +```python +def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: + shard_block = signed_shard_block.message + + # 1. Check shard parent exists + assert shard_block.shard_parent_root in store.shard_block_states[shard] + pre_shard_state = store.shard_block_states[shard][shard_block.shard_parent_root] + + # 2. Check beacon parent exists + assert shard_block.beacon_parent_root in store.block_states + beacon_state = store.block_states[shard_block.beacon_parent_root] + + # 3. Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert shard_block.slot > finalized_slot + + # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + assert ( + shard_block.beacon_parent_root == store.finalized_checkpoint.root + or get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root + ) + + # Add new block to the store + store.shard_blocks[shard][hash_tree_root(shard_block)] = shard_block + + # Check the block is valid and compute the post-state + verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) + verify_shard_block_signature(beacon_state, signed_shard_block) + post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) + # Add new state for this block to the store + store.shard_block_states[shard][hash_tree_root(shard_block)] = post_state +``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py index 17d4f644f..e25aad18f 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -1,41 +1,13 @@ from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.fork_choice import add_attestation_to_store, add_block_to_store, get_anchor_root from eth2spec.test.helpers.state import ( next_epoch, state_transition_and_sign_block, ) -def add_block_to_store(spec, store, signed_block): - pre_state = store.block_states[signed_block.message.parent_root] - block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT - - if store.time < block_time: - spec.on_tick(store, block_time) - - spec.on_block(store, signed_block) - - -def add_attestation_to_store(spec, store, attestation): - parent_block = store.blocks[attestation.data.beacon_block_root] - pre_state = store.block_states[spec.hash_tree_root(parent_block)] - block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT - next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT - - if store.time < next_epoch_time: - spec.on_tick(store, next_epoch_time) - - spec.on_attestation(store, attestation) - - -def get_anchor_root(spec, state): - anchor_block_header = state.latest_block_header.copy() - if anchor_block_header.state_root == spec.Bytes32(): - anchor_block_header.state_root = spec.hash_tree_root(state) - return spec.hash_tree_root(anchor_block_header) - - @with_all_phases @spec_state_test def test_genesis(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py new file mode 100644 index 000000000..8e72e214e --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -0,0 +1,97 @@ +from eth2spec.utils.ssz.ssz_impl import hash_tree_root + +from eth2spec.test.context import spec_state_test, with_all_phases_except, PHASE0 +from eth2spec.test.helpers.shard_block import ( + build_attestation_with_shard_transition, + build_shard_block, + build_shard_transitions_till_slot, +) +from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root +from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block +from eth2spec.test.helpers.block import build_empty_block + + +def run_on_shard_block(spec, store, shard, signed_block, valid=True): + if not valid: + try: + spec.on_shard_block(store, shard, signed_block) + except AssertionError: + return + else: + assert False + + spec.on_shard_block(store, shard, signed_block) + assert store.shard_blocks[shard][hash_tree_root(signed_block.message)] == signed_block.message + + +def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): + store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + + # Create SignedShardBlock + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_blocks = [shard_block] + + # Attester creates `attestation` + # Use temporary next state to get ShardTransition of shard block + shard_transitions = build_shard_transitions_till_slot( + spec, + state, + shards=[shard, ], + shard_blocks={shard: shard_blocks}, + target_len_offset_slot=1, + ) + shard_transition = shard_transitions[shard] + attestation = build_attestation_with_shard_transition( + spec, + state, + slot=state.slot, + index=committee_index, + target_len_offset_slot=1, + shard_transition=shard_transition, + ) + + # Propose beacon block at slot + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) + beacon_block.body.attestations = [attestation] + beacon_block.body.shard_transitions = shard_transitions + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + + run_on_shard_block(spec, store, shard, shard_block) + add_block_to_store(spec, store, signed_beacon_block) + + assert spec.get_head(store) == beacon_block.hash_tree_root() + assert spec.get_shard_head(store, shard) == shard_block.message.hash_tree_root() + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_basic(spec, state): + spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking + state = spec.upgrade_to_phase1(state) + next_slot(spec, state) + + # Initialization + shard_count = len(state.shard_states) + # Genesis shard blocks + anchor_shard_blocks = { + shard: { + state.shard_states[shard].latest_block_root: spec.ShardBlock( + slot=state.slot, + ) + } + for shard in map(spec.Shard, range(shard_count)) + } + shard_init_slots = { + shard: state.slot + for shard in map(spec.Shard, range(shard_count)) + } + store = spec.get_forkchoice_store(state, shard_init_slots, anchor_shard_blocks) + anchor_root = get_anchor_root(spec, state) + assert spec.get_head(store) == anchor_root + + committee_index = spec.CommitteeIndex(0) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + + run_apply_shard_and_beacon(spec, state, store, shard, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard, committee_index) diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py new file mode 100644 index 000000000..04e36ea84 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -0,0 +1,27 @@ +def get_anchor_root(spec, state): + anchor_block_header = state.latest_block_header.copy() + if anchor_block_header.state_root == spec.Bytes32(): + anchor_block_header.state_root = spec.hash_tree_root(state) + return spec.hash_tree_root(anchor_block_header) + + +def add_block_to_store(spec, store, signed_block): + pre_state = store.block_states[signed_block.message.parent_root] + block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT + + if store.time < block_time: + spec.on_tick(store, block_time) + + spec.on_block(store, signed_block) + + +def add_attestation_to_store(spec, store, attestation): + parent_block = store.blocks[attestation.data.beacon_block_root] + pre_state = store.block_states[spec.hash_tree_root(parent_block)] + block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT + next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT + + if store.time < next_epoch_time: + spec.on_tick(store, next_epoch_time) + + spec.on_attestation(store, attestation) From cddf9cf114744c07d8a1ce455c5cf73ff5ad89db Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 Apr 2020 20:06:20 +0800 Subject: [PATCH 02/30] Refactor --- specs/phase1/shard-fork-choice.md | 56 ++++++++++--------- .../test/fork_choice/test_on_shard_head.py | 18 +----- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 46e467e12..e9026b991 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -35,7 +35,13 @@ This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase ```python @dataclass -class Store(object): +class Store: + + @dataclass + class ShardStore: + blocks: Dict[Root, ShardBlock] = field(default_factory=dict) + block_states: Dict[Root, ShardState] = field(default_factory=dict) + time: uint64 genesis_time: uint64 justified_checkpoint: Checkpoint @@ -46,17 +52,13 @@ class Store(object): checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) # shard chain - shard_init_slots: Dict[Shard, Slot] = field(default_factory=dict) - shard_blocks: Dict[Shard, Dict[Root, ShardBlock]] = field(default_factory=dict) - shard_block_states: Dict[Shard, Dict[Root, ShardState]] = field(default_factory=dict) + shards: Dict[Shard, ShardStore] = field(default_factory=dict) # noqa: F821 ``` #### Updated `get_forkchoice_store` ```python -def get_forkchoice_store(anchor_state: BeaconState, - shard_init_slots: Dict[Shard, Slot], - anchor_state_shard_blocks: Dict[Shard, Dict[Root, ShardBlock]]) -> Store: +def get_forkchoice_store(anchor_state: BeaconState) -> Store: shard_count = len(anchor_state.shard_states) anchor_block_header = anchor_state.latest_block_header.copy() if anchor_block_header.state_root == Bytes32(): @@ -65,6 +67,14 @@ def get_forkchoice_store(anchor_state: BeaconState, anchor_epoch = get_current_epoch(anchor_state) justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + + shard_stores = {} + for shard in map(Shard, range(shard_count)): + shard_stores[shard] = Store.ShardStore( + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, + block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, + ) + return Store( time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, genesis_time=anchor_state.genesis_time, @@ -75,14 +85,7 @@ def get_forkchoice_store(anchor_state: BeaconState, block_states={anchor_root: anchor_state.copy()}, checkpoint_states={justified_checkpoint: anchor_state.copy()}, # shard chain - shard_init_slots=shard_init_slots, - shard_blocks=anchor_state_shard_blocks, - shard_block_states={ - shard: { - anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard] - } - for shard in map(Shard, range(shard_count)) - }, + shards=shard_stores, ) ``` @@ -96,7 +99,7 @@ def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) - state.validators[i].effective_balance for i in active_indices if ( i in store.latest_messages and get_shard_ancestor( - store, shard, store.latest_messages[i].root, store.shard_blocks[shard][root].slot + store, shard, store.latest_messages[i].root, store.shards[shard].blocks[root].slot ) == root ) )) @@ -127,7 +130,7 @@ def get_shard_head(store: Store, shard: Shard) -> Root: ```python def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: - block = store.shard_blocks[shard][root] + block = store.shards[shard].blocks[root] if block.slot > slot: return get_shard_ancestor(store, shard, block.shard_parent_root, slot) elif block.slot == slot: @@ -141,13 +144,11 @@ def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Ro ```python def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: - block = store.shard_blocks[shard][block_root] + shard_store = store.shards[shard] + block = shard_store.blocks[block_root] children = [ - root for root in store.shard_blocks[shard].keys() - if ( - store.shard_blocks[shard][root].shard_parent_root == block_root - and store.shard_blocks[shard][root].slot != store.shard_init_slots[shard] - ) + root for root in shard_store.blocks.keys() + if shard_store.blocks[root].shard_parent_root == block_root ] if any(children): @@ -178,10 +179,11 @@ def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, Shar ```python def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: shard_block = signed_shard_block.message + shard_store = store.shards[shard] # 1. Check shard parent exists - assert shard_block.shard_parent_root in store.shard_block_states[shard] - pre_shard_state = store.shard_block_states[shard][shard_block.shard_parent_root] + assert shard_block.shard_parent_root in shard_store.block_states + pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] # 2. Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states @@ -198,12 +200,12 @@ def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBl ) # Add new block to the store - store.shard_blocks[shard][hash_tree_root(shard_block)] = shard_block + shard_store.blocks[hash_tree_root(shard_block)] = shard_block # Check the block is valid and compute the post-state verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) verify_shard_block_signature(beacon_state, signed_shard_block) post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) # Add new state for this block to the store - store.shard_block_states[shard][hash_tree_root(shard_block)] = post_state + shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 8e72e214e..220c510e7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -21,7 +21,7 @@ def run_on_shard_block(spec, store, shard, signed_block, valid=True): assert False spec.on_shard_block(store, shard, signed_block) - assert store.shard_blocks[shard][hash_tree_root(signed_block.message)] == signed_block.message + assert store.shards[shard].blocks[hash_tree_root(signed_block.message)] == signed_block.message def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): @@ -72,21 +72,7 @@ def test_basic(spec, state): next_slot(spec, state) # Initialization - shard_count = len(state.shard_states) - # Genesis shard blocks - anchor_shard_blocks = { - shard: { - state.shard_states[shard].latest_block_root: spec.ShardBlock( - slot=state.slot, - ) - } - for shard in map(spec.Shard, range(shard_count)) - } - shard_init_slots = { - shard: state.slot - for shard in map(spec.Shard, range(shard_count)) - } - store = spec.get_forkchoice_store(state, shard_init_slots, anchor_shard_blocks) + store = spec.get_forkchoice_store(state) anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root From 8fafb6a9e5b17ee3e93fa0c1b3a8795608f7336b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 May 2020 10:25:58 +0800 Subject: [PATCH 03/30] Make `ShardStore` an independent object --- specs/phase1/shard-fork-choice.md | 97 ++++++------------- .../test/fork_choice/test_on_shard_head.py | 20 ++-- 2 files changed, 43 insertions(+), 74 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index e9026b991..4b3f42194 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -31,75 +31,38 @@ This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase ### Helpers -#### Extended `Store` +#### `ShardStore` ```python @dataclass -class Store: - - @dataclass - class ShardStore: - blocks: Dict[Root, ShardBlock] = field(default_factory=dict) - block_states: Dict[Root, ShardState] = field(default_factory=dict) - - time: uint64 - genesis_time: uint64 - justified_checkpoint: Checkpoint - finalized_checkpoint: Checkpoint - best_justified_checkpoint: Checkpoint - blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) - block_states: Dict[Root, BeaconState] = field(default_factory=dict) - checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) - latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) - # shard chain - shards: Dict[Shard, ShardStore] = field(default_factory=dict) # noqa: F821 +class ShardStore: + shard: Shard + blocks: Dict[Root, ShardBlock] = field(default_factory=dict) + block_states: Dict[Root, ShardState] = field(default_factory=dict) ``` -#### Updated `get_forkchoice_store` +#### Updated `get_forkchoice_shard_store` ```python -def get_forkchoice_store(anchor_state: BeaconState) -> Store: - shard_count = len(anchor_state.shard_states) - anchor_block_header = anchor_state.latest_block_header.copy() - if anchor_block_header.state_root == Bytes32(): - anchor_block_header.state_root = hash_tree_root(anchor_state) - anchor_root = hash_tree_root(anchor_block_header) - anchor_epoch = get_current_epoch(anchor_state) - justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - - shard_stores = {} - for shard in map(Shard, range(shard_count)): - shard_stores[shard] = Store.ShardStore( - blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, - block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, - ) - - return Store( - time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, - genesis_time=anchor_state.genesis_time, - justified_checkpoint=justified_checkpoint, - finalized_checkpoint=finalized_checkpoint, - best_justified_checkpoint=justified_checkpoint, - blocks={anchor_root: anchor_block_header}, - block_states={anchor_root: anchor_state.copy()}, - checkpoint_states={justified_checkpoint: anchor_state.copy()}, - # shard chain - shards=shard_stores, +def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: + return ShardStore( + shard=shard, + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, + block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, ) ``` #### `get_shard_latest_attesting_balance` ```python -def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) -> Gwei: +def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, root: Root) -> Gwei: state = store.checkpoint_states[store.justified_checkpoint] active_indices = get_active_validator_indices(state, get_current_epoch(state)) return Gwei(sum( state.validators[i].effective_balance for i in active_indices if ( i in store.latest_messages and get_shard_ancestor( - store, shard, store.latest_messages[i].root, store.shards[shard].blocks[root].slot + store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot ) == root ) )) @@ -108,13 +71,13 @@ def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) - #### `get_shard_head` ```python -def get_shard_head(store: Store, shard: Shard) -> Root: +def get_shard_head(store: Store, shard_store: ShardStore) -> Root: # Get filtered block tree that only includes viable branches - blocks = get_filtered_shard_block_tree(store, shard) + blocks = get_filtered_shard_block_tree(store, shard_store) # Execute the LMD-GHOST fork choice head_beacon_root = get_head(store) - head_shard_root = store.block_states[head_beacon_root].shard_states[shard].latest_block_root + head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root while True: children = [ root for root in blocks.keys() @@ -123,16 +86,18 @@ def get_shard_head(store: Store, shard: Shard) -> Root: if len(children) == 0: return head_shard_root # Sort by latest attesting balance with ties broken lexicographically - head_shard_root = max(children, key=lambda root: (get_shard_latest_attesting_balance(store, shard, root), root)) + head_shard_root = max( + children, key=lambda root: (get_shard_latest_attesting_balance(store, shard_store, root), root) + ) ``` #### `get_shard_ancestor` ```python -def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: - block = store.shards[shard].blocks[root] +def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: Slot) -> Root: + block = shard_store.blocks[root] if block.slot > slot: - return get_shard_ancestor(store, shard, block.shard_parent_root, slot) + return get_shard_ancestor(store, shard_store, block.shard_parent_root, slot) elif block.slot == slot: return root else: @@ -143,8 +108,10 @@ def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Ro #### `filter_shard_block_tree` ```python -def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: - shard_store = store.shards[shard] +def filter_shard_block_tree(store: Store, + shard_store: ShardStore, + block_root: Root, + blocks: Dict[Root, ShardBlock]) -> bool: block = shard_store.blocks[block_root] children = [ root for root in shard_store.blocks.keys() @@ -152,7 +119,7 @@ def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks ] if any(children): - filter_block_tree_result = [filter_shard_block_tree(store, shard, child, blocks) for child in children] + filter_block_tree_result = [filter_shard_block_tree(store, shard_store, child, blocks) for child in children] if any(filter_block_tree_result): blocks[block_root] = block return True @@ -164,11 +131,12 @@ def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks #### `get_filtered_block_tree` ```python -def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, ShardBlock]: +def get_filtered_shard_block_tree(store: Store, shard_store: ShardStore) -> Dict[Root, ShardBlock]: + shard = shard_store.shard base_beacon_block_root = get_head(store) base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root blocks: Dict[Root, ShardBlock] = {} - filter_shard_block_tree(store, shard, base_shard_block_root, blocks) + filter_shard_block_tree(store, shard_store, base_shard_block_root, blocks) return blocks ``` @@ -177,10 +145,9 @@ def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, Shar #### `on_shard_block` ```python -def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: +def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: SignedShardBlock) -> None: shard_block = signed_shard_block.message - shard_store = store.shards[shard] - + shard = shard_store.shard # 1. Check shard parent exists assert shard_block.shard_parent_root in shard_store.block_states pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 220c510e7..f4b883f06 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -11,20 +11,21 @@ from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_blo from eth2spec.test.helpers.block import build_empty_block -def run_on_shard_block(spec, store, shard, signed_block, valid=True): +def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): if not valid: try: - spec.on_shard_block(store, shard, signed_block) + spec.on_shard_block(store, shard_store, signed_block) except AssertionError: return else: assert False - spec.on_shard_block(store, shard, signed_block) - assert store.shards[shard].blocks[hash_tree_root(signed_block.message)] == signed_block.message + spec.on_shard_block(store, shard_store, signed_block) + assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): +def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index): + shard = shard_store.shard store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH # Create SignedShardBlock @@ -57,11 +58,11 @@ def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): beacon_block.body.shard_transitions = shard_transitions signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) - run_on_shard_block(spec, store, shard, shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) add_block_to_store(spec, store, signed_beacon_block) assert spec.get_head(store) == beacon_block.hash_tree_root() - assert spec.get_shard_head(store, shard) == shard_block.message.hash_tree_root() + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() @with_all_phases_except([PHASE0]) @@ -78,6 +79,7 @@ def test_basic(spec, state): committee_index = spec.CommitteeIndex(0) shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard_store = spec.get_forkchoice_shard_store(state, shard) - run_apply_shard_and_beacon(spec, state, store, shard, committee_index) - run_apply_shard_and_beacon(spec, state, store, shard, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) From fca1bbccb948b132fef22a9732c2f0a6aef0b2a4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 May 2020 10:33:23 +0800 Subject: [PATCH 04/30] Remove `get_filtered_shard_block_tree` --- specs/phase1/shard-fork-choice.md | 50 ++++--------------------------- 1 file changed, 5 insertions(+), 45 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 4b3f42194..b026aabfc 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -11,13 +11,11 @@ - [Introduction](#introduction) - [Fork choice](#fork-choice) - [Helpers](#helpers) - - [Extended `Store`](#extended-store) - - [Updated `get_forkchoice_store`](#updated-get_forkchoice_store) + - [`ShardStore`](#shardstore) + - [`get_forkchoice_shard_store`](#get_forkchoice_shard_store) - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) - [`get_shard_head`](#get_shard_head) - [`get_shard_ancestor`](#get_shard_ancestor) - - [`filter_shard_block_tree`](#filter_shard_block_tree) - - [`get_filtered_block_tree`](#get_filtered_block_tree) - [Handlers](#handlers) - [`on_shard_block`](#on_shard_block) @@ -41,7 +39,7 @@ class ShardStore: block_states: Dict[Root, ShardState] = field(default_factory=dict) ``` -#### Updated `get_forkchoice_shard_store` +#### `get_forkchoice_shard_store` ```python def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: @@ -72,16 +70,13 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro ```python def get_shard_head(store: Store, shard_store: ShardStore) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_shard_block_tree(store, shard_store) - # Execute the LMD-GHOST fork choice head_beacon_root = get_head(store) head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root while True: children = [ - root for root in blocks.keys() - if blocks[root].shard_parent_root == head_shard_root + root for root in shard_store.blocks.keys() + if shard_store.blocks[root].shard_parent_root == head_shard_root ] if len(children) == 0: return head_shard_root @@ -105,41 +100,6 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: return root ``` -#### `filter_shard_block_tree` - -```python -def filter_shard_block_tree(store: Store, - shard_store: ShardStore, - block_root: Root, - blocks: Dict[Root, ShardBlock]) -> bool: - block = shard_store.blocks[block_root] - children = [ - root for root in shard_store.blocks.keys() - if shard_store.blocks[root].shard_parent_root == block_root - ] - - if any(children): - filter_block_tree_result = [filter_shard_block_tree(store, shard_store, child, blocks) for child in children] - if any(filter_block_tree_result): - blocks[block_root] = block - return True - return False - - return False -``` - -#### `get_filtered_block_tree` - -```python -def get_filtered_shard_block_tree(store: Store, shard_store: ShardStore) -> Dict[Root, ShardBlock]: - shard = shard_store.shard - base_beacon_block_root = get_head(store) - base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root - blocks: Dict[Root, ShardBlock] = {} - filter_shard_block_tree(store, shard_store, base_shard_block_root, blocks) - return blocks -``` - ### Handlers #### `on_shard_block` From 79b1b4bdbe0c4822d4a2e197a7bf59930ffe69c1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 May 2020 10:52:45 +0800 Subject: [PATCH 05/30] Add `(shard, shard_root)` to `LatestMessage` --- specs/phase1/fork-choice.md | 32 ++++++++++++++++++- .../test/fork_choice/test_on_attestation.py | 13 ++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md index d8bf7fa09..f4e771ddb 100644 --- a/specs/phase1/fork-choice.md +++ b/specs/phase1/fork-choice.md @@ -10,6 +10,9 @@ - [Introduction](#introduction) - [Fork choice](#fork-choice) + - [Helpers](#helpers) + - [Extended `LatestMessage`](#extended-latestmessage) + - [Updated `update_latest_messages`](#updated-update_latest_messages) - [Handlers](#handlers) @@ -25,6 +28,33 @@ Due to the changes in the structure of `IndexedAttestation` in Phase 1, `on_atte The rest of the fork choice remains stable. +### Helpers + +#### Extended `LatestMessage` + +```python +@dataclass(eq=True, frozen=True) +class LatestMessage(object): + epoch: Epoch + root: Root + shard: Shard + shard_root: Root +``` + +#### Updated `update_latest_messages` + +```python +def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: + target = attestation.data.target + beacon_block_root = attestation.data.beacon_block_root + shard = get_shard(store.block_states[beacon_block_root], attestation) + for i in attesting_indices: + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage( + epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.head_shard_root + ) +``` + ### Handlers ```python @@ -49,4 +79,4 @@ def on_attestation(store: Store, attestation: Attestation) -> None: if attestation.aggregation_bits[i] ] update_latest_messages(store, attesting_indices, attestation) -``` \ No newline at end of file +``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 360c18ccd..1bce42ca3 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -18,18 +18,25 @@ def run_on_attestation(spec, state, store, attestation, valid=True): if spec.fork == PHASE0: sample_index = indexed_attestation.attesting_indices[0] + latest_message = spec.LatestMessage( + epoch=attestation.data.target.epoch, + root=attestation.data.beacon_block_root, + ) else: attesting_indices = [ index for i, index in enumerate(indexed_attestation.committee) if attestation.aggregation_bits[i] ] sample_index = attesting_indices[0] - assert ( - store.latest_messages[sample_index] == - spec.LatestMessage( + latest_message = spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, + shard=spec.get_shard(state, attestation), + shard_root=attestation.data.head_shard_root, ) + + assert ( + store.latest_messages[sample_index] == latest_message ) From 870ad8b921705cb658ebb0b88ff4e08bdeb54609 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 30 May 2020 03:56:56 +0800 Subject: [PATCH 06/30] Fix test --- .../pyspec/eth2spec/test/fork_choice/test_on_shard_head.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index f4b883f06..5b4205e91 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -30,6 +30,7 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + target_len_offset_slot = 1 shard_block = build_shard_block(spec, state, shard, body=body, signed=True) shard_blocks = [shard_block] @@ -38,17 +39,15 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) shard_transitions = build_shard_transitions_till_slot( spec, state, - shards=[shard, ], shard_blocks={shard: shard_blocks}, - target_len_offset_slot=1, + on_time_slot=state.slot + target_len_offset_slot, ) shard_transition = shard_transitions[shard] attestation = build_attestation_with_shard_transition( spec, state, - slot=state.slot, index=committee_index, - target_len_offset_slot=1, + on_time_slot=state.slot + target_len_offset_slot, shard_transition=shard_transition, ) From 142ba17451e05b0e6478796fe9bea9eb2de4c488 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 2 Jun 2020 18:08:28 +0800 Subject: [PATCH 07/30] PR review from Danny --- specs/phase1/shard-fork-choice.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index b026aabfc..867730203 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -59,7 +59,9 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro return Gwei(sum( state.validators[i].effective_balance for i in active_indices if ( - i in store.latest_messages and get_shard_ancestor( + i in store.latest_messages and + store.latest_messages[i].shard == shard_store.shard and + get_shard_ancestor( store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot ) == root ) @@ -71,12 +73,18 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro ```python def get_shard_head(store: Store, shard_store: ShardStore) -> Root: # Execute the LMD-GHOST fork choice + shard_blocks = shard_store.blocks head_beacon_root = get_head(store) - head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root + head_shard_state = store.block_states[head_beacon_root].shard_states[shard_store.shard] + head_shard_root = head_shard_state.latest_block_root while True: + # Find the valid child block roots children = [ root for root in shard_store.blocks.keys() - if shard_store.blocks[root].shard_parent_root == head_shard_root + if ( + shard_blocks[root].shard_parent_root == head_shard_root + and shard_blocks[root].slot > head_shard_state.slot + ) ] if len(children) == 0: return head_shard_root @@ -116,14 +124,15 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si assert shard_block.beacon_parent_root in store.block_states beacon_state = store.block_states[shard_block.beacon_parent_root] - # 3. Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert shard_block.slot > finalized_slot + # 3. Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) + finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] + finalized_shard_state = finalized_beacon_state.shard_states[shard] + assert shard_block.slot > finalized_shard_state.slot # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert ( - shard_block.beacon_parent_root == store.finalized_checkpoint.root - or get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root + get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root ) # Add new block to the store From 5c5cedd60d1c08583d8627e93db1b1b179f6b7f4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 3 Jun 2020 22:31:16 +0800 Subject: [PATCH 08/30] Apply PR feedback from Danny and Terence --- specs/phase1/shard-fork-choice.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 867730203..3b6bc5ac9 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -6,7 +6,7 @@ -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + - [Introduction](#introduction) - [Fork choice](#fork-choice) @@ -23,7 +23,7 @@ ## Introduction -This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. +This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. It assumes the [beacon chain fork choice spec](./fork-choice.md). ## Fork choice @@ -59,9 +59,9 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro return Gwei(sum( state.validators[i].effective_balance for i in active_indices if ( - i in store.latest_messages and - store.latest_messages[i].shard == shard_store.shard and - get_shard_ancestor( + i in store.latest_messages + and store.latest_messages[i].shard == shard_store.shard + and get_shard_ancestor( store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot ) == root ) @@ -116,20 +116,25 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: SignedShardBlock) -> None: shard_block = signed_shard_block.message shard = shard_store.shard - # 1. Check shard parent exists + + # Check shard + # TODO: check it in networking spec + assert shard_block.shard == shard + + # Check shard parent exists assert shard_block.shard_parent_root in shard_store.block_states pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] - # 2. Check beacon parent exists + # Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states beacon_state = store.block_states[shard_block.beacon_parent_root] - # 3. Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) + # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] finalized_shard_state = finalized_beacon_state.shard_states[shard] assert shard_block.slot > finalized_shard_state.slot - # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert ( get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root From e1981a7bfdeb2baf296cbfdb1f74169b56341883 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 01:00:52 +0800 Subject: [PATCH 09/30] `head_shard_root` -> `shard_head_root` --- specs/phase1/fork-choice.md | 2 +- specs/phase1/shard-fork-choice.md | 8 ++++---- .../eth2spec/test/fork_choice/test_on_attestation.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md index f4e771ddb..3f9fbdbfb 100644 --- a/specs/phase1/fork-choice.md +++ b/specs/phase1/fork-choice.md @@ -51,7 +51,7 @@ def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIn for i in attesting_indices: if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: store.latest_messages[i] = LatestMessage( - epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.head_shard_root + epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.shard_head_root ) ``` diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 3b6bc5ac9..fb98893ac 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -76,20 +76,20 @@ def get_shard_head(store: Store, shard_store: ShardStore) -> Root: shard_blocks = shard_store.blocks head_beacon_root = get_head(store) head_shard_state = store.block_states[head_beacon_root].shard_states[shard_store.shard] - head_shard_root = head_shard_state.latest_block_root + shard_head_root = head_shard_state.latest_block_root while True: # Find the valid child block roots children = [ root for root in shard_store.blocks.keys() if ( - shard_blocks[root].shard_parent_root == head_shard_root + shard_blocks[root].shard_parent_root == shard_head_root and shard_blocks[root].slot > head_shard_state.slot ) ] if len(children) == 0: - return head_shard_root + return shard_head_root # Sort by latest attesting balance with ties broken lexicographically - head_shard_root = max( + shard_head_root = max( children, key=lambda root: (get_shard_latest_attesting_balance(store, shard_store, root), root) ) ``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 04d2588d9..a5334c5c7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -32,7 +32,7 @@ def run_on_attestation(spec, state, store, attestation, valid=True): epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, shard=spec.get_shard(state, attestation), - shard_root=attestation.data.head_shard_root, + shard_root=attestation.data.shard_head_root, ) assert ( From c0108afe7713ac10542c478b8d07514bdad42c3b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:06:04 +0800 Subject: [PATCH 10/30] Use shard_block.slot to get seed for proposer selection --- specs/phase1/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 484b24cf1..fccb93f55 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -538,7 +538,8 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) - r = bytes_to_int(get_seed(beacon_state, get_current_epoch(beacon_state), DOMAIN_SHARD_COMMITTEE)[:8]) + epoch = compute_epoch_at_slot(slot) + r = bytes_to_int(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE)[:8]) return committee[r % len(committee)] ``` From d3445217416ad3a8f73dbf45e4669970d063682a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:30:13 +0800 Subject: [PATCH 11/30] Bugfix: should set `shard` for empty proposal --- specs/phase1/shard-transition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index e6221a980..5e8616568 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -221,7 +221,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, validate_signature=validate_signature, ) if len(choices) == 0: - block = ShardBlock(slot=slot) + block = ShardBlock(slot=slot, shard=shard) proposal = SignedShardBlock(message=block) elif len(choices) == 1: proposal = choices[0] From 26aae40941aac98d731655a475e3085e777c0110 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:31:53 +0800 Subject: [PATCH 12/30] Use epoch of the shard_block.slot for generating seed --- specs/phase1/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 484b24cf1..fccb93f55 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -538,7 +538,8 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) - r = bytes_to_int(get_seed(beacon_state, get_current_epoch(beacon_state), DOMAIN_SHARD_COMMITTEE)[:8]) + epoch = compute_epoch_at_slot(slot) + r = bytes_to_int(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE)[:8]) return committee[r % len(committee)] ``` From 376c83619024caee0dfd5f1c7da552ca78cedc10 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 4 Jun 2020 12:49:22 +1000 Subject: [PATCH 13/30] Clarify proposer slashing gossip conditions --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 9949db66f..767748e89 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -263,7 +263,7 @@ Additional global topics are used to propagate lower frequency validator message - _[IGNORE]_ The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. - _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. The following validations MUST pass before forwarding the `proposer_slashing` on to the network - - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.index`. + - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.header_1.proposer_index`. - _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. - _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). From 8afb93f5a36adac5ef312ee3fa4678632df2b6c4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 11:19:04 +0800 Subject: [PATCH 14/30] Add `shard_block.slot` to seed --- specs/phase1/beacon-chain.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index fccb93f55..4719d2e6f 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -537,9 +537,13 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: - committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) + """ + Return the proposer's index of shard block at ``slot``. + """ epoch = compute_epoch_at_slot(slot) - r = bytes_to_int(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE)[:8]) + committee = get_shard_committee(beacon_state, epoch, shard) + seed = hash(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE) + int_to_bytes(slot, length=8)) + r = bytes_to_int(seed[:8]) return committee[r % len(committee)] ``` From c2b7ff7422e449dc2486261adad6196259015579 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 4 Jun 2020 16:13:49 +1000 Subject: [PATCH 15/30] Re-clarify proposer slashing check Fixes a typo from #1871 --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 767748e89..bdf0919e0 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -263,7 +263,7 @@ Additional global topics are used to propagate lower frequency validator message - _[IGNORE]_ The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. - _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. The following validations MUST pass before forwarding the `proposer_slashing` on to the network - - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.header_1.proposer_index`. + - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.signed_header_1.message.proposer_index`. - _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. - _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). From 7e44456be543193b3c4848faead8ea9f7baa1583 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 5 Jun 2020 09:50:49 -0600 Subject: [PATCH 16/30] mod compute_subnet_for_attestation to be usable for lookahead --- specs/phase0/p2p-interface.md | 4 ++-- specs/phase0/validator.md | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index bdf0919e0..6c4cdd2a6 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -275,7 +275,7 @@ Additional global topics are used to propagate lower frequency validator message Attestation subnets are used to propagate unaggregated attestations to subsections of the network. Their `Name`s are: - `beacon_attestation_{subnet_id}` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation) == subnet_id`). + - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.index) == subnet_id`). - _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len(get_attesting_indices(state, attestation.data, attestation.aggregation_bits)) == 1`). - _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. @@ -286,7 +286,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. The correct subnet for an attestation can be calculated with `compute_subnet_for_attestation`. `beacon_attestation_{subnet_id}` topics, are rotated through throughout the epoch in a similar fashion to rotating through shards in committees in Phase 1. -Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` as `Attestation`s. +Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.index)}` as `Attestation`s. Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `AggregateAndProof`s. diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 5e8ddc977..35c52f346 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -199,8 +199,8 @@ The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahe Specifically a validator should: * Call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. -* Find peers of the pubsub topic `committee_index{committee_index % ATTESTATION_SUBNET_COUNT}_beacon_attestation`. - * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][committee_index % ATTESTATION_SUBNET_COUNT] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field. +* Find peers of the pubsub topic `beacon_attestation_{compute_subnet_for_attestation(state, slot, committee_index)}`. + * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][compute_subnet_for_attestation(state, slot, committee_index)] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field. * If the validator is assigned to be an aggregator for the slot (see `is_aggregator()`), then subscribe to the topic. *Note*: If the validator is _not_ assigned to be an aggregator, the validator only needs sufficient number of peers on the topic to be able to publish messages. The validator does not need to _subscribe_ and listen to all messages on the topic. @@ -425,18 +425,18 @@ def get_attestation_signature(state: BeaconState, attestation_data: AttestationD #### Broadcast attestation -Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` pubsub topic. +Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.committee_index)}` pubsub topic. ```python -def compute_subnet_for_attestation(state: BeaconState, attestation: Attestation) -> uint64: +def compute_subnet_for_attestation(state: BeaconState, slot: Slot, committee_index: CommitteeIndex) -> uint64: """ Compute the correct subnet for an attestation for Phase 0. Note, this mimics expected Phase 1 behavior where attestations will be mapped to their shard subnet. """ - slots_since_epoch_start = attestation.data.slot % SLOTS_PER_EPOCH - committees_since_epoch_start = get_committee_count_at_slot(state, attestation.data.slot) * slots_since_epoch_start + slots_since_epoch_start = slot % SLOTS_PER_EPOCH + committees_since_epoch_start = get_committee_count_at_slot(state, slot) * slots_since_epoch_start - return (committees_since_epoch_start + attestation.data.index) % ATTESTATION_SUBNET_COUNT + return (committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT ``` ### Attestation aggregation From c9a53b8039cb2eada12cbb09ca1cd74509356549 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:33:01 +0800 Subject: [PATCH 17/30] WIP test case --- .../test/fork_choice/test_on_shard_head.py | 91 ++++++++++++------- .../eth2spec/test/helpers/shard_block.py | 26 ++++-- 2 files changed, 78 insertions(+), 39 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 5b4205e91..1151d18d7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -5,6 +5,7 @@ from eth2spec.test.helpers.shard_block import ( build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, + get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block @@ -24,44 +25,65 @@ def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index): +def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): shard = shard_store.shard + committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) + has_shard_committee = committee_index is not None store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH # Create SignedShardBlock - body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - target_len_offset_slot = 1 - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) - shard_blocks = [shard_block] + # Check offsets + temp_state = state.copy() + next_slot(spec, temp_state) + offset_slots = spec.get_offset_slots(temp_state, shard) + if state.slot in offset_slots: + # Build block + body = b'\x56' * 4 + shard_head_root = spec.get_shard_head(store, shard_store) + shard_parent_state = shard_store.block_states[shard_head_root] + assert shard_parent_state.slot != state.slot + shard_block = build_shard_block( + spec, state, shard, + shard_parent_state=shard_parent_state, slot=state.slot, body=body, signed=True + ) + shard_blocks_buffer.append(shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) # Attester creates `attestation` - # Use temporary next state to get ShardTransition of shard block - shard_transitions = build_shard_transitions_till_slot( - spec, - state, - shard_blocks={shard: shard_blocks}, - on_time_slot=state.slot + target_len_offset_slot, - ) - shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( - spec, - state, - index=committee_index, - on_time_slot=state.slot + target_len_offset_slot, - shard_transition=shard_transition, - ) + if has_shard_committee and len(shard_blocks_buffer) > 0: + # Use temporary next state to get ShardTransition of shard block + shard_transitions = build_shard_transitions_till_slot( + spec, + state, + shard_blocks={shard: shard_blocks_buffer}, + on_time_slot=state.slot + 1, + ) + shard_transition = shard_transitions[shard] + + attestation = build_attestation_with_shard_transition( + spec, + state, + index=committee_index, + on_time_slot=state.slot + 1, + shard_transition=shard_transition, + ) + assert attestation.data.slot == state.slot + assert spec.get_shard(state, attestation) == shard + beacon_block.body.attestations = [attestation] + beacon_block.body.shard_transitions = shard_transitions - # Propose beacon block at slot - beacon_block = build_empty_block(spec, state, slot=state.slot + 1) - beacon_block.body.attestations = [attestation] - beacon_block.body.shard_transitions = shard_transitions signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) - run_on_shard_block(spec, store, shard_store, shard_block) add_block_to_store(spec, store, signed_beacon_block) - assert spec.get_head(store) == beacon_block.hash_tree_root() - assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + if has_shard_committee: + shard_blocks_buffer = [] # clear buffer + + return has_shard_committee, shard_blocks_buffer @with_all_phases_except([PHASE0]) @@ -69,16 +91,19 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) def test_basic(spec, state): spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking state = spec.upgrade_to_phase1(state) - next_slot(spec, state) # Initialization store = spec.get_forkchoice_store(state) anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root - committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.Shard(1) shard_store = spec.get_forkchoice_shard_store(state, shard) - - run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) - run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) + shard_block_count = 2 + shard_blocks_buffer = [] + while shard_block_count > 0: + has_shard_committee, shard_blocks_buffer = run_apply_shard_and_beacon( + spec, state, store, shard_store, shard_blocks_buffer + ) + if has_shard_committee: + shard_block_count -= 1 diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 58efada83..410213edd 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -23,19 +23,24 @@ def build_shard_block(spec, shard, slot=None, body=None, + shard_parent_state=None, signed=False): - shard_state = beacon_state.shard_states[shard] + if shard_parent_state is None: + shard_parent_state = beacon_state.shard_states[shard] + if slot is None: - slot = shard_state.slot + 1 + slot = shard_parent_state.slot + 1 if body is None: body = b'\x56' * 128 - proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) + temp_state = beacon_state.copy() + transition_to(spec, temp_state, slot) beacon_state, beacon_parent_root = get_state_and_beacon_parent_root_at_slot(spec, beacon_state, slot) - + assert beacon_state == temp_state + proposer_index = spec.get_shard_proposer_index(temp_state, slot, shard) block = spec.ShardBlock( - shard_parent_root=shard_state.latest_block_root, + shard_parent_root=shard_parent_state.latest_block_root, beacon_parent_root=beacon_parent_root, slot=slot, shard=shard, @@ -59,7 +64,6 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): for shard, blocks in shard_blocks.items(): offset_slots = spec.get_offset_slots(temp_state, shard) len_offset_slots = len(offset_slots) - assert len_offset_slots == on_time_slot - state.shard_states[shard].slot - 1 shard_transition = spec.get_shard_transition(temp_state, shard, blocks) if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() @@ -84,3 +88,13 @@ def build_attestation_with_shard_transition(spec, state, index, on_time_slot, sh if shard_transition is not None: assert attestation.data.shard_transition_root == shard_transition.hash_tree_root() return attestation + + +def get_committee_index_of_shard(spec, state, slot, shard): # Optional[CommitteeIndex] + active_shard_count = spec.get_active_shard_count(state) + committee_count = spec.get_committee_count_at_slot(state, slot) + start_shard = spec.get_start_shard(state, slot) + for committee_index in range(committee_count): + if (start_shard + committee_index) % active_shard_count == shard: + return committee_index + return None From 727353c054f259aa81e7738099fc055722e9734c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 18:39:27 +0800 Subject: [PATCH 18/30] Verify shard_block.slot fits the expected offset_slots --- specs/phase1/shard-fork-choice.md | 9 +++++---- specs/phase1/shard-transition.md | 3 +++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index fb98893ac..b60edd948 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -127,7 +127,6 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si # Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states - beacon_state = store.block_states[shard_block.beacon_parent_root] # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] @@ -144,9 +143,11 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si shard_store.blocks[hash_tree_root(shard_block)] = shard_block # Check the block is valid and compute the post-state - verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) - verify_shard_block_signature(beacon_state, signed_shard_block) - post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) + beacon_head_root = get_head(store) + beacon_head_state = store.block_states[beacon_head_root] + assert verify_shard_block_message(beacon_head_state, pre_shard_state, shard_block, shard_block.slot, shard) + assert verify_shard_block_signature(beacon_head_state, signed_shard_block) + post_state = get_post_shard_state(beacon_head_state, pre_shard_state, shard_block) # Add new state for this block to the store shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 5e8616568..6c4f652d2 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -50,6 +50,9 @@ def verify_shard_block_message(beacon_state: BeaconState, shard: Shard) -> bool: assert block.shard_parent_root == shard_state.latest_block_root assert block.slot == slot + next_slot = Slot(beacon_state.slot + 1) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), next_slot) + assert slot in offset_slots assert block.shard == shard assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE From f8597d296545c6711d1c7cf8fe25d5dbb5b1987b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 20:31:54 +0800 Subject: [PATCH 19/30] Add `get_pendings_shard_blocks` --- specs/phase1/shard-fork-choice.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index b60edd948..5bc2cbe4e 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -16,6 +16,7 @@ - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) - [`get_shard_head`](#get_shard_head) - [`get_shard_ancestor`](#get_shard_ancestor) + - [`get_pendings_shard_blocks`](#get_pendings_shard_blocks) - [Handlers](#handlers) - [`on_shard_block`](#on_shard_block) @@ -108,6 +109,31 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: return root ``` +#### `get_pendings_shard_blocks` + +```python +def get_pendings_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: + """ + Return the shard blocks branch that from shard head to beacon head. + """ + shard = shard_store.shard + + beacon_head_root = get_head(store) + beacon_head_state = store.block_states[beacon_head_root] + latest_shard_block_root = beacon_head_state.shard_states[shard].latest_block_root + + shard_head_root = get_shard_head(store, shard_store) + root = shard_head_root + shard_blocks = [] + while root != latest_shard_block_root: + shard_block = shard_store.blocks[root] + shard_blocks.append(shard_block) + root = shard_block.shard_parent_root + + shard_blocks.reverse() + return shard_blocks +``` + ### Handlers #### `on_shard_block` From ab42eee4c04818d58fbcea5ba83d1c5890d0c3b8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 20:32:31 +0800 Subject: [PATCH 20/30] Update shard fork choice rule to be able to handle mainnet config --- .../test/fork_choice/test_on_shard_head.py | 97 ++++++++++++------- 1 file changed, 62 insertions(+), 35 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 1151d18d7..7e94ddd8e 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.test.context import spec_state_test, with_all_phases_except, PHASE0 +from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls from eth2spec.test.helpers.shard_block import ( build_attestation_with_shard_transition, build_shard_block, @@ -8,7 +8,7 @@ from eth2spec.test.helpers.shard_block import ( get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root -from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block +from eth2spec.test.helpers.state import state_transition_and_sign_block from eth2spec.test.helpers.block import build_empty_block @@ -25,35 +25,51 @@ def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): +def apply_shard_block(spec, store, shard_store, beacon_head_state, shard_blocks_buffer): shard = shard_store.shard - committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) - has_shard_committee = committee_index is not None + body = b'\x56' * 4 + shard_head_root = spec.get_shard_head(store, shard_store) + shard_parent_state = shard_store.block_states[shard_head_root] + assert shard_parent_state.slot != beacon_head_state.slot + shard_block = build_shard_block( + spec, beacon_head_state, shard, + shard_parent_state=shard_parent_state, slot=beacon_head_state.slot, body=body, signed=True + ) + shard_blocks_buffer.append(shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + +def check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer): + pending_shard_blocks = [ + spec.SignedShardBlock(message=b) + for b in spec.get_pendings_shard_blocks(store, shard_store) + ] + assert pending_shard_blocks == shard_blocks_buffer + + +def is_in_offset_sets(spec, beacon_head_state, shard): + offset_slots = spec.compute_offset_slots( + beacon_head_state.shard_states[shard].slot, beacon_head_state.slot + 1 + ) + return beacon_head_state.slot in offset_slots + + +def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH - # Create SignedShardBlock - # Check offsets - temp_state = state.copy() - next_slot(spec, temp_state) - offset_slots = spec.get_offset_slots(temp_state, shard) - if state.slot in offset_slots: - # Build block - body = b'\x56' * 4 - shard_head_root = spec.get_shard_head(store, shard_store) - shard_parent_state = shard_store.block_states[shard_head_root] - assert shard_parent_state.slot != state.slot - shard_block = build_shard_block( - spec, state, shard, - shard_parent_state=shard_parent_state, slot=state.slot, body=body, signed=True - ) - shard_blocks_buffer.append(shard_block) - run_on_shard_block(spec, store, shard_store, shard_block) - assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + shard = shard_store.shard + committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) + has_shard_committee = committee_index is not None # has committee of `shard` at this slot + # On beacon blocks at `state.slot + 1` beacon_block = build_empty_block(spec, state, slot=state.slot + 1) - # Attester creates `attestation` + # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block if has_shard_committee and len(shard_blocks_buffer) > 0: + # Sanity check `get_pendings_shard_blocks` function + check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) + # Use temporary next state to get ShardTransition of shard block shard_transitions = build_shard_transitions_till_slot( spec, @@ -62,7 +78,6 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buf on_time_slot=state.slot + 1, ) shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( spec, state, @@ -75,35 +90,47 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buf beacon_block.body.attestations = [attestation] beacon_block.body.shard_transitions = shard_transitions - signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + # Clear buffer + shard_blocks_buffer.clear() + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) add_block_to_store(spec, store, signed_beacon_block) assert spec.get_head(store) == beacon_block.hash_tree_root() - if has_shard_committee: - shard_blocks_buffer = [] # clear buffer + # On shard block at updated `state.slot` + if is_in_offset_sets(spec, state, shard): + # The created shard block would be appended to `shard_blocks_buffer` + apply_shard_block(spec, store, shard_store, state, shard_blocks_buffer) - return has_shard_committee, shard_blocks_buffer + return has_shard_committee @with_all_phases_except([PHASE0]) @spec_state_test +@never_bls # Set to never_bls for testing `check_pending_shard_blocks` def test_basic(spec, state): - spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking + spec.PHASE_1_GENESIS_SLOT = 0 # NOTE: mock genesis slot here state = spec.upgrade_to_phase1(state) + shard = spec.Shard(1) # Initialization store = spec.get_forkchoice_store(state) anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root - shard = spec.Shard(1) shard_store = spec.get_forkchoice_shard_store(state, shard) - shard_block_count = 2 + shard_head_root = spec.get_shard_head(store, shard_store) + assert shard_head_root == state.shard_states[shard].latest_block_root + assert shard_store.block_states[shard_head_root].slot == 1 + assert shard_store.block_states[shard_head_root] == state.shard_states[shard] + + # For mainnet config, it's possible that only one committee of `shard` per epoch. + # we set this counter to test more rounds. + shard_committee_counter = 2 shard_blocks_buffer = [] - while shard_block_count > 0: - has_shard_committee, shard_blocks_buffer = run_apply_shard_and_beacon( + while shard_committee_counter > 0: + has_shard_committee = apply_shard_and_beacon( spec, state, store, shard_store, shard_blocks_buffer ) if has_shard_committee: - shard_block_count -= 1 + shard_committee_counter -= 1 From 6f9c290bfb97240a9f41370b026d066ed8df6917 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 20:40:09 +0800 Subject: [PATCH 21/30] Add TODO flag of latest message --- specs/phase1/shard-fork-choice.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 5bc2cbe4e..a474c8214 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -61,6 +61,8 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro state.validators[i].effective_balance for i in active_indices if ( i in store.latest_messages + # TODO: check the latest message logic: currently, validator's previous vote of another shard + # would be ignored once their newer vote is accepted. Check if it makes sense. and store.latest_messages[i].shard == shard_store.shard and get_shard_ancestor( store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot From a154d0c22b9a3175e74383e4df53e5b09f6df0fd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 21:24:17 +0800 Subject: [PATCH 22/30] Fix typo --- specs/phase1/shard-fork-choice.md | 6 +++--- .../pyspec/eth2spec/test/fork_choice/test_on_shard_head.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index a474c8214..427b72784 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -16,7 +16,7 @@ - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) - [`get_shard_head`](#get_shard_head) - [`get_shard_ancestor`](#get_shard_ancestor) - - [`get_pendings_shard_blocks`](#get_pendings_shard_blocks) + - [`get_pending_shard_blocks`](#get_pending_shard_blocks) - [Handlers](#handlers) - [`on_shard_block`](#on_shard_block) @@ -111,10 +111,10 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: return root ``` -#### `get_pendings_shard_blocks` +#### `get_pending_shard_blocks` ```python -def get_pendings_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: +def get_pending_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: """ Return the shard blocks branch that from shard head to beacon head. """ diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 7e94ddd8e..1ba15968f 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -43,7 +43,7 @@ def apply_shard_block(spec, store, shard_store, beacon_head_state, shard_blocks_ def check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer): pending_shard_blocks = [ spec.SignedShardBlock(message=b) - for b in spec.get_pendings_shard_blocks(store, shard_store) + for b in spec.get_pending_shard_blocks(store, shard_store) ] assert pending_shard_blocks == shard_blocks_buffer @@ -67,7 +67,7 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block if has_shard_committee and len(shard_blocks_buffer) > 0: - # Sanity check `get_pendings_shard_blocks` function + # Sanity check `get_pending_shard_blocks` function check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) # Use temporary next state to get ShardTransition of shard block @@ -129,6 +129,7 @@ def test_basic(spec, state): shard_committee_counter = 2 shard_blocks_buffer = [] while shard_committee_counter > 0: + print(f'state.slot', state.slot) has_shard_committee = apply_shard_and_beacon( spec, state, store, shard_store, shard_blocks_buffer ) From 2d4788fe7d032f473fe5d60d732c40505ad8f485 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 5 Jun 2020 16:19:25 +0800 Subject: [PATCH 23/30] Fix `verify_shard_block_message` Add check for `block.beacon_parent_root` per Terence's suggestion Update `get_shard_transition` 1. Disable verification: it will be fix in v-guide 2. Use `on_time_slot` to compute offset_slots Rework tests --- specs/phase1/shard-fork-choice.md | 11 ++-- specs/phase1/shard-transition.md | 66 +++++++++++-------- .../test/fork_choice/test_on_shard_head.py | 19 ++---- .../eth2spec/test/helpers/attestations.py | 2 +- .../eth2spec/test/helpers/shard_block.py | 36 +++------- .../test/helpers/shard_transitions.py | 5 +- .../test_process_shard_transition.py | 31 ++++----- .../test/phase_1/sanity/test_blocks.py | 56 ++++++---------- 8 files changed, 94 insertions(+), 132 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 427b72784..844dbfb86 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -151,10 +151,11 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si # Check shard parent exists assert shard_block.shard_parent_root in shard_store.block_states - pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] + shard_parent_state = shard_store.block_states[shard_block.shard_parent_root] # Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states + beacon_parent_state = store.block_states[shard_block.beacon_parent_root] # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] @@ -171,11 +172,9 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si shard_store.blocks[hash_tree_root(shard_block)] = shard_block # Check the block is valid and compute the post-state - beacon_head_root = get_head(store) - beacon_head_state = store.block_states[beacon_head_root] - assert verify_shard_block_message(beacon_head_state, pre_shard_state, shard_block, shard_block.slot, shard) - assert verify_shard_block_signature(beacon_head_state, signed_shard_block) - post_state = get_post_shard_state(beacon_head_state, pre_shard_state, shard_block) + assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block, shard_block.slot, shard) + assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) + post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) # Add new state for this block to the store shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 6c4f652d2..8d75879f5 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -30,7 +30,7 @@ This document describes the shard transition function and fraud proofs as part o ### Misc ```python -def compute_shard_transition_digest(beacon_state: BeaconState, +def compute_shard_transition_digest(beacon_parent_state: BeaconState, shard_state: ShardState, beacon_parent_root: Root, shard_body_root: Root) -> Bytes32: @@ -44,17 +44,33 @@ def compute_shard_transition_digest(beacon_state: BeaconState, ```python def verify_shard_block_message(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, block: ShardBlock, slot: Slot, - shard: Shard) -> bool: - assert block.shard_parent_root == shard_state.latest_block_root - assert block.slot == slot - next_slot = Slot(beacon_state.slot + 1) + shard: Shard, + beacon_parent_slot: Slot=None) -> bool: + # Check `shard_parent_root` field + assert block.shard_parent_root == shard_parent_state.latest_block_root + # Check `beacon_parent_root` field + if beacon_parent_slot is None: + beacon_parent_slot = beacon_state.slot + if beacon_parent_slot == beacon_state.slot: + beacon_parent_block_header = beacon_state.latest_block_header.copy() + if beacon_parent_block_header.state_root == Root(): + beacon_parent_block_header.state_root = hash_tree_root(beacon_state) + beacon_parent_root = hash_tree_root(beacon_parent_block_header) + else: + beacon_parent_root = get_block_root_at_slot(beacon_state, beacon_parent_slot) + assert block.beacon_parent_root == beacon_parent_root + # Check `slot` field + next_slot = Slot(slot + 1) offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), next_slot) assert slot in offset_slots + # Check `shard` field assert block.shard == shard + # Check `proposer_index` field assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) + # Check `body` field assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE return True ``` @@ -177,7 +193,7 @@ def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[ ```python def get_proposal_choices_at_slot(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, slot: Slot, shard: Shard, shard_blocks: Sequence[SignedShardBlock], @@ -188,25 +204,16 @@ def get_proposal_choices_at_slot(beacon_state: BeaconState, """ choices = [] shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] + shard_state = shard_parent_state.copy() for block in shard_blocks_at_slot: - try: - # Verify block message and signature - # TODO these validations should have been checked upon receiving shard blocks. - assert verify_shard_block_message(beacon_state, shard_state, block.message, slot, shard) - if validate_signature: - assert verify_shard_block_signature(beacon_state, block) - - shard_state = get_post_shard_state(beacon_state, shard_state, block.message) - except Exception: - pass # TODO: throw error in the test helper - else: - choices.append(block) + shard_state = get_post_shard_state(beacon_state, shard_state, block.message) + choices.append(block) return choices ``` ```python def get_proposal_at_slot(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, slot: Shard, shard: Shard, shard_blocks: Sequence[SignedShardBlock], @@ -217,7 +224,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, """ choices = get_proposal_choices_at_slot( beacon_state=beacon_state, - shard_state=shard_state, + shard_parent_state=shard_parent_state, slot=slot, shard=shard, shard_blocks=shard_blocks, @@ -232,7 +239,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, proposal = get_winning_proposal(beacon_state, choices) # Apply state transition - shard_state = get_post_shard_state(beacon_state, shard_state, proposal.message) + shard_state = get_post_shard_state(beacon_state, shard_parent_state, proposal.message) return proposal, shard_state ``` @@ -242,15 +249,17 @@ def get_shard_state_transition_result( beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock], + on_time_slot: Slot, validate_signature: bool=True, ) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: proposals = [] shard_states = [] shard_state = beacon_state.shard_states[shard] - for slot in get_offset_slots(beacon_state, shard): + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) + for slot in offset_slots: proposal, shard_state = get_proposal_at_slot( beacon_state=beacon_state, - shard_state=shard_state, + shard_parent_state=shard_state, slot=slot, shard=shard, shard_blocks=shard_blocks, @@ -271,9 +280,12 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y ```python def get_shard_transition(beacon_state: BeaconState, shard: Shard, - shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: - offset_slots = get_offset_slots(beacon_state, shard) - proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks) + shard_blocks: Sequence[SignedShardBlock], + on_time_slot: Slot) -> ShardTransition: + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) + proposals, shard_states, shard_data_roots = get_shard_state_transition_result( + beacon_state, shard, shard_blocks, on_time_slot + ) shard_block_lengths = [] proposer_signatures = [] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 1ba15968f..ca79cfd23 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -1,8 +1,8 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, get_committee_index_of_shard, @@ -25,15 +25,15 @@ def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def apply_shard_block(spec, store, shard_store, beacon_head_state, shard_blocks_buffer): +def apply_shard_block(spec, store, shard_store, beacon_parent_state, shard_blocks_buffer): shard = shard_store.shard body = b'\x56' * 4 shard_head_root = spec.get_shard_head(store, shard_store) shard_parent_state = shard_store.block_states[shard_head_root] - assert shard_parent_state.slot != beacon_head_state.slot + assert shard_parent_state.slot != beacon_parent_state.slot shard_block = build_shard_block( - spec, beacon_head_state, shard, - shard_parent_state=shard_parent_state, slot=beacon_head_state.slot, body=body, signed=True + spec, beacon_parent_state, shard, + shard_parent_state=shard_parent_state, slot=beacon_parent_state.slot, body=body, signed=True ) shard_blocks_buffer.append(shard_block) run_on_shard_block(spec, store, shard_store, shard_block) @@ -62,30 +62,26 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) has_shard_committee = committee_index is not None # has committee of `shard` at this slot - # On beacon blocks at `state.slot + 1` beacon_block = build_empty_block(spec, state, slot=state.slot + 1) # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block if has_shard_committee and len(shard_blocks_buffer) > 0: # Sanity check `get_pending_shard_blocks` function check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) - # Use temporary next state to get ShardTransition of shard block shard_transitions = build_shard_transitions_till_slot( spec, state, shard_blocks={shard: shard_blocks_buffer}, - on_time_slot=state.slot + 1, ) shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( + attestation = get_valid_on_time_attestation( spec, state, index=committee_index, - on_time_slot=state.slot + 1, shard_transition=shard_transition, + signed=False, ) - assert attestation.data.slot == state.slot assert spec.get_shard(state, attestation) == shard beacon_block.body.attestations = [attestation] beacon_block.body.shard_transitions = shard_transitions @@ -129,7 +125,6 @@ def test_basic(spec, state): shard_committee_counter = 2 shard_blocks_buffer = [] while shard_committee_counter > 0: - print(f'state.slot', state.slot) has_shard_committee = apply_shard_and_beacon( spec, state, store, shard_store, shard_blocks_buffer ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 1372b0654..106069dd6 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -87,7 +87,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t if on_time: temp_state = state.copy() next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[]) + shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[], on_time_slot=slot + 1) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 410213edd..0a2ed67d2 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -1,6 +1,4 @@ -from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import get_state_and_beacon_parent_root_at_slot -from eth2spec.test.helpers.state import transition_to from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls @@ -34,11 +32,8 @@ def build_shard_block(spec, if body is None: body = b'\x56' * 128 - temp_state = beacon_state.copy() - transition_to(spec, temp_state, slot) beacon_state, beacon_parent_root = get_state_and_beacon_parent_root_at_slot(spec, beacon_state, slot) - assert beacon_state == temp_state - proposer_index = spec.get_shard_proposer_index(temp_state, slot, shard) + proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) block = spec.ShardBlock( shard_parent_root=shard_parent_state.latest_block_root, beacon_parent_root=beacon_parent_root, @@ -57,14 +52,17 @@ def build_shard_block(spec, return signed_block -def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot) +def build_shard_transitions_till_slot(spec, parent_beacon_state, shard_blocks): shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS + on_time_slot = parent_beacon_state.slot + 1 for shard, blocks in shard_blocks.items(): - offset_slots = spec.get_offset_slots(temp_state, shard) + offset_slots = spec.compute_offset_slots( + spec.get_latest_slot_for_shard(parent_beacon_state, shard), + on_time_slot, + ) len_offset_slots = len(offset_slots) - shard_transition = spec.get_shard_transition(temp_state, shard, blocks) + shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks, on_time_slot) + if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() assert shard_transition.shard_states[len_offset_slots - 1].latest_block_root == shard_block_root @@ -74,22 +72,6 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): return shard_transitions -def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot - 1) - attestation = get_valid_on_time_attestation( - spec, - temp_state, - index=index, - shard_transition=shard_transition, - signed=True, - ) - assert attestation.data.slot == temp_state.slot - if shard_transition is not None: - assert attestation.data.shard_transition_root == shard_transition.hash_tree_root() - return attestation - - def get_committee_index_of_shard(spec, state, slot, shard): # Optional[CommitteeIndex] active_shard_count = spec.get_active_shard_count(state) committee_count = spec.get_committee_count_at_slot(state, slot) diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index abb5e7278..8e62b2f27 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -1,5 +1,4 @@ from eth2spec.test.context import expect_assertion_error -from eth2spec.test.helpers.state import transition_to def run_shard_transitions_processing(spec, state, shard_transitions, attestations, valid=True): @@ -37,7 +36,5 @@ def get_shard_transition_of_committee(spec, state, committee_index, slot=None, s slot = state.slot shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - temp_state = state.copy() - transition_to(spec, temp_state, slot + 1) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=shard_blocks) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks, on_time_slot=slot + 1) return shard_transition diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index 00ffbe0a8..dab4973da 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -2,71 +2,64 @@ from eth2spec.test.context import ( PHASE0, with_all_phases_except, spec_state_test, - always_bls, ) +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_transitions import run_shard_transitions_processing from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, ) -from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot +from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot, next_slot def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): state = transition_to_valid_shard_slot(spec, state) - init_slot = state.slot committee_index = spec.CommitteeIndex(0) shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 + transition_to(spec, state, state.slot + target_len_offset_slot) + assert state.shard_states[shard].slot == state.slot - target_len_offset_slot - 1 # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks = [shard_block] - # Create a shard_transitions that would be included at beacon block `state.slot + target_len_offset_slot` shard_transitions = build_shard_transitions_till_slot( spec, state, shard_blocks={shard: shard_blocks}, - on_time_slot=state.slot + target_len_offset_slot, ) shard_transition = shard_transitions[shard] - # Create an attestation that would be included at beacon block `state.slot + target_len_offset_slot` - attestation = build_attestation_with_shard_transition( + attestation = get_valid_on_time_attestation( spec, state, index=committee_index, - on_time_slot=state.slot + target_len_offset_slot, shard_transition=shard_transition, + signed=False, ) + next_slot(spec, state) pre_gasprice = state.shard_states[shard].gasprice - - transition_to(spec, state, state.slot + target_len_offset_slot) pre_shard_state = state.shard_states[shard] - yield from run_shard_transitions_processing(spec, state, shard_transitions, [attestation], valid=valid) if valid: - # After state transition, - assert state.slot == init_slot + target_len_offset_slot shard_state = state.shard_states[shard] assert shard_state != pre_shard_state assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] - + assert shard_state.latest_block_root == shard_block.message.hash_tree_root() if target_len_offset_slot == 1: assert shard_state.gasprice > pre_gasprice @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_basic_crosslinks(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=1, valid=True) @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_multiple_offset_slots(spec, state): - yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=3, valid=True) + # NOTE: this test is only for full crosslink (minimal config), not for mainnet + yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=2, valid=True) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 0175bd40d..828cd19d7 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -4,37 +4,40 @@ from eth2spec.test.context import ( PHASE0, with_all_phases_except, spec_state_test, - always_bls, ) +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, ) -from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot +from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to -def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index, valid=True): - shard_transitions = build_shard_transitions_till_slot( - spec, state, shard_blocks, on_time_slot=state.slot + target_len_offset_slot - ) +def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard, valid=True): + transition_to(spec, state, state.slot + target_len_offset_slot) + + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) + shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} + + shard_transitions = build_shard_transitions_till_slot(spec, state, shard_blocks) attestations = [ - build_attestation_with_shard_transition( + get_valid_on_time_attestation( spec, state, - on_time_slot=state.slot + target_len_offset_slot, index=committee_index, shard_transition=shard_transitions[shard], + signed=True, ) for shard in shard_blocks.keys() ] - # Propose beacon block at slot `x + 1` - beacon_block = build_empty_block(spec, state, slot=state.slot + target_len_offset_slot) + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) beacon_block.body.attestations = attestations beacon_block.body.shard_transitions = shard_transitions + pre_gasprice = state.shard_states[shard].gasprice pre_shard_states = state.shard_states.copy() yield 'pre', state.copy() yield 'block', beacon_block @@ -52,17 +55,18 @@ def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_off assert post_shard_state == shard_transitions[shard].shard_states[ len(shard_transitions[shard].shard_states) - 1 ] - assert beacon_block.slot == shard_transitions[shard].shard_states[0].slot + target_len_offset_slot assert post_shard_state.slot == state.slot - 1 if len(shard_blocks[shard]) == 0: # `latest_block_root` is the same assert post_shard_state.latest_block_root == pre_shard_states[shard].latest_block_root + if target_len_offset_slot == 1 and len(shard_blocks) > 0: + assert post_shard_state.gasprice > pre_gasprice @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_process_beacon_block_with_normal_shard_transition(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 @@ -70,25 +74,13 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 - pre_gasprice = state.shard_states[shard].gasprice - - # Create SignedShardBlock at slot `shard_state.slot + 1` - body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) - shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} - - yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) - - shard_state = state.shard_states[shard] - - if target_len_offset_slot == 1 and len(shard_blocks) > 0: - assert shard_state.gasprice > pre_gasprice + yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_process_beacon_block_with_empty_proposal_transition(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 @@ -96,12 +88,4 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 - # No new shard block - shard_blocks = {} - - pre_gasprice = state.shard_states[shard].gasprice - - yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) - - if target_len_offset_slot == 1 and len(shard_blocks) > 0: - assert state.shard_states[shard].gasprice > pre_gasprice + yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) From 2afa315cb3b1d8abcef3af2ef7df9046eb4f8f23 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 5 Jun 2020 21:49:50 +0800 Subject: [PATCH 24/30] clean leftover --- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 106069dd6..ef90a71aa 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -85,9 +85,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: - temp_state = state.copy() - next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[], on_time_slot=slot + 1) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[], on_time_slot=slot + 1) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() From a71c0a5ccc4c74f5ae8c3c071aa47506aa5cc03a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 6 Jun 2020 02:35:46 +0800 Subject: [PATCH 25/30] Per #1704 discussion, remove `on_time_slot`: the given `beacon_state` should be transitioned. --- specs/phase1/shard-transition.md | 4 ++-- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 6 ++---- tests/core/pyspec/eth2spec/test/helpers/shard_block.py | 2 +- .../core/pyspec/eth2spec/test/helpers/shard_transitions.py | 7 ++----- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 8d75879f5..191dbd1aa 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -280,8 +280,8 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y ```python def get_shard_transition(beacon_state: BeaconState, shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - on_time_slot: Slot) -> ShardTransition: + shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: + on_time_slot = Slot(beacon_state.slot + 1) offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) proposals, shard_states, shard_data_roots = get_shard_state_transition_result( beacon_state, shard, shard_blocks, on_time_slot diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index ef90a71aa..1e0560405 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -85,7 +85,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: - shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[], on_time_slot=slot + 1) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[]) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() @@ -318,9 +318,7 @@ def next_epoch_with_attestations(spec, for index in range(committees_per_slot): if spec.fork == PHASE1: shard = spec.compute_shard_from_committee_index(post_state, index, slot_to_attest) - shard_transition = get_shard_transition_of_committee( - spec, post_state, index, slot=slot_to_attest - ) + shard_transition = get_shard_transition_of_committee(spec, post_state, index) block.body.shard_transitions[shard] = shard_transition else: shard_transition = None diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 0a2ed67d2..e63096b92 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -61,7 +61,7 @@ def build_shard_transitions_till_slot(spec, parent_beacon_state, shard_blocks): on_time_slot, ) len_offset_slots = len(offset_slots) - shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks, on_time_slot) + shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks) if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index 8e62b2f27..d10d1ee7b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -28,13 +28,10 @@ def run_shard_transitions_processing(spec, state, shard_transitions, attestation yield 'post', state -def get_shard_transition_of_committee(spec, state, committee_index, slot=None, shard_blocks=None): +def get_shard_transition_of_committee(spec, state, committee_index, shard_blocks=None): if shard_blocks is None: shard_blocks = [] - if slot is None: - slot = state.slot - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks, on_time_slot=slot + 1) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks) return shard_transition From a4cc189f2b34b9e99d53df32dc24b383d3b4d3bd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 6 Jun 2020 05:19:46 +0800 Subject: [PATCH 26/30] Apply PR feedback from Danny --- specs/phase1/shard-fork-choice.md | 10 +-- specs/phase1/shard-transition.md | 63 ++++--------------- .../test/fork_choice/test_on_shard_head.py | 4 +- 3 files changed, 21 insertions(+), 56 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 844dbfb86..b1fc3080e 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -46,7 +46,7 @@ class ShardStore: def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: return ShardStore( shard=shard, - blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot, shard=shard)}, block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, ) ``` @@ -168,13 +168,15 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root ) - # Add new block to the store - shard_store.blocks[hash_tree_root(shard_block)] = shard_block - # Check the block is valid and compute the post-state assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block, shard_block.slot, shard) assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) + post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) + + # Add new block to the store + shard_store.blocks[hash_tree_root(shard_block)] = shard_block + # Add new state for this block to the store shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 191dbd1aa..fe3223933 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -47,20 +47,14 @@ def verify_shard_block_message(beacon_state: BeaconState, shard_parent_state: ShardState, block: ShardBlock, slot: Slot, - shard: Shard, - beacon_parent_slot: Slot=None) -> bool: + shard: Shard) -> bool: # Check `shard_parent_root` field assert block.shard_parent_root == shard_parent_state.latest_block_root # Check `beacon_parent_root` field - if beacon_parent_slot is None: - beacon_parent_slot = beacon_state.slot - if beacon_parent_slot == beacon_state.slot: - beacon_parent_block_header = beacon_state.latest_block_header.copy() - if beacon_parent_block_header.state_root == Root(): - beacon_parent_block_header.state_root = hash_tree_root(beacon_state) - beacon_parent_root = hash_tree_root(beacon_parent_block_header) - else: - beacon_parent_root = get_block_root_at_slot(beacon_state, beacon_parent_slot) + beacon_parent_block_header = beacon_state.latest_block_header.copy() + if beacon_parent_block_header.state_root == Root(): + beacon_parent_block_header.state_root = hash_tree_root(beacon_state) + beacon_parent_root = hash_tree_root(beacon_parent_block_header) assert block.beacon_parent_root == beacon_parent_root # Check `slot` field next_slot = Slot(slot + 1) @@ -191,26 +185,6 @@ def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[ return [hash_tree_root(proposal.message.body) for proposal in proposals] ``` -```python -def get_proposal_choices_at_slot(beacon_state: BeaconState, - shard_parent_state: ShardState, - slot: Slot, - shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - validate_signature: bool=True) -> Sequence[SignedShardBlock]: - """ - Return the valid shard blocks at the given ``slot``. - Note that this function doesn't change the state. - """ - choices = [] - shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] - shard_state = shard_parent_state.copy() - for block in shard_blocks_at_slot: - shard_state = get_post_shard_state(beacon_state, shard_state, block.message) - choices.append(block) - return choices -``` - ```python def get_proposal_at_slot(beacon_state: BeaconState, shard_parent_state: ShardState, @@ -222,21 +196,14 @@ def get_proposal_at_slot(beacon_state: BeaconState, Return ``proposal``, ``shard_state`` of the given ``slot``. Note that this function doesn't change the state. """ - choices = get_proposal_choices_at_slot( - beacon_state=beacon_state, - shard_parent_state=shard_parent_state, - slot=slot, - shard=shard, - shard_blocks=shard_blocks, - validate_signature=validate_signature, - ) - if len(choices) == 0: + shard_blocks = [block for block in shard_blocks if block.message.slot == slot] + if len(shard_blocks) == 0: block = ShardBlock(slot=slot, shard=shard) proposal = SignedShardBlock(message=block) - elif len(choices) == 1: - proposal = choices[0] + elif len(shard_blocks) == 1: + proposal = shard_blocks[0] else: - proposal = get_winning_proposal(beacon_state, choices) + proposal = get_winning_proposal(beacon_state, shard_blocks) # Apply state transition shard_state = get_post_shard_state(beacon_state, shard_parent_state, proposal.message) @@ -249,13 +216,12 @@ def get_shard_state_transition_result( beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock], - on_time_slot: Slot, validate_signature: bool=True, ) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: proposals = [] shard_states = [] shard_state = beacon_state.shard_states[shard] - offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), Slot(beacon_state.slot + 1)) for slot in offset_slots: proposal, shard_state = get_proposal_at_slot( beacon_state=beacon_state, @@ -281,11 +247,8 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y def get_shard_transition(beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: - on_time_slot = Slot(beacon_state.slot + 1) - offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) - proposals, shard_states, shard_data_roots = get_shard_state_transition_result( - beacon_state, shard, shard_blocks, on_time_slot - ) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), Slot(beacon_state.slot + 1)) + proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks) shard_block_lengths = [] proposer_signatures = [] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index ca79cfd23..1a9042960 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -89,11 +89,11 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) # Clear buffer shard_blocks_buffer.clear() - signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) # transition! add_block_to_store(spec, store, signed_beacon_block) assert spec.get_head(store) == beacon_block.hash_tree_root() - # On shard block at updated `state.slot` + # On shard block at transitioned `state.slot` if is_in_offset_sets(spec, state, shard): # The created shard block would be appended to `shard_blocks_buffer` apply_shard_block(spec, store, shard_store, state, shard_blocks_buffer) From 435505746cd13f336c0bad1c71829df7fd322ca2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 8 Jun 2020 17:12:46 +0800 Subject: [PATCH 27/30] PR feedback from Terence: fix `get_shard_latest_attesting_balance` Co-authored-by: terence tsao --- specs/phase1/shard-fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 427b72784..5fef81868 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -65,7 +65,7 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro # would be ignored once their newer vote is accepted. Check if it makes sense. and store.latest_messages[i].shard == shard_store.shard and get_shard_ancestor( - store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot + store, shard_store, store.latest_messages[i].shard_root, shard_store.blocks[root].slot ) == root ) )) From 7e67aaeb35c6d8ffc4d8ced61f274161b6c04660 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 8 Jun 2020 18:15:14 +0800 Subject: [PATCH 28/30] Rename `build_shard_transitions_till_slot` to `get_shard_transitions` --- .../pyspec/eth2spec/test/fork_choice/test_on_shard_head.py | 4 ++-- tests/core/pyspec/eth2spec/test/helpers/shard_block.py | 2 +- .../phase_1/block_processing/test_process_shard_transition.py | 4 ++-- tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 1a9042960..24eeaedbe 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -4,7 +4,7 @@ from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_excep from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_block import ( build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root @@ -69,7 +69,7 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) # Sanity check `get_pending_shard_blocks` function check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) # Use temporary next state to get ShardTransition of shard block - shard_transitions = build_shard_transitions_till_slot( + shard_transitions = get_shard_transitions( spec, state, shard_blocks={shard: shard_blocks_buffer}, diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index e63096b92..f8b4a155f 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -52,7 +52,7 @@ def build_shard_block(spec, return signed_block -def build_shard_transitions_till_slot(spec, parent_beacon_state, shard_blocks): +def get_shard_transitions(spec, parent_beacon_state, shard_blocks): shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS on_time_slot = parent_beacon_state.slot + 1 for shard, blocks in shard_blocks.items(): diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index dab4973da..e97cc90a8 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -7,7 +7,7 @@ from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_transitions import run_shard_transitions_processing from eth2spec.test.helpers.shard_block import ( build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, ) from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot, next_slot @@ -24,7 +24,7 @@ def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks = [shard_block] - shard_transitions = build_shard_transitions_till_slot( + shard_transitions = get_shard_transitions( spec, state, shard_blocks={shard: shard_blocks}, diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 828cd19d7..33b0beac7 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -9,7 +9,7 @@ from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.shard_block import ( build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, ) from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to @@ -21,7 +21,7 @@ def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, comm shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} - shard_transitions = build_shard_transitions_till_slot(spec, state, shard_blocks) + shard_transitions = get_shard_transitions(spec, state, shard_blocks) attestations = [ get_valid_on_time_attestation( spec, From e03a970eaf88e82ade443ec37ac7451f5903966c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 8 Jun 2020 23:49:24 +0800 Subject: [PATCH 29/30] PR feedback from danny: simplify `verify_shard_block_message` params --- specs/phase1/shard-fork-choice.md | 2 +- specs/phase1/shard-transition.md | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index b1fc3080e..6c431a68d 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -169,7 +169,7 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si ) # Check the block is valid and compute the post-state - assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block, shard_block.slot, shard) + assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block) assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index fe3223933..c62b059ee 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -43,27 +43,26 @@ def compute_shard_transition_digest(beacon_parent_state: BeaconState, ### Shard block verification functions ```python -def verify_shard_block_message(beacon_state: BeaconState, +def verify_shard_block_message(beacon_parent_state: BeaconState, shard_parent_state: ShardState, - block: ShardBlock, - slot: Slot, - shard: Shard) -> bool: + block: ShardBlock) -> bool: # Check `shard_parent_root` field assert block.shard_parent_root == shard_parent_state.latest_block_root # Check `beacon_parent_root` field - beacon_parent_block_header = beacon_state.latest_block_header.copy() + beacon_parent_block_header = beacon_parent_state.latest_block_header.copy() if beacon_parent_block_header.state_root == Root(): - beacon_parent_block_header.state_root = hash_tree_root(beacon_state) + beacon_parent_block_header.state_root = hash_tree_root(beacon_parent_state) beacon_parent_root = hash_tree_root(beacon_parent_block_header) assert block.beacon_parent_root == beacon_parent_root # Check `slot` field - next_slot = Slot(slot + 1) - offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), next_slot) - assert slot in offset_slots + shard = block.shard + next_slot = Slot(block.slot + 1) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_parent_state, shard), next_slot) + assert block.slot in offset_slots # Check `shard` field assert block.shard == shard # Check `proposer_index` field - assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) + assert block.proposer_index == get_shard_proposer_index(beacon_parent_state, block.slot, shard) # Check `body` field assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE return True From 2d895e9388cd7364448fb357a769a9f03d7a5141 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Jun 2020 00:13:27 +0800 Subject: [PATCH 30/30] PR feedback from danny --- specs/phase1/shard-fork-choice.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 61e4cd36f..0607613e8 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -76,18 +76,18 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro ```python def get_shard_head(store: Store, shard_store: ShardStore) -> Root: # Execute the LMD-GHOST fork choice - shard_blocks = shard_store.blocks - head_beacon_root = get_head(store) - head_shard_state = store.block_states[head_beacon_root].shard_states[shard_store.shard] - shard_head_root = head_shard_state.latest_block_root + beacon_head_root = get_head(store) + shard_head_state = store.block_states[beacon_head_root].shard_states[shard_store.shard] + shard_head_root = shard_head_state.latest_block_root + shard_blocks = { + root: shard_block for root, shard_block in shard_store.blocks.items() + if shard_block.slot > shard_head_state.slot + } while True: # Find the valid child block roots children = [ - root for root in shard_store.blocks.keys() - if ( - shard_blocks[root].shard_parent_root == shard_head_root - and shard_blocks[root].slot > head_shard_state.slot - ) + root for root, shard_block in shard_blocks.items() + if shard_block.shard_parent_root == shard_head_root ] if len(children) == 0: return shard_head_root @@ -107,7 +107,7 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: elif block.slot == slot: return root else: - # root is older than queried slot, thus a skip slot. Return earliest root prior to slot + # root is older than queried slot, thus a skip slot. Return most recent root prior to slot return root ``` @@ -116,7 +116,7 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: ```python def get_pending_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: """ - Return the shard blocks branch that from shard head to beacon head. + Return the canonical shard block branch that has not yet been crosslinked. """ shard = shard_store.shard