From dab5a936c4b1fdfae0cc9c9b5c98ac0643097bd7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 Apr 2020 23:55:46 +0800 Subject: [PATCH 01/25] wip shard fork choice rule --- setup.py | 1 + specs/phase1/shard-fork-choice.md | 209 ++++++++++++++++++ .../test/fork_choice/test_get_head.py | 30 +-- .../test/fork_choice/test_on_shard_head.py | 97 ++++++++ .../eth2spec/test/helpers/fork_choice.py | 27 +++ 5 files changed, 335 insertions(+), 29 deletions(-) create mode 100644 specs/phase1/shard-fork-choice.md create mode 100644 tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py create mode 100644 tests/core/pyspec/eth2spec/test/helpers/fork_choice.py diff --git a/setup.py b/setup.py index e0d6561dd..316a7d32a 100644 --- a/setup.py +++ b/setup.py @@ -378,6 +378,7 @@ class PySpecCommand(Command): specs/phase1/shard-transition.md specs/phase1/fork-choice.md specs/phase1/phase1-fork.md + specs/phase1/shard-fork-choice.md """ else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md new file mode 100644 index 000000000..46e467e12 --- /dev/null +++ b/specs/phase1/shard-fork-choice.md @@ -0,0 +1,209 @@ +# Ethereum 2.0 Phase 1 -- Beacon Chain + Shard Chain Fork Choice + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [Introduction](#introduction) +- [Fork choice](#fork-choice) + - [Helpers](#helpers) + - [Extended `Store`](#extended-store) + - [Updated `get_forkchoice_store`](#updated-get_forkchoice_store) + - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) + - [`get_shard_head`](#get_shard_head) + - [`get_shard_ancestor`](#get_shard_ancestor) + - [`filter_shard_block_tree`](#filter_shard_block_tree) + - [`get_filtered_block_tree`](#get_filtered_block_tree) + - [Handlers](#handlers) + - [`on_shard_block`](#on_shard_block) + + + +## Introduction + +This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. + +## Fork choice + +### Helpers + +#### Extended `Store` + +```python +@dataclass +class Store(object): + time: uint64 + genesis_time: uint64 + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint + best_justified_checkpoint: Checkpoint + blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) + block_states: Dict[Root, BeaconState] = field(default_factory=dict) + checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) + latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) + # shard chain + shard_init_slots: Dict[Shard, Slot] = field(default_factory=dict) + shard_blocks: Dict[Shard, Dict[Root, ShardBlock]] = field(default_factory=dict) + shard_block_states: Dict[Shard, Dict[Root, ShardState]] = field(default_factory=dict) +``` + +#### Updated `get_forkchoice_store` + +```python +def get_forkchoice_store(anchor_state: BeaconState, + shard_init_slots: Dict[Shard, Slot], + anchor_state_shard_blocks: Dict[Shard, Dict[Root, ShardBlock]]) -> Store: + shard_count = len(anchor_state.shard_states) + anchor_block_header = anchor_state.latest_block_header.copy() + if anchor_block_header.state_root == Bytes32(): + anchor_block_header.state_root = hash_tree_root(anchor_state) + anchor_root = hash_tree_root(anchor_block_header) + anchor_epoch = get_current_epoch(anchor_state) + justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + return Store( + time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, + genesis_time=anchor_state.genesis_time, + justified_checkpoint=justified_checkpoint, + finalized_checkpoint=finalized_checkpoint, + best_justified_checkpoint=justified_checkpoint, + blocks={anchor_root: anchor_block_header}, + block_states={anchor_root: anchor_state.copy()}, + checkpoint_states={justified_checkpoint: anchor_state.copy()}, + # shard chain + shard_init_slots=shard_init_slots, + shard_blocks=anchor_state_shard_blocks, + shard_block_states={ + shard: { + anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard] + } + for shard in map(Shard, range(shard_count)) + }, + ) +``` + +#### `get_shard_latest_attesting_balance` + +```python +def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) -> Gwei: + state = store.checkpoint_states[store.justified_checkpoint] + active_indices = get_active_validator_indices(state, get_current_epoch(state)) + return Gwei(sum( + state.validators[i].effective_balance for i in active_indices + if ( + i in store.latest_messages and get_shard_ancestor( + store, shard, store.latest_messages[i].root, store.shard_blocks[shard][root].slot + ) == root + ) + )) +``` + +#### `get_shard_head` + +```python +def get_shard_head(store: Store, shard: Shard) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_shard_block_tree(store, shard) + + # Execute the LMD-GHOST fork choice + head_beacon_root = get_head(store) + head_shard_root = store.block_states[head_beacon_root].shard_states[shard].latest_block_root + while True: + children = [ + root for root in blocks.keys() + if blocks[root].shard_parent_root == head_shard_root + ] + if len(children) == 0: + return head_shard_root + # Sort by latest attesting balance with ties broken lexicographically + head_shard_root = max(children, key=lambda root: (get_shard_latest_attesting_balance(store, shard, root), root)) +``` + +#### `get_shard_ancestor` + +```python +def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: + block = store.shard_blocks[shard][root] + if block.slot > slot: + return get_shard_ancestor(store, shard, block.shard_parent_root, slot) + elif block.slot == slot: + return root + else: + # root is older than queried slot, thus a skip slot. Return earliest root prior to slot + return root +``` + +#### `filter_shard_block_tree` + +```python +def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: + block = store.shard_blocks[shard][block_root] + children = [ + root for root in store.shard_blocks[shard].keys() + if ( + store.shard_blocks[shard][root].shard_parent_root == block_root + and store.shard_blocks[shard][root].slot != store.shard_init_slots[shard] + ) + ] + + if any(children): + filter_block_tree_result = [filter_shard_block_tree(store, shard, child, blocks) for child in children] + if any(filter_block_tree_result): + blocks[block_root] = block + return True + return False + + return False +``` + +#### `get_filtered_block_tree` + +```python +def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, ShardBlock]: + base_beacon_block_root = get_head(store) + base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root + blocks: Dict[Root, ShardBlock] = {} + filter_shard_block_tree(store, shard, base_shard_block_root, blocks) + return blocks +``` + +### Handlers + +#### `on_shard_block` + +```python +def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: + shard_block = signed_shard_block.message + + # 1. Check shard parent exists + assert shard_block.shard_parent_root in store.shard_block_states[shard] + pre_shard_state = store.shard_block_states[shard][shard_block.shard_parent_root] + + # 2. Check beacon parent exists + assert shard_block.beacon_parent_root in store.block_states + beacon_state = store.block_states[shard_block.beacon_parent_root] + + # 3. Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert shard_block.slot > finalized_slot + + # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + assert ( + shard_block.beacon_parent_root == store.finalized_checkpoint.root + or get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root + ) + + # Add new block to the store + store.shard_blocks[shard][hash_tree_root(shard_block)] = shard_block + + # Check the block is valid and compute the post-state + verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) + verify_shard_block_signature(beacon_state, signed_shard_block) + post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) + # Add new state for this block to the store + store.shard_block_states[shard][hash_tree_root(shard_block)] = post_state +``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py index 17d4f644f..e25aad18f 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -1,41 +1,13 @@ from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.fork_choice import add_attestation_to_store, add_block_to_store, get_anchor_root from eth2spec.test.helpers.state import ( next_epoch, state_transition_and_sign_block, ) -def add_block_to_store(spec, store, signed_block): - pre_state = store.block_states[signed_block.message.parent_root] - block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT - - if store.time < block_time: - spec.on_tick(store, block_time) - - spec.on_block(store, signed_block) - - -def add_attestation_to_store(spec, store, attestation): - parent_block = store.blocks[attestation.data.beacon_block_root] - pre_state = store.block_states[spec.hash_tree_root(parent_block)] - block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT - next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT - - if store.time < next_epoch_time: - spec.on_tick(store, next_epoch_time) - - spec.on_attestation(store, attestation) - - -def get_anchor_root(spec, state): - anchor_block_header = state.latest_block_header.copy() - if anchor_block_header.state_root == spec.Bytes32(): - anchor_block_header.state_root = spec.hash_tree_root(state) - return spec.hash_tree_root(anchor_block_header) - - @with_all_phases @spec_state_test def test_genesis(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py new file mode 100644 index 000000000..8e72e214e --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -0,0 +1,97 @@ +from eth2spec.utils.ssz.ssz_impl import hash_tree_root + +from eth2spec.test.context import spec_state_test, with_all_phases_except, PHASE0 +from eth2spec.test.helpers.shard_block import ( + build_attestation_with_shard_transition, + build_shard_block, + build_shard_transitions_till_slot, +) +from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root +from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block +from eth2spec.test.helpers.block import build_empty_block + + +def run_on_shard_block(spec, store, shard, signed_block, valid=True): + if not valid: + try: + spec.on_shard_block(store, shard, signed_block) + except AssertionError: + return + else: + assert False + + spec.on_shard_block(store, shard, signed_block) + assert store.shard_blocks[shard][hash_tree_root(signed_block.message)] == signed_block.message + + +def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): + store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + + # Create SignedShardBlock + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_blocks = [shard_block] + + # Attester creates `attestation` + # Use temporary next state to get ShardTransition of shard block + shard_transitions = build_shard_transitions_till_slot( + spec, + state, + shards=[shard, ], + shard_blocks={shard: shard_blocks}, + target_len_offset_slot=1, + ) + shard_transition = shard_transitions[shard] + attestation = build_attestation_with_shard_transition( + spec, + state, + slot=state.slot, + index=committee_index, + target_len_offset_slot=1, + shard_transition=shard_transition, + ) + + # Propose beacon block at slot + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) + beacon_block.body.attestations = [attestation] + beacon_block.body.shard_transitions = shard_transitions + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + + run_on_shard_block(spec, store, shard, shard_block) + add_block_to_store(spec, store, signed_beacon_block) + + assert spec.get_head(store) == beacon_block.hash_tree_root() + assert spec.get_shard_head(store, shard) == shard_block.message.hash_tree_root() + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_basic(spec, state): + spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking + state = spec.upgrade_to_phase1(state) + next_slot(spec, state) + + # Initialization + shard_count = len(state.shard_states) + # Genesis shard blocks + anchor_shard_blocks = { + shard: { + state.shard_states[shard].latest_block_root: spec.ShardBlock( + slot=state.slot, + ) + } + for shard in map(spec.Shard, range(shard_count)) + } + shard_init_slots = { + shard: state.slot + for shard in map(spec.Shard, range(shard_count)) + } + store = spec.get_forkchoice_store(state, shard_init_slots, anchor_shard_blocks) + anchor_root = get_anchor_root(spec, state) + assert spec.get_head(store) == anchor_root + + committee_index = spec.CommitteeIndex(0) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + + run_apply_shard_and_beacon(spec, state, store, shard, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard, committee_index) diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py new file mode 100644 index 000000000..04e36ea84 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -0,0 +1,27 @@ +def get_anchor_root(spec, state): + anchor_block_header = state.latest_block_header.copy() + if anchor_block_header.state_root == spec.Bytes32(): + anchor_block_header.state_root = spec.hash_tree_root(state) + return spec.hash_tree_root(anchor_block_header) + + +def add_block_to_store(spec, store, signed_block): + pre_state = store.block_states[signed_block.message.parent_root] + block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT + + if store.time < block_time: + spec.on_tick(store, block_time) + + spec.on_block(store, signed_block) + + +def add_attestation_to_store(spec, store, attestation): + parent_block = store.blocks[attestation.data.beacon_block_root] + pre_state = store.block_states[spec.hash_tree_root(parent_block)] + block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT + next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT + + if store.time < next_epoch_time: + spec.on_tick(store, next_epoch_time) + + spec.on_attestation(store, attestation) From cddf9cf114744c07d8a1ce455c5cf73ff5ad89db Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 Apr 2020 20:06:20 +0800 Subject: [PATCH 02/25] Refactor --- specs/phase1/shard-fork-choice.md | 56 ++++++++++--------- .../test/fork_choice/test_on_shard_head.py | 18 +----- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 46e467e12..e9026b991 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -35,7 +35,13 @@ This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase ```python @dataclass -class Store(object): +class Store: + + @dataclass + class ShardStore: + blocks: Dict[Root, ShardBlock] = field(default_factory=dict) + block_states: Dict[Root, ShardState] = field(default_factory=dict) + time: uint64 genesis_time: uint64 justified_checkpoint: Checkpoint @@ -46,17 +52,13 @@ class Store(object): checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) # shard chain - shard_init_slots: Dict[Shard, Slot] = field(default_factory=dict) - shard_blocks: Dict[Shard, Dict[Root, ShardBlock]] = field(default_factory=dict) - shard_block_states: Dict[Shard, Dict[Root, ShardState]] = field(default_factory=dict) + shards: Dict[Shard, ShardStore] = field(default_factory=dict) # noqa: F821 ``` #### Updated `get_forkchoice_store` ```python -def get_forkchoice_store(anchor_state: BeaconState, - shard_init_slots: Dict[Shard, Slot], - anchor_state_shard_blocks: Dict[Shard, Dict[Root, ShardBlock]]) -> Store: +def get_forkchoice_store(anchor_state: BeaconState) -> Store: shard_count = len(anchor_state.shard_states) anchor_block_header = anchor_state.latest_block_header.copy() if anchor_block_header.state_root == Bytes32(): @@ -65,6 +67,14 @@ def get_forkchoice_store(anchor_state: BeaconState, anchor_epoch = get_current_epoch(anchor_state) justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + + shard_stores = {} + for shard in map(Shard, range(shard_count)): + shard_stores[shard] = Store.ShardStore( + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, + block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, + ) + return Store( time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, genesis_time=anchor_state.genesis_time, @@ -75,14 +85,7 @@ def get_forkchoice_store(anchor_state: BeaconState, block_states={anchor_root: anchor_state.copy()}, checkpoint_states={justified_checkpoint: anchor_state.copy()}, # shard chain - shard_init_slots=shard_init_slots, - shard_blocks=anchor_state_shard_blocks, - shard_block_states={ - shard: { - anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard] - } - for shard in map(Shard, range(shard_count)) - }, + shards=shard_stores, ) ``` @@ -96,7 +99,7 @@ def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) - state.validators[i].effective_balance for i in active_indices if ( i in store.latest_messages and get_shard_ancestor( - store, shard, store.latest_messages[i].root, store.shard_blocks[shard][root].slot + store, shard, store.latest_messages[i].root, store.shards[shard].blocks[root].slot ) == root ) )) @@ -127,7 +130,7 @@ def get_shard_head(store: Store, shard: Shard) -> Root: ```python def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: - block = store.shard_blocks[shard][root] + block = store.shards[shard].blocks[root] if block.slot > slot: return get_shard_ancestor(store, shard, block.shard_parent_root, slot) elif block.slot == slot: @@ -141,13 +144,11 @@ def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Ro ```python def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: - block = store.shard_blocks[shard][block_root] + shard_store = store.shards[shard] + block = shard_store.blocks[block_root] children = [ - root for root in store.shard_blocks[shard].keys() - if ( - store.shard_blocks[shard][root].shard_parent_root == block_root - and store.shard_blocks[shard][root].slot != store.shard_init_slots[shard] - ) + root for root in shard_store.blocks.keys() + if shard_store.blocks[root].shard_parent_root == block_root ] if any(children): @@ -178,10 +179,11 @@ def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, Shar ```python def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: shard_block = signed_shard_block.message + shard_store = store.shards[shard] # 1. Check shard parent exists - assert shard_block.shard_parent_root in store.shard_block_states[shard] - pre_shard_state = store.shard_block_states[shard][shard_block.shard_parent_root] + assert shard_block.shard_parent_root in shard_store.block_states + pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] # 2. Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states @@ -198,12 +200,12 @@ def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBl ) # Add new block to the store - store.shard_blocks[shard][hash_tree_root(shard_block)] = shard_block + shard_store.blocks[hash_tree_root(shard_block)] = shard_block # Check the block is valid and compute the post-state verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) verify_shard_block_signature(beacon_state, signed_shard_block) post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) # Add new state for this block to the store - store.shard_block_states[shard][hash_tree_root(shard_block)] = post_state + shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 8e72e214e..220c510e7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -21,7 +21,7 @@ def run_on_shard_block(spec, store, shard, signed_block, valid=True): assert False spec.on_shard_block(store, shard, signed_block) - assert store.shard_blocks[shard][hash_tree_root(signed_block.message)] == signed_block.message + assert store.shards[shard].blocks[hash_tree_root(signed_block.message)] == signed_block.message def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): @@ -72,21 +72,7 @@ def test_basic(spec, state): next_slot(spec, state) # Initialization - shard_count = len(state.shard_states) - # Genesis shard blocks - anchor_shard_blocks = { - shard: { - state.shard_states[shard].latest_block_root: spec.ShardBlock( - slot=state.slot, - ) - } - for shard in map(spec.Shard, range(shard_count)) - } - shard_init_slots = { - shard: state.slot - for shard in map(spec.Shard, range(shard_count)) - } - store = spec.get_forkchoice_store(state, shard_init_slots, anchor_shard_blocks) + store = spec.get_forkchoice_store(state) anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root From 8fafb6a9e5b17ee3e93fa0c1b3a8795608f7336b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 May 2020 10:25:58 +0800 Subject: [PATCH 03/25] Make `ShardStore` an independent object --- specs/phase1/shard-fork-choice.md | 97 ++++++------------- .../test/fork_choice/test_on_shard_head.py | 20 ++-- 2 files changed, 43 insertions(+), 74 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index e9026b991..4b3f42194 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -31,75 +31,38 @@ This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase ### Helpers -#### Extended `Store` +#### `ShardStore` ```python @dataclass -class Store: - - @dataclass - class ShardStore: - blocks: Dict[Root, ShardBlock] = field(default_factory=dict) - block_states: Dict[Root, ShardState] = field(default_factory=dict) - - time: uint64 - genesis_time: uint64 - justified_checkpoint: Checkpoint - finalized_checkpoint: Checkpoint - best_justified_checkpoint: Checkpoint - blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) - block_states: Dict[Root, BeaconState] = field(default_factory=dict) - checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) - latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) - # shard chain - shards: Dict[Shard, ShardStore] = field(default_factory=dict) # noqa: F821 +class ShardStore: + shard: Shard + blocks: Dict[Root, ShardBlock] = field(default_factory=dict) + block_states: Dict[Root, ShardState] = field(default_factory=dict) ``` -#### Updated `get_forkchoice_store` +#### Updated `get_forkchoice_shard_store` ```python -def get_forkchoice_store(anchor_state: BeaconState) -> Store: - shard_count = len(anchor_state.shard_states) - anchor_block_header = anchor_state.latest_block_header.copy() - if anchor_block_header.state_root == Bytes32(): - anchor_block_header.state_root = hash_tree_root(anchor_state) - anchor_root = hash_tree_root(anchor_block_header) - anchor_epoch = get_current_epoch(anchor_state) - justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - - shard_stores = {} - for shard in map(Shard, range(shard_count)): - shard_stores[shard] = Store.ShardStore( - blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, - block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, - ) - - return Store( - time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, - genesis_time=anchor_state.genesis_time, - justified_checkpoint=justified_checkpoint, - finalized_checkpoint=finalized_checkpoint, - best_justified_checkpoint=justified_checkpoint, - blocks={anchor_root: anchor_block_header}, - block_states={anchor_root: anchor_state.copy()}, - checkpoint_states={justified_checkpoint: anchor_state.copy()}, - # shard chain - shards=shard_stores, +def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: + return ShardStore( + shard=shard, + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, + block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, ) ``` #### `get_shard_latest_attesting_balance` ```python -def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) -> Gwei: +def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, root: Root) -> Gwei: state = store.checkpoint_states[store.justified_checkpoint] active_indices = get_active_validator_indices(state, get_current_epoch(state)) return Gwei(sum( state.validators[i].effective_balance for i in active_indices if ( i in store.latest_messages and get_shard_ancestor( - store, shard, store.latest_messages[i].root, store.shards[shard].blocks[root].slot + store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot ) == root ) )) @@ -108,13 +71,13 @@ def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) - #### `get_shard_head` ```python -def get_shard_head(store: Store, shard: Shard) -> Root: +def get_shard_head(store: Store, shard_store: ShardStore) -> Root: # Get filtered block tree that only includes viable branches - blocks = get_filtered_shard_block_tree(store, shard) + blocks = get_filtered_shard_block_tree(store, shard_store) # Execute the LMD-GHOST fork choice head_beacon_root = get_head(store) - head_shard_root = store.block_states[head_beacon_root].shard_states[shard].latest_block_root + head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root while True: children = [ root for root in blocks.keys() @@ -123,16 +86,18 @@ def get_shard_head(store: Store, shard: Shard) -> Root: if len(children) == 0: return head_shard_root # Sort by latest attesting balance with ties broken lexicographically - head_shard_root = max(children, key=lambda root: (get_shard_latest_attesting_balance(store, shard, root), root)) + head_shard_root = max( + children, key=lambda root: (get_shard_latest_attesting_balance(store, shard_store, root), root) + ) ``` #### `get_shard_ancestor` ```python -def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: - block = store.shards[shard].blocks[root] +def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: Slot) -> Root: + block = shard_store.blocks[root] if block.slot > slot: - return get_shard_ancestor(store, shard, block.shard_parent_root, slot) + return get_shard_ancestor(store, shard_store, block.shard_parent_root, slot) elif block.slot == slot: return root else: @@ -143,8 +108,10 @@ def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Ro #### `filter_shard_block_tree` ```python -def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: - shard_store = store.shards[shard] +def filter_shard_block_tree(store: Store, + shard_store: ShardStore, + block_root: Root, + blocks: Dict[Root, ShardBlock]) -> bool: block = shard_store.blocks[block_root] children = [ root for root in shard_store.blocks.keys() @@ -152,7 +119,7 @@ def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks ] if any(children): - filter_block_tree_result = [filter_shard_block_tree(store, shard, child, blocks) for child in children] + filter_block_tree_result = [filter_shard_block_tree(store, shard_store, child, blocks) for child in children] if any(filter_block_tree_result): blocks[block_root] = block return True @@ -164,11 +131,12 @@ def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks #### `get_filtered_block_tree` ```python -def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, ShardBlock]: +def get_filtered_shard_block_tree(store: Store, shard_store: ShardStore) -> Dict[Root, ShardBlock]: + shard = shard_store.shard base_beacon_block_root = get_head(store) base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root blocks: Dict[Root, ShardBlock] = {} - filter_shard_block_tree(store, shard, base_shard_block_root, blocks) + filter_shard_block_tree(store, shard_store, base_shard_block_root, blocks) return blocks ``` @@ -177,10 +145,9 @@ def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, Shar #### `on_shard_block` ```python -def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: +def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: SignedShardBlock) -> None: shard_block = signed_shard_block.message - shard_store = store.shards[shard] - + shard = shard_store.shard # 1. Check shard parent exists assert shard_block.shard_parent_root in shard_store.block_states pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 220c510e7..f4b883f06 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -11,20 +11,21 @@ from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_blo from eth2spec.test.helpers.block import build_empty_block -def run_on_shard_block(spec, store, shard, signed_block, valid=True): +def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): if not valid: try: - spec.on_shard_block(store, shard, signed_block) + spec.on_shard_block(store, shard_store, signed_block) except AssertionError: return else: assert False - spec.on_shard_block(store, shard, signed_block) - assert store.shards[shard].blocks[hash_tree_root(signed_block.message)] == signed_block.message + spec.on_shard_block(store, shard_store, signed_block) + assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): +def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index): + shard = shard_store.shard store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH # Create SignedShardBlock @@ -57,11 +58,11 @@ def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): beacon_block.body.shard_transitions = shard_transitions signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) - run_on_shard_block(spec, store, shard, shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) add_block_to_store(spec, store, signed_beacon_block) assert spec.get_head(store) == beacon_block.hash_tree_root() - assert spec.get_shard_head(store, shard) == shard_block.message.hash_tree_root() + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() @with_all_phases_except([PHASE0]) @@ -78,6 +79,7 @@ def test_basic(spec, state): committee_index = spec.CommitteeIndex(0) shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard_store = spec.get_forkchoice_shard_store(state, shard) - run_apply_shard_and_beacon(spec, state, store, shard, committee_index) - run_apply_shard_and_beacon(spec, state, store, shard, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) From fca1bbccb948b132fef22a9732c2f0a6aef0b2a4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 May 2020 10:33:23 +0800 Subject: [PATCH 04/25] Remove `get_filtered_shard_block_tree` --- specs/phase1/shard-fork-choice.md | 50 ++++--------------------------- 1 file changed, 5 insertions(+), 45 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 4b3f42194..b026aabfc 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -11,13 +11,11 @@ - [Introduction](#introduction) - [Fork choice](#fork-choice) - [Helpers](#helpers) - - [Extended `Store`](#extended-store) - - [Updated `get_forkchoice_store`](#updated-get_forkchoice_store) + - [`ShardStore`](#shardstore) + - [`get_forkchoice_shard_store`](#get_forkchoice_shard_store) - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) - [`get_shard_head`](#get_shard_head) - [`get_shard_ancestor`](#get_shard_ancestor) - - [`filter_shard_block_tree`](#filter_shard_block_tree) - - [`get_filtered_block_tree`](#get_filtered_block_tree) - [Handlers](#handlers) - [`on_shard_block`](#on_shard_block) @@ -41,7 +39,7 @@ class ShardStore: block_states: Dict[Root, ShardState] = field(default_factory=dict) ``` -#### Updated `get_forkchoice_shard_store` +#### `get_forkchoice_shard_store` ```python def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: @@ -72,16 +70,13 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro ```python def get_shard_head(store: Store, shard_store: ShardStore) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_shard_block_tree(store, shard_store) - # Execute the LMD-GHOST fork choice head_beacon_root = get_head(store) head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root while True: children = [ - root for root in blocks.keys() - if blocks[root].shard_parent_root == head_shard_root + root for root in shard_store.blocks.keys() + if shard_store.blocks[root].shard_parent_root == head_shard_root ] if len(children) == 0: return head_shard_root @@ -105,41 +100,6 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: return root ``` -#### `filter_shard_block_tree` - -```python -def filter_shard_block_tree(store: Store, - shard_store: ShardStore, - block_root: Root, - blocks: Dict[Root, ShardBlock]) -> bool: - block = shard_store.blocks[block_root] - children = [ - root for root in shard_store.blocks.keys() - if shard_store.blocks[root].shard_parent_root == block_root - ] - - if any(children): - filter_block_tree_result = [filter_shard_block_tree(store, shard_store, child, blocks) for child in children] - if any(filter_block_tree_result): - blocks[block_root] = block - return True - return False - - return False -``` - -#### `get_filtered_block_tree` - -```python -def get_filtered_shard_block_tree(store: Store, shard_store: ShardStore) -> Dict[Root, ShardBlock]: - shard = shard_store.shard - base_beacon_block_root = get_head(store) - base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root - blocks: Dict[Root, ShardBlock] = {} - filter_shard_block_tree(store, shard_store, base_shard_block_root, blocks) - return blocks -``` - ### Handlers #### `on_shard_block` From 79b1b4bdbe0c4822d4a2e197a7bf59930ffe69c1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 May 2020 10:52:45 +0800 Subject: [PATCH 05/25] Add `(shard, shard_root)` to `LatestMessage` --- specs/phase1/fork-choice.md | 32 ++++++++++++++++++- .../test/fork_choice/test_on_attestation.py | 13 ++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md index d8bf7fa09..f4e771ddb 100644 --- a/specs/phase1/fork-choice.md +++ b/specs/phase1/fork-choice.md @@ -10,6 +10,9 @@ - [Introduction](#introduction) - [Fork choice](#fork-choice) + - [Helpers](#helpers) + - [Extended `LatestMessage`](#extended-latestmessage) + - [Updated `update_latest_messages`](#updated-update_latest_messages) - [Handlers](#handlers) @@ -25,6 +28,33 @@ Due to the changes in the structure of `IndexedAttestation` in Phase 1, `on_atte The rest of the fork choice remains stable. +### Helpers + +#### Extended `LatestMessage` + +```python +@dataclass(eq=True, frozen=True) +class LatestMessage(object): + epoch: Epoch + root: Root + shard: Shard + shard_root: Root +``` + +#### Updated `update_latest_messages` + +```python +def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: + target = attestation.data.target + beacon_block_root = attestation.data.beacon_block_root + shard = get_shard(store.block_states[beacon_block_root], attestation) + for i in attesting_indices: + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage( + epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.head_shard_root + ) +``` + ### Handlers ```python @@ -49,4 +79,4 @@ def on_attestation(store: Store, attestation: Attestation) -> None: if attestation.aggregation_bits[i] ] update_latest_messages(store, attesting_indices, attestation) -``` \ No newline at end of file +``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 360c18ccd..1bce42ca3 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -18,18 +18,25 @@ def run_on_attestation(spec, state, store, attestation, valid=True): if spec.fork == PHASE0: sample_index = indexed_attestation.attesting_indices[0] + latest_message = spec.LatestMessage( + epoch=attestation.data.target.epoch, + root=attestation.data.beacon_block_root, + ) else: attesting_indices = [ index for i, index in enumerate(indexed_attestation.committee) if attestation.aggregation_bits[i] ] sample_index = attesting_indices[0] - assert ( - store.latest_messages[sample_index] == - spec.LatestMessage( + latest_message = spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, + shard=spec.get_shard(state, attestation), + shard_root=attestation.data.head_shard_root, ) + + assert ( + store.latest_messages[sample_index] == latest_message ) From 870ad8b921705cb658ebb0b88ff4e08bdeb54609 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 30 May 2020 03:56:56 +0800 Subject: [PATCH 06/25] Fix test --- .../pyspec/eth2spec/test/fork_choice/test_on_shard_head.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index f4b883f06..5b4205e91 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -30,6 +30,7 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + target_len_offset_slot = 1 shard_block = build_shard_block(spec, state, shard, body=body, signed=True) shard_blocks = [shard_block] @@ -38,17 +39,15 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) shard_transitions = build_shard_transitions_till_slot( spec, state, - shards=[shard, ], shard_blocks={shard: shard_blocks}, - target_len_offset_slot=1, + on_time_slot=state.slot + target_len_offset_slot, ) shard_transition = shard_transitions[shard] attestation = build_attestation_with_shard_transition( spec, state, - slot=state.slot, index=committee_index, - target_len_offset_slot=1, + on_time_slot=state.slot + target_len_offset_slot, shard_transition=shard_transition, ) From 142ba17451e05b0e6478796fe9bea9eb2de4c488 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 2 Jun 2020 18:08:28 +0800 Subject: [PATCH 07/25] PR review from Danny --- specs/phase1/shard-fork-choice.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index b026aabfc..867730203 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -59,7 +59,9 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro return Gwei(sum( state.validators[i].effective_balance for i in active_indices if ( - i in store.latest_messages and get_shard_ancestor( + i in store.latest_messages and + store.latest_messages[i].shard == shard_store.shard and + get_shard_ancestor( store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot ) == root ) @@ -71,12 +73,18 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro ```python def get_shard_head(store: Store, shard_store: ShardStore) -> Root: # Execute the LMD-GHOST fork choice + shard_blocks = shard_store.blocks head_beacon_root = get_head(store) - head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root + head_shard_state = store.block_states[head_beacon_root].shard_states[shard_store.shard] + head_shard_root = head_shard_state.latest_block_root while True: + # Find the valid child block roots children = [ root for root in shard_store.blocks.keys() - if shard_store.blocks[root].shard_parent_root == head_shard_root + if ( + shard_blocks[root].shard_parent_root == head_shard_root + and shard_blocks[root].slot > head_shard_state.slot + ) ] if len(children) == 0: return head_shard_root @@ -116,14 +124,15 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si assert shard_block.beacon_parent_root in store.block_states beacon_state = store.block_states[shard_block.beacon_parent_root] - # 3. Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert shard_block.slot > finalized_slot + # 3. Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) + finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] + finalized_shard_state = finalized_beacon_state.shard_states[shard] + assert shard_block.slot > finalized_shard_state.slot # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert ( - shard_block.beacon_parent_root == store.finalized_checkpoint.root - or get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root + get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root ) # Add new block to the store From 5c5cedd60d1c08583d8627e93db1b1b179f6b7f4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 3 Jun 2020 22:31:16 +0800 Subject: [PATCH 08/25] Apply PR feedback from Danny and Terence --- specs/phase1/shard-fork-choice.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 867730203..3b6bc5ac9 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -6,7 +6,7 @@ -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + - [Introduction](#introduction) - [Fork choice](#fork-choice) @@ -23,7 +23,7 @@ ## Introduction -This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. +This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. It assumes the [beacon chain fork choice spec](./fork-choice.md). ## Fork choice @@ -59,9 +59,9 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro return Gwei(sum( state.validators[i].effective_balance for i in active_indices if ( - i in store.latest_messages and - store.latest_messages[i].shard == shard_store.shard and - get_shard_ancestor( + i in store.latest_messages + and store.latest_messages[i].shard == shard_store.shard + and get_shard_ancestor( store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot ) == root ) @@ -116,20 +116,25 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: SignedShardBlock) -> None: shard_block = signed_shard_block.message shard = shard_store.shard - # 1. Check shard parent exists + + # Check shard + # TODO: check it in networking spec + assert shard_block.shard == shard + + # Check shard parent exists assert shard_block.shard_parent_root in shard_store.block_states pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] - # 2. Check beacon parent exists + # Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states beacon_state = store.block_states[shard_block.beacon_parent_root] - # 3. Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) + # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] finalized_shard_state = finalized_beacon_state.shard_states[shard] assert shard_block.slot > finalized_shard_state.slot - # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert ( get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root From e1981a7bfdeb2baf296cbfdb1f74169b56341883 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 01:00:52 +0800 Subject: [PATCH 09/25] `head_shard_root` -> `shard_head_root` --- specs/phase1/fork-choice.md | 2 +- specs/phase1/shard-fork-choice.md | 8 ++++---- .../eth2spec/test/fork_choice/test_on_attestation.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md index f4e771ddb..3f9fbdbfb 100644 --- a/specs/phase1/fork-choice.md +++ b/specs/phase1/fork-choice.md @@ -51,7 +51,7 @@ def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIn for i in attesting_indices: if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: store.latest_messages[i] = LatestMessage( - epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.head_shard_root + epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.shard_head_root ) ``` diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 3b6bc5ac9..fb98893ac 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -76,20 +76,20 @@ def get_shard_head(store: Store, shard_store: ShardStore) -> Root: shard_blocks = shard_store.blocks head_beacon_root = get_head(store) head_shard_state = store.block_states[head_beacon_root].shard_states[shard_store.shard] - head_shard_root = head_shard_state.latest_block_root + shard_head_root = head_shard_state.latest_block_root while True: # Find the valid child block roots children = [ root for root in shard_store.blocks.keys() if ( - shard_blocks[root].shard_parent_root == head_shard_root + shard_blocks[root].shard_parent_root == shard_head_root and shard_blocks[root].slot > head_shard_state.slot ) ] if len(children) == 0: - return head_shard_root + return shard_head_root # Sort by latest attesting balance with ties broken lexicographically - head_shard_root = max( + shard_head_root = max( children, key=lambda root: (get_shard_latest_attesting_balance(store, shard_store, root), root) ) ``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 04d2588d9..a5334c5c7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -32,7 +32,7 @@ def run_on_attestation(spec, state, store, attestation, valid=True): epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, shard=spec.get_shard(state, attestation), - shard_root=attestation.data.head_shard_root, + shard_root=attestation.data.shard_head_root, ) assert ( From d3445217416ad3a8f73dbf45e4669970d063682a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:30:13 +0800 Subject: [PATCH 10/25] Bugfix: should set `shard` for empty proposal --- specs/phase1/shard-transition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index e6221a980..5e8616568 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -221,7 +221,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, validate_signature=validate_signature, ) if len(choices) == 0: - block = ShardBlock(slot=slot) + block = ShardBlock(slot=slot, shard=shard) proposal = SignedShardBlock(message=block) elif len(choices) == 1: proposal = choices[0] From 26aae40941aac98d731655a475e3085e777c0110 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:31:53 +0800 Subject: [PATCH 11/25] Use epoch of the shard_block.slot for generating seed --- specs/phase1/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 484b24cf1..fccb93f55 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -538,7 +538,8 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) - r = bytes_to_int(get_seed(beacon_state, get_current_epoch(beacon_state), DOMAIN_SHARD_COMMITTEE)[:8]) + epoch = compute_epoch_at_slot(slot) + r = bytes_to_int(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE)[:8]) return committee[r % len(committee)] ``` From c9a53b8039cb2eada12cbb09ca1cd74509356549 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:33:01 +0800 Subject: [PATCH 12/25] WIP test case --- .../test/fork_choice/test_on_shard_head.py | 91 ++++++++++++------- .../eth2spec/test/helpers/shard_block.py | 26 ++++-- 2 files changed, 78 insertions(+), 39 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 5b4205e91..1151d18d7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -5,6 +5,7 @@ from eth2spec.test.helpers.shard_block import ( build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, + get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block @@ -24,44 +25,65 @@ def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index): +def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): shard = shard_store.shard + committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) + has_shard_committee = committee_index is not None store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH # Create SignedShardBlock - body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - target_len_offset_slot = 1 - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) - shard_blocks = [shard_block] + # Check offsets + temp_state = state.copy() + next_slot(spec, temp_state) + offset_slots = spec.get_offset_slots(temp_state, shard) + if state.slot in offset_slots: + # Build block + body = b'\x56' * 4 + shard_head_root = spec.get_shard_head(store, shard_store) + shard_parent_state = shard_store.block_states[shard_head_root] + assert shard_parent_state.slot != state.slot + shard_block = build_shard_block( + spec, state, shard, + shard_parent_state=shard_parent_state, slot=state.slot, body=body, signed=True + ) + shard_blocks_buffer.append(shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) # Attester creates `attestation` - # Use temporary next state to get ShardTransition of shard block - shard_transitions = build_shard_transitions_till_slot( - spec, - state, - shard_blocks={shard: shard_blocks}, - on_time_slot=state.slot + target_len_offset_slot, - ) - shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( - spec, - state, - index=committee_index, - on_time_slot=state.slot + target_len_offset_slot, - shard_transition=shard_transition, - ) + if has_shard_committee and len(shard_blocks_buffer) > 0: + # Use temporary next state to get ShardTransition of shard block + shard_transitions = build_shard_transitions_till_slot( + spec, + state, + shard_blocks={shard: shard_blocks_buffer}, + on_time_slot=state.slot + 1, + ) + shard_transition = shard_transitions[shard] + + attestation = build_attestation_with_shard_transition( + spec, + state, + index=committee_index, + on_time_slot=state.slot + 1, + shard_transition=shard_transition, + ) + assert attestation.data.slot == state.slot + assert spec.get_shard(state, attestation) == shard + beacon_block.body.attestations = [attestation] + beacon_block.body.shard_transitions = shard_transitions - # Propose beacon block at slot - beacon_block = build_empty_block(spec, state, slot=state.slot + 1) - beacon_block.body.attestations = [attestation] - beacon_block.body.shard_transitions = shard_transitions signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) - run_on_shard_block(spec, store, shard_store, shard_block) add_block_to_store(spec, store, signed_beacon_block) - assert spec.get_head(store) == beacon_block.hash_tree_root() - assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + if has_shard_committee: + shard_blocks_buffer = [] # clear buffer + + return has_shard_committee, shard_blocks_buffer @with_all_phases_except([PHASE0]) @@ -69,16 +91,19 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) def test_basic(spec, state): spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking state = spec.upgrade_to_phase1(state) - next_slot(spec, state) # Initialization store = spec.get_forkchoice_store(state) anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root - committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.Shard(1) shard_store = spec.get_forkchoice_shard_store(state, shard) - - run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) - run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) + shard_block_count = 2 + shard_blocks_buffer = [] + while shard_block_count > 0: + has_shard_committee, shard_blocks_buffer = run_apply_shard_and_beacon( + spec, state, store, shard_store, shard_blocks_buffer + ) + if has_shard_committee: + shard_block_count -= 1 diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 58efada83..410213edd 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -23,19 +23,24 @@ def build_shard_block(spec, shard, slot=None, body=None, + shard_parent_state=None, signed=False): - shard_state = beacon_state.shard_states[shard] + if shard_parent_state is None: + shard_parent_state = beacon_state.shard_states[shard] + if slot is None: - slot = shard_state.slot + 1 + slot = shard_parent_state.slot + 1 if body is None: body = b'\x56' * 128 - proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) + temp_state = beacon_state.copy() + transition_to(spec, temp_state, slot) beacon_state, beacon_parent_root = get_state_and_beacon_parent_root_at_slot(spec, beacon_state, slot) - + assert beacon_state == temp_state + proposer_index = spec.get_shard_proposer_index(temp_state, slot, shard) block = spec.ShardBlock( - shard_parent_root=shard_state.latest_block_root, + shard_parent_root=shard_parent_state.latest_block_root, beacon_parent_root=beacon_parent_root, slot=slot, shard=shard, @@ -59,7 +64,6 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): for shard, blocks in shard_blocks.items(): offset_slots = spec.get_offset_slots(temp_state, shard) len_offset_slots = len(offset_slots) - assert len_offset_slots == on_time_slot - state.shard_states[shard].slot - 1 shard_transition = spec.get_shard_transition(temp_state, shard, blocks) if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() @@ -84,3 +88,13 @@ def build_attestation_with_shard_transition(spec, state, index, on_time_slot, sh if shard_transition is not None: assert attestation.data.shard_transition_root == shard_transition.hash_tree_root() return attestation + + +def get_committee_index_of_shard(spec, state, slot, shard): # Optional[CommitteeIndex] + active_shard_count = spec.get_active_shard_count(state) + committee_count = spec.get_committee_count_at_slot(state, slot) + start_shard = spec.get_start_shard(state, slot) + for committee_index in range(committee_count): + if (start_shard + committee_index) % active_shard_count == shard: + return committee_index + return None From 727353c054f259aa81e7738099fc055722e9734c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 18:39:27 +0800 Subject: [PATCH 13/25] Verify shard_block.slot fits the expected offset_slots --- specs/phase1/shard-fork-choice.md | 9 +++++---- specs/phase1/shard-transition.md | 3 +++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index fb98893ac..b60edd948 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -127,7 +127,6 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si # Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states - beacon_state = store.block_states[shard_block.beacon_parent_root] # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] @@ -144,9 +143,11 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si shard_store.blocks[hash_tree_root(shard_block)] = shard_block # Check the block is valid and compute the post-state - verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) - verify_shard_block_signature(beacon_state, signed_shard_block) - post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) + beacon_head_root = get_head(store) + beacon_head_state = store.block_states[beacon_head_root] + assert verify_shard_block_message(beacon_head_state, pre_shard_state, shard_block, shard_block.slot, shard) + assert verify_shard_block_signature(beacon_head_state, signed_shard_block) + post_state = get_post_shard_state(beacon_head_state, pre_shard_state, shard_block) # Add new state for this block to the store shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 5e8616568..6c4f652d2 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -50,6 +50,9 @@ def verify_shard_block_message(beacon_state: BeaconState, shard: Shard) -> bool: assert block.shard_parent_root == shard_state.latest_block_root assert block.slot == slot + next_slot = Slot(beacon_state.slot + 1) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), next_slot) + assert slot in offset_slots assert block.shard == shard assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE From f8597d296545c6711d1c7cf8fe25d5dbb5b1987b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 20:31:54 +0800 Subject: [PATCH 14/25] Add `get_pendings_shard_blocks` --- specs/phase1/shard-fork-choice.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index b60edd948..5bc2cbe4e 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -16,6 +16,7 @@ - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) - [`get_shard_head`](#get_shard_head) - [`get_shard_ancestor`](#get_shard_ancestor) + - [`get_pendings_shard_blocks`](#get_pendings_shard_blocks) - [Handlers](#handlers) - [`on_shard_block`](#on_shard_block) @@ -108,6 +109,31 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: return root ``` +#### `get_pendings_shard_blocks` + +```python +def get_pendings_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: + """ + Return the shard blocks branch that from shard head to beacon head. + """ + shard = shard_store.shard + + beacon_head_root = get_head(store) + beacon_head_state = store.block_states[beacon_head_root] + latest_shard_block_root = beacon_head_state.shard_states[shard].latest_block_root + + shard_head_root = get_shard_head(store, shard_store) + root = shard_head_root + shard_blocks = [] + while root != latest_shard_block_root: + shard_block = shard_store.blocks[root] + shard_blocks.append(shard_block) + root = shard_block.shard_parent_root + + shard_blocks.reverse() + return shard_blocks +``` + ### Handlers #### `on_shard_block` From ab42eee4c04818d58fbcea5ba83d1c5890d0c3b8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 20:32:31 +0800 Subject: [PATCH 15/25] Update shard fork choice rule to be able to handle mainnet config --- .../test/fork_choice/test_on_shard_head.py | 97 ++++++++++++------- 1 file changed, 62 insertions(+), 35 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 1151d18d7..7e94ddd8e 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.test.context import spec_state_test, with_all_phases_except, PHASE0 +from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls from eth2spec.test.helpers.shard_block import ( build_attestation_with_shard_transition, build_shard_block, @@ -8,7 +8,7 @@ from eth2spec.test.helpers.shard_block import ( get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root -from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block +from eth2spec.test.helpers.state import state_transition_and_sign_block from eth2spec.test.helpers.block import build_empty_block @@ -25,35 +25,51 @@ def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): +def apply_shard_block(spec, store, shard_store, beacon_head_state, shard_blocks_buffer): shard = shard_store.shard - committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) - has_shard_committee = committee_index is not None + body = b'\x56' * 4 + shard_head_root = spec.get_shard_head(store, shard_store) + shard_parent_state = shard_store.block_states[shard_head_root] + assert shard_parent_state.slot != beacon_head_state.slot + shard_block = build_shard_block( + spec, beacon_head_state, shard, + shard_parent_state=shard_parent_state, slot=beacon_head_state.slot, body=body, signed=True + ) + shard_blocks_buffer.append(shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + +def check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer): + pending_shard_blocks = [ + spec.SignedShardBlock(message=b) + for b in spec.get_pendings_shard_blocks(store, shard_store) + ] + assert pending_shard_blocks == shard_blocks_buffer + + +def is_in_offset_sets(spec, beacon_head_state, shard): + offset_slots = spec.compute_offset_slots( + beacon_head_state.shard_states[shard].slot, beacon_head_state.slot + 1 + ) + return beacon_head_state.slot in offset_slots + + +def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH - # Create SignedShardBlock - # Check offsets - temp_state = state.copy() - next_slot(spec, temp_state) - offset_slots = spec.get_offset_slots(temp_state, shard) - if state.slot in offset_slots: - # Build block - body = b'\x56' * 4 - shard_head_root = spec.get_shard_head(store, shard_store) - shard_parent_state = shard_store.block_states[shard_head_root] - assert shard_parent_state.slot != state.slot - shard_block = build_shard_block( - spec, state, shard, - shard_parent_state=shard_parent_state, slot=state.slot, body=body, signed=True - ) - shard_blocks_buffer.append(shard_block) - run_on_shard_block(spec, store, shard_store, shard_block) - assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + shard = shard_store.shard + committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) + has_shard_committee = committee_index is not None # has committee of `shard` at this slot + # On beacon blocks at `state.slot + 1` beacon_block = build_empty_block(spec, state, slot=state.slot + 1) - # Attester creates `attestation` + # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block if has_shard_committee and len(shard_blocks_buffer) > 0: + # Sanity check `get_pendings_shard_blocks` function + check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) + # Use temporary next state to get ShardTransition of shard block shard_transitions = build_shard_transitions_till_slot( spec, @@ -62,7 +78,6 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buf on_time_slot=state.slot + 1, ) shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( spec, state, @@ -75,35 +90,47 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buf beacon_block.body.attestations = [attestation] beacon_block.body.shard_transitions = shard_transitions - signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + # Clear buffer + shard_blocks_buffer.clear() + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) add_block_to_store(spec, store, signed_beacon_block) assert spec.get_head(store) == beacon_block.hash_tree_root() - if has_shard_committee: - shard_blocks_buffer = [] # clear buffer + # On shard block at updated `state.slot` + if is_in_offset_sets(spec, state, shard): + # The created shard block would be appended to `shard_blocks_buffer` + apply_shard_block(spec, store, shard_store, state, shard_blocks_buffer) - return has_shard_committee, shard_blocks_buffer + return has_shard_committee @with_all_phases_except([PHASE0]) @spec_state_test +@never_bls # Set to never_bls for testing `check_pending_shard_blocks` def test_basic(spec, state): - spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking + spec.PHASE_1_GENESIS_SLOT = 0 # NOTE: mock genesis slot here state = spec.upgrade_to_phase1(state) + shard = spec.Shard(1) # Initialization store = spec.get_forkchoice_store(state) anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root - shard = spec.Shard(1) shard_store = spec.get_forkchoice_shard_store(state, shard) - shard_block_count = 2 + shard_head_root = spec.get_shard_head(store, shard_store) + assert shard_head_root == state.shard_states[shard].latest_block_root + assert shard_store.block_states[shard_head_root].slot == 1 + assert shard_store.block_states[shard_head_root] == state.shard_states[shard] + + # For mainnet config, it's possible that only one committee of `shard` per epoch. + # we set this counter to test more rounds. + shard_committee_counter = 2 shard_blocks_buffer = [] - while shard_block_count > 0: - has_shard_committee, shard_blocks_buffer = run_apply_shard_and_beacon( + while shard_committee_counter > 0: + has_shard_committee = apply_shard_and_beacon( spec, state, store, shard_store, shard_blocks_buffer ) if has_shard_committee: - shard_block_count -= 1 + shard_committee_counter -= 1 From 6f9c290bfb97240a9f41370b026d066ed8df6917 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 20:40:09 +0800 Subject: [PATCH 16/25] Add TODO flag of latest message --- specs/phase1/shard-fork-choice.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 5bc2cbe4e..a474c8214 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -61,6 +61,8 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro state.validators[i].effective_balance for i in active_indices if ( i in store.latest_messages + # TODO: check the latest message logic: currently, validator's previous vote of another shard + # would be ignored once their newer vote is accepted. Check if it makes sense. and store.latest_messages[i].shard == shard_store.shard and get_shard_ancestor( store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot From a154d0c22b9a3175e74383e4df53e5b09f6df0fd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 21:24:17 +0800 Subject: [PATCH 17/25] Fix typo --- specs/phase1/shard-fork-choice.md | 6 +++--- .../pyspec/eth2spec/test/fork_choice/test_on_shard_head.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index a474c8214..427b72784 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -16,7 +16,7 @@ - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) - [`get_shard_head`](#get_shard_head) - [`get_shard_ancestor`](#get_shard_ancestor) - - [`get_pendings_shard_blocks`](#get_pendings_shard_blocks) + - [`get_pending_shard_blocks`](#get_pending_shard_blocks) - [Handlers](#handlers) - [`on_shard_block`](#on_shard_block) @@ -111,10 +111,10 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: return root ``` -#### `get_pendings_shard_blocks` +#### `get_pending_shard_blocks` ```python -def get_pendings_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: +def get_pending_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: """ Return the shard blocks branch that from shard head to beacon head. """ diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 7e94ddd8e..1ba15968f 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -43,7 +43,7 @@ def apply_shard_block(spec, store, shard_store, beacon_head_state, shard_blocks_ def check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer): pending_shard_blocks = [ spec.SignedShardBlock(message=b) - for b in spec.get_pendings_shard_blocks(store, shard_store) + for b in spec.get_pending_shard_blocks(store, shard_store) ] assert pending_shard_blocks == shard_blocks_buffer @@ -67,7 +67,7 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block if has_shard_committee and len(shard_blocks_buffer) > 0: - # Sanity check `get_pendings_shard_blocks` function + # Sanity check `get_pending_shard_blocks` function check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) # Use temporary next state to get ShardTransition of shard block @@ -129,6 +129,7 @@ def test_basic(spec, state): shard_committee_counter = 2 shard_blocks_buffer = [] while shard_committee_counter > 0: + print(f'state.slot', state.slot) has_shard_committee = apply_shard_and_beacon( spec, state, store, shard_store, shard_blocks_buffer ) From 2d4788fe7d032f473fe5d60d732c40505ad8f485 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 5 Jun 2020 16:19:25 +0800 Subject: [PATCH 18/25] Fix `verify_shard_block_message` Add check for `block.beacon_parent_root` per Terence's suggestion Update `get_shard_transition` 1. Disable verification: it will be fix in v-guide 2. Use `on_time_slot` to compute offset_slots Rework tests --- specs/phase1/shard-fork-choice.md | 11 ++-- specs/phase1/shard-transition.md | 66 +++++++++++-------- .../test/fork_choice/test_on_shard_head.py | 19 ++---- .../eth2spec/test/helpers/attestations.py | 2 +- .../eth2spec/test/helpers/shard_block.py | 36 +++------- .../test/helpers/shard_transitions.py | 5 +- .../test_process_shard_transition.py | 31 ++++----- .../test/phase_1/sanity/test_blocks.py | 56 ++++++---------- 8 files changed, 94 insertions(+), 132 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 427b72784..844dbfb86 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -151,10 +151,11 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si # Check shard parent exists assert shard_block.shard_parent_root in shard_store.block_states - pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] + shard_parent_state = shard_store.block_states[shard_block.shard_parent_root] # Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states + beacon_parent_state = store.block_states[shard_block.beacon_parent_root] # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] @@ -171,11 +172,9 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si shard_store.blocks[hash_tree_root(shard_block)] = shard_block # Check the block is valid and compute the post-state - beacon_head_root = get_head(store) - beacon_head_state = store.block_states[beacon_head_root] - assert verify_shard_block_message(beacon_head_state, pre_shard_state, shard_block, shard_block.slot, shard) - assert verify_shard_block_signature(beacon_head_state, signed_shard_block) - post_state = get_post_shard_state(beacon_head_state, pre_shard_state, shard_block) + assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block, shard_block.slot, shard) + assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) + post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) # Add new state for this block to the store shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 6c4f652d2..8d75879f5 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -30,7 +30,7 @@ This document describes the shard transition function and fraud proofs as part o ### Misc ```python -def compute_shard_transition_digest(beacon_state: BeaconState, +def compute_shard_transition_digest(beacon_parent_state: BeaconState, shard_state: ShardState, beacon_parent_root: Root, shard_body_root: Root) -> Bytes32: @@ -44,17 +44,33 @@ def compute_shard_transition_digest(beacon_state: BeaconState, ```python def verify_shard_block_message(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, block: ShardBlock, slot: Slot, - shard: Shard) -> bool: - assert block.shard_parent_root == shard_state.latest_block_root - assert block.slot == slot - next_slot = Slot(beacon_state.slot + 1) + shard: Shard, + beacon_parent_slot: Slot=None) -> bool: + # Check `shard_parent_root` field + assert block.shard_parent_root == shard_parent_state.latest_block_root + # Check `beacon_parent_root` field + if beacon_parent_slot is None: + beacon_parent_slot = beacon_state.slot + if beacon_parent_slot == beacon_state.slot: + beacon_parent_block_header = beacon_state.latest_block_header.copy() + if beacon_parent_block_header.state_root == Root(): + beacon_parent_block_header.state_root = hash_tree_root(beacon_state) + beacon_parent_root = hash_tree_root(beacon_parent_block_header) + else: + beacon_parent_root = get_block_root_at_slot(beacon_state, beacon_parent_slot) + assert block.beacon_parent_root == beacon_parent_root + # Check `slot` field + next_slot = Slot(slot + 1) offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), next_slot) assert slot in offset_slots + # Check `shard` field assert block.shard == shard + # Check `proposer_index` field assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) + # Check `body` field assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE return True ``` @@ -177,7 +193,7 @@ def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[ ```python def get_proposal_choices_at_slot(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, slot: Slot, shard: Shard, shard_blocks: Sequence[SignedShardBlock], @@ -188,25 +204,16 @@ def get_proposal_choices_at_slot(beacon_state: BeaconState, """ choices = [] shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] + shard_state = shard_parent_state.copy() for block in shard_blocks_at_slot: - try: - # Verify block message and signature - # TODO these validations should have been checked upon receiving shard blocks. - assert verify_shard_block_message(beacon_state, shard_state, block.message, slot, shard) - if validate_signature: - assert verify_shard_block_signature(beacon_state, block) - - shard_state = get_post_shard_state(beacon_state, shard_state, block.message) - except Exception: - pass # TODO: throw error in the test helper - else: - choices.append(block) + shard_state = get_post_shard_state(beacon_state, shard_state, block.message) + choices.append(block) return choices ``` ```python def get_proposal_at_slot(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, slot: Shard, shard: Shard, shard_blocks: Sequence[SignedShardBlock], @@ -217,7 +224,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, """ choices = get_proposal_choices_at_slot( beacon_state=beacon_state, - shard_state=shard_state, + shard_parent_state=shard_parent_state, slot=slot, shard=shard, shard_blocks=shard_blocks, @@ -232,7 +239,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, proposal = get_winning_proposal(beacon_state, choices) # Apply state transition - shard_state = get_post_shard_state(beacon_state, shard_state, proposal.message) + shard_state = get_post_shard_state(beacon_state, shard_parent_state, proposal.message) return proposal, shard_state ``` @@ -242,15 +249,17 @@ def get_shard_state_transition_result( beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock], + on_time_slot: Slot, validate_signature: bool=True, ) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: proposals = [] shard_states = [] shard_state = beacon_state.shard_states[shard] - for slot in get_offset_slots(beacon_state, shard): + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) + for slot in offset_slots: proposal, shard_state = get_proposal_at_slot( beacon_state=beacon_state, - shard_state=shard_state, + shard_parent_state=shard_state, slot=slot, shard=shard, shard_blocks=shard_blocks, @@ -271,9 +280,12 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y ```python def get_shard_transition(beacon_state: BeaconState, shard: Shard, - shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: - offset_slots = get_offset_slots(beacon_state, shard) - proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks) + shard_blocks: Sequence[SignedShardBlock], + on_time_slot: Slot) -> ShardTransition: + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) + proposals, shard_states, shard_data_roots = get_shard_state_transition_result( + beacon_state, shard, shard_blocks, on_time_slot + ) shard_block_lengths = [] proposer_signatures = [] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 1ba15968f..ca79cfd23 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -1,8 +1,8 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, get_committee_index_of_shard, @@ -25,15 +25,15 @@ def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def apply_shard_block(spec, store, shard_store, beacon_head_state, shard_blocks_buffer): +def apply_shard_block(spec, store, shard_store, beacon_parent_state, shard_blocks_buffer): shard = shard_store.shard body = b'\x56' * 4 shard_head_root = spec.get_shard_head(store, shard_store) shard_parent_state = shard_store.block_states[shard_head_root] - assert shard_parent_state.slot != beacon_head_state.slot + assert shard_parent_state.slot != beacon_parent_state.slot shard_block = build_shard_block( - spec, beacon_head_state, shard, - shard_parent_state=shard_parent_state, slot=beacon_head_state.slot, body=body, signed=True + spec, beacon_parent_state, shard, + shard_parent_state=shard_parent_state, slot=beacon_parent_state.slot, body=body, signed=True ) shard_blocks_buffer.append(shard_block) run_on_shard_block(spec, store, shard_store, shard_block) @@ -62,30 +62,26 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) has_shard_committee = committee_index is not None # has committee of `shard` at this slot - # On beacon blocks at `state.slot + 1` beacon_block = build_empty_block(spec, state, slot=state.slot + 1) # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block if has_shard_committee and len(shard_blocks_buffer) > 0: # Sanity check `get_pending_shard_blocks` function check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) - # Use temporary next state to get ShardTransition of shard block shard_transitions = build_shard_transitions_till_slot( spec, state, shard_blocks={shard: shard_blocks_buffer}, - on_time_slot=state.slot + 1, ) shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( + attestation = get_valid_on_time_attestation( spec, state, index=committee_index, - on_time_slot=state.slot + 1, shard_transition=shard_transition, + signed=False, ) - assert attestation.data.slot == state.slot assert spec.get_shard(state, attestation) == shard beacon_block.body.attestations = [attestation] beacon_block.body.shard_transitions = shard_transitions @@ -129,7 +125,6 @@ def test_basic(spec, state): shard_committee_counter = 2 shard_blocks_buffer = [] while shard_committee_counter > 0: - print(f'state.slot', state.slot) has_shard_committee = apply_shard_and_beacon( spec, state, store, shard_store, shard_blocks_buffer ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 1372b0654..106069dd6 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -87,7 +87,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t if on_time: temp_state = state.copy() next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[]) + shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[], on_time_slot=slot + 1) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 410213edd..0a2ed67d2 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -1,6 +1,4 @@ -from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import get_state_and_beacon_parent_root_at_slot -from eth2spec.test.helpers.state import transition_to from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls @@ -34,11 +32,8 @@ def build_shard_block(spec, if body is None: body = b'\x56' * 128 - temp_state = beacon_state.copy() - transition_to(spec, temp_state, slot) beacon_state, beacon_parent_root = get_state_and_beacon_parent_root_at_slot(spec, beacon_state, slot) - assert beacon_state == temp_state - proposer_index = spec.get_shard_proposer_index(temp_state, slot, shard) + proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) block = spec.ShardBlock( shard_parent_root=shard_parent_state.latest_block_root, beacon_parent_root=beacon_parent_root, @@ -57,14 +52,17 @@ def build_shard_block(spec, return signed_block -def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot) +def build_shard_transitions_till_slot(spec, parent_beacon_state, shard_blocks): shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS + on_time_slot = parent_beacon_state.slot + 1 for shard, blocks in shard_blocks.items(): - offset_slots = spec.get_offset_slots(temp_state, shard) + offset_slots = spec.compute_offset_slots( + spec.get_latest_slot_for_shard(parent_beacon_state, shard), + on_time_slot, + ) len_offset_slots = len(offset_slots) - shard_transition = spec.get_shard_transition(temp_state, shard, blocks) + shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks, on_time_slot) + if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() assert shard_transition.shard_states[len_offset_slots - 1].latest_block_root == shard_block_root @@ -74,22 +72,6 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): return shard_transitions -def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot - 1) - attestation = get_valid_on_time_attestation( - spec, - temp_state, - index=index, - shard_transition=shard_transition, - signed=True, - ) - assert attestation.data.slot == temp_state.slot - if shard_transition is not None: - assert attestation.data.shard_transition_root == shard_transition.hash_tree_root() - return attestation - - def get_committee_index_of_shard(spec, state, slot, shard): # Optional[CommitteeIndex] active_shard_count = spec.get_active_shard_count(state) committee_count = spec.get_committee_count_at_slot(state, slot) diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index abb5e7278..8e62b2f27 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -1,5 +1,4 @@ from eth2spec.test.context import expect_assertion_error -from eth2spec.test.helpers.state import transition_to def run_shard_transitions_processing(spec, state, shard_transitions, attestations, valid=True): @@ -37,7 +36,5 @@ def get_shard_transition_of_committee(spec, state, committee_index, slot=None, s slot = state.slot shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - temp_state = state.copy() - transition_to(spec, temp_state, slot + 1) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=shard_blocks) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks, on_time_slot=slot + 1) return shard_transition diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index 00ffbe0a8..dab4973da 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -2,71 +2,64 @@ from eth2spec.test.context import ( PHASE0, with_all_phases_except, spec_state_test, - always_bls, ) +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_transitions import run_shard_transitions_processing from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, ) -from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot +from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot, next_slot def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): state = transition_to_valid_shard_slot(spec, state) - init_slot = state.slot committee_index = spec.CommitteeIndex(0) shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 + transition_to(spec, state, state.slot + target_len_offset_slot) + assert state.shard_states[shard].slot == state.slot - target_len_offset_slot - 1 # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks = [shard_block] - # Create a shard_transitions that would be included at beacon block `state.slot + target_len_offset_slot` shard_transitions = build_shard_transitions_till_slot( spec, state, shard_blocks={shard: shard_blocks}, - on_time_slot=state.slot + target_len_offset_slot, ) shard_transition = shard_transitions[shard] - # Create an attestation that would be included at beacon block `state.slot + target_len_offset_slot` - attestation = build_attestation_with_shard_transition( + attestation = get_valid_on_time_attestation( spec, state, index=committee_index, - on_time_slot=state.slot + target_len_offset_slot, shard_transition=shard_transition, + signed=False, ) + next_slot(spec, state) pre_gasprice = state.shard_states[shard].gasprice - - transition_to(spec, state, state.slot + target_len_offset_slot) pre_shard_state = state.shard_states[shard] - yield from run_shard_transitions_processing(spec, state, shard_transitions, [attestation], valid=valid) if valid: - # After state transition, - assert state.slot == init_slot + target_len_offset_slot shard_state = state.shard_states[shard] assert shard_state != pre_shard_state assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] - + assert shard_state.latest_block_root == shard_block.message.hash_tree_root() if target_len_offset_slot == 1: assert shard_state.gasprice > pre_gasprice @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_basic_crosslinks(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=1, valid=True) @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_multiple_offset_slots(spec, state): - yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=3, valid=True) + # NOTE: this test is only for full crosslink (minimal config), not for mainnet + yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=2, valid=True) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 0175bd40d..828cd19d7 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -4,37 +4,40 @@ from eth2spec.test.context import ( PHASE0, with_all_phases_except, spec_state_test, - always_bls, ) +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, ) -from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot +from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to -def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index, valid=True): - shard_transitions = build_shard_transitions_till_slot( - spec, state, shard_blocks, on_time_slot=state.slot + target_len_offset_slot - ) +def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard, valid=True): + transition_to(spec, state, state.slot + target_len_offset_slot) + + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) + shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} + + shard_transitions = build_shard_transitions_till_slot(spec, state, shard_blocks) attestations = [ - build_attestation_with_shard_transition( + get_valid_on_time_attestation( spec, state, - on_time_slot=state.slot + target_len_offset_slot, index=committee_index, shard_transition=shard_transitions[shard], + signed=True, ) for shard in shard_blocks.keys() ] - # Propose beacon block at slot `x + 1` - beacon_block = build_empty_block(spec, state, slot=state.slot + target_len_offset_slot) + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) beacon_block.body.attestations = attestations beacon_block.body.shard_transitions = shard_transitions + pre_gasprice = state.shard_states[shard].gasprice pre_shard_states = state.shard_states.copy() yield 'pre', state.copy() yield 'block', beacon_block @@ -52,17 +55,18 @@ def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_off assert post_shard_state == shard_transitions[shard].shard_states[ len(shard_transitions[shard].shard_states) - 1 ] - assert beacon_block.slot == shard_transitions[shard].shard_states[0].slot + target_len_offset_slot assert post_shard_state.slot == state.slot - 1 if len(shard_blocks[shard]) == 0: # `latest_block_root` is the same assert post_shard_state.latest_block_root == pre_shard_states[shard].latest_block_root + if target_len_offset_slot == 1 and len(shard_blocks) > 0: + assert post_shard_state.gasprice > pre_gasprice @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_process_beacon_block_with_normal_shard_transition(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 @@ -70,25 +74,13 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 - pre_gasprice = state.shard_states[shard].gasprice - - # Create SignedShardBlock at slot `shard_state.slot + 1` - body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) - shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} - - yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) - - shard_state = state.shard_states[shard] - - if target_len_offset_slot == 1 and len(shard_blocks) > 0: - assert shard_state.gasprice > pre_gasprice + yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_process_beacon_block_with_empty_proposal_transition(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 @@ -96,12 +88,4 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 - # No new shard block - shard_blocks = {} - - pre_gasprice = state.shard_states[shard].gasprice - - yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) - - if target_len_offset_slot == 1 and len(shard_blocks) > 0: - assert state.shard_states[shard].gasprice > pre_gasprice + yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) From 2afa315cb3b1d8abcef3af2ef7df9046eb4f8f23 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 5 Jun 2020 21:49:50 +0800 Subject: [PATCH 19/25] clean leftover --- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 106069dd6..ef90a71aa 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -85,9 +85,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: - temp_state = state.copy() - next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[], on_time_slot=slot + 1) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[], on_time_slot=slot + 1) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() From a71c0a5ccc4c74f5ae8c3c071aa47506aa5cc03a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 6 Jun 2020 02:35:46 +0800 Subject: [PATCH 20/25] Per #1704 discussion, remove `on_time_slot`: the given `beacon_state` should be transitioned. --- specs/phase1/shard-transition.md | 4 ++-- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 6 ++---- tests/core/pyspec/eth2spec/test/helpers/shard_block.py | 2 +- .../core/pyspec/eth2spec/test/helpers/shard_transitions.py | 7 ++----- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 8d75879f5..191dbd1aa 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -280,8 +280,8 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y ```python def get_shard_transition(beacon_state: BeaconState, shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - on_time_slot: Slot) -> ShardTransition: + shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: + on_time_slot = Slot(beacon_state.slot + 1) offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) proposals, shard_states, shard_data_roots = get_shard_state_transition_result( beacon_state, shard, shard_blocks, on_time_slot diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index ef90a71aa..1e0560405 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -85,7 +85,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: - shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[], on_time_slot=slot + 1) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[]) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() @@ -318,9 +318,7 @@ def next_epoch_with_attestations(spec, for index in range(committees_per_slot): if spec.fork == PHASE1: shard = spec.compute_shard_from_committee_index(post_state, index, slot_to_attest) - shard_transition = get_shard_transition_of_committee( - spec, post_state, index, slot=slot_to_attest - ) + shard_transition = get_shard_transition_of_committee(spec, post_state, index) block.body.shard_transitions[shard] = shard_transition else: shard_transition = None diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 0a2ed67d2..e63096b92 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -61,7 +61,7 @@ def build_shard_transitions_till_slot(spec, parent_beacon_state, shard_blocks): on_time_slot, ) len_offset_slots = len(offset_slots) - shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks, on_time_slot) + shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks) if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index 8e62b2f27..d10d1ee7b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -28,13 +28,10 @@ def run_shard_transitions_processing(spec, state, shard_transitions, attestation yield 'post', state -def get_shard_transition_of_committee(spec, state, committee_index, slot=None, shard_blocks=None): +def get_shard_transition_of_committee(spec, state, committee_index, shard_blocks=None): if shard_blocks is None: shard_blocks = [] - if slot is None: - slot = state.slot - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks, on_time_slot=slot + 1) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks) return shard_transition From a4cc189f2b34b9e99d53df32dc24b383d3b4d3bd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 6 Jun 2020 05:19:46 +0800 Subject: [PATCH 21/25] Apply PR feedback from Danny --- specs/phase1/shard-fork-choice.md | 10 +-- specs/phase1/shard-transition.md | 63 ++++--------------- .../test/fork_choice/test_on_shard_head.py | 4 +- 3 files changed, 21 insertions(+), 56 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 844dbfb86..b1fc3080e 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -46,7 +46,7 @@ class ShardStore: def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: return ShardStore( shard=shard, - blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot, shard=shard)}, block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, ) ``` @@ -168,13 +168,15 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root ) - # Add new block to the store - shard_store.blocks[hash_tree_root(shard_block)] = shard_block - # Check the block is valid and compute the post-state assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block, shard_block.slot, shard) assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) + post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) + + # Add new block to the store + shard_store.blocks[hash_tree_root(shard_block)] = shard_block + # Add new state for this block to the store shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 191dbd1aa..fe3223933 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -47,20 +47,14 @@ def verify_shard_block_message(beacon_state: BeaconState, shard_parent_state: ShardState, block: ShardBlock, slot: Slot, - shard: Shard, - beacon_parent_slot: Slot=None) -> bool: + shard: Shard) -> bool: # Check `shard_parent_root` field assert block.shard_parent_root == shard_parent_state.latest_block_root # Check `beacon_parent_root` field - if beacon_parent_slot is None: - beacon_parent_slot = beacon_state.slot - if beacon_parent_slot == beacon_state.slot: - beacon_parent_block_header = beacon_state.latest_block_header.copy() - if beacon_parent_block_header.state_root == Root(): - beacon_parent_block_header.state_root = hash_tree_root(beacon_state) - beacon_parent_root = hash_tree_root(beacon_parent_block_header) - else: - beacon_parent_root = get_block_root_at_slot(beacon_state, beacon_parent_slot) + beacon_parent_block_header = beacon_state.latest_block_header.copy() + if beacon_parent_block_header.state_root == Root(): + beacon_parent_block_header.state_root = hash_tree_root(beacon_state) + beacon_parent_root = hash_tree_root(beacon_parent_block_header) assert block.beacon_parent_root == beacon_parent_root # Check `slot` field next_slot = Slot(slot + 1) @@ -191,26 +185,6 @@ def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[ return [hash_tree_root(proposal.message.body) for proposal in proposals] ``` -```python -def get_proposal_choices_at_slot(beacon_state: BeaconState, - shard_parent_state: ShardState, - slot: Slot, - shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - validate_signature: bool=True) -> Sequence[SignedShardBlock]: - """ - Return the valid shard blocks at the given ``slot``. - Note that this function doesn't change the state. - """ - choices = [] - shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] - shard_state = shard_parent_state.copy() - for block in shard_blocks_at_slot: - shard_state = get_post_shard_state(beacon_state, shard_state, block.message) - choices.append(block) - return choices -``` - ```python def get_proposal_at_slot(beacon_state: BeaconState, shard_parent_state: ShardState, @@ -222,21 +196,14 @@ def get_proposal_at_slot(beacon_state: BeaconState, Return ``proposal``, ``shard_state`` of the given ``slot``. Note that this function doesn't change the state. """ - choices = get_proposal_choices_at_slot( - beacon_state=beacon_state, - shard_parent_state=shard_parent_state, - slot=slot, - shard=shard, - shard_blocks=shard_blocks, - validate_signature=validate_signature, - ) - if len(choices) == 0: + shard_blocks = [block for block in shard_blocks if block.message.slot == slot] + if len(shard_blocks) == 0: block = ShardBlock(slot=slot, shard=shard) proposal = SignedShardBlock(message=block) - elif len(choices) == 1: - proposal = choices[0] + elif len(shard_blocks) == 1: + proposal = shard_blocks[0] else: - proposal = get_winning_proposal(beacon_state, choices) + proposal = get_winning_proposal(beacon_state, shard_blocks) # Apply state transition shard_state = get_post_shard_state(beacon_state, shard_parent_state, proposal.message) @@ -249,13 +216,12 @@ def get_shard_state_transition_result( beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock], - on_time_slot: Slot, validate_signature: bool=True, ) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: proposals = [] shard_states = [] shard_state = beacon_state.shard_states[shard] - offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), Slot(beacon_state.slot + 1)) for slot in offset_slots: proposal, shard_state = get_proposal_at_slot( beacon_state=beacon_state, @@ -281,11 +247,8 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y def get_shard_transition(beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: - on_time_slot = Slot(beacon_state.slot + 1) - offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) - proposals, shard_states, shard_data_roots = get_shard_state_transition_result( - beacon_state, shard, shard_blocks, on_time_slot - ) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), Slot(beacon_state.slot + 1)) + proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks) shard_block_lengths = [] proposer_signatures = [] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index ca79cfd23..1a9042960 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -89,11 +89,11 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) # Clear buffer shard_blocks_buffer.clear() - signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) # transition! add_block_to_store(spec, store, signed_beacon_block) assert spec.get_head(store) == beacon_block.hash_tree_root() - # On shard block at updated `state.slot` + # On shard block at transitioned `state.slot` if is_in_offset_sets(spec, state, shard): # The created shard block would be appended to `shard_blocks_buffer` apply_shard_block(spec, store, shard_store, state, shard_blocks_buffer) From 435505746cd13f336c0bad1c71829df7fd322ca2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 8 Jun 2020 17:12:46 +0800 Subject: [PATCH 22/25] PR feedback from Terence: fix `get_shard_latest_attesting_balance` Co-authored-by: terence tsao --- specs/phase1/shard-fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 427b72784..5fef81868 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -65,7 +65,7 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro # would be ignored once their newer vote is accepted. Check if it makes sense. and store.latest_messages[i].shard == shard_store.shard and get_shard_ancestor( - store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot + store, shard_store, store.latest_messages[i].shard_root, shard_store.blocks[root].slot ) == root ) )) From 7e67aaeb35c6d8ffc4d8ced61f274161b6c04660 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 8 Jun 2020 18:15:14 +0800 Subject: [PATCH 23/25] Rename `build_shard_transitions_till_slot` to `get_shard_transitions` --- .../pyspec/eth2spec/test/fork_choice/test_on_shard_head.py | 4 ++-- tests/core/pyspec/eth2spec/test/helpers/shard_block.py | 2 +- .../phase_1/block_processing/test_process_shard_transition.py | 4 ++-- tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 1a9042960..24eeaedbe 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -4,7 +4,7 @@ from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_excep from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_block import ( build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root @@ -69,7 +69,7 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) # Sanity check `get_pending_shard_blocks` function check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) # Use temporary next state to get ShardTransition of shard block - shard_transitions = build_shard_transitions_till_slot( + shard_transitions = get_shard_transitions( spec, state, shard_blocks={shard: shard_blocks_buffer}, diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index e63096b92..f8b4a155f 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -52,7 +52,7 @@ def build_shard_block(spec, return signed_block -def build_shard_transitions_till_slot(spec, parent_beacon_state, shard_blocks): +def get_shard_transitions(spec, parent_beacon_state, shard_blocks): shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS on_time_slot = parent_beacon_state.slot + 1 for shard, blocks in shard_blocks.items(): diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index dab4973da..e97cc90a8 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -7,7 +7,7 @@ from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_transitions import run_shard_transitions_processing from eth2spec.test.helpers.shard_block import ( build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, ) from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot, next_slot @@ -24,7 +24,7 @@ def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks = [shard_block] - shard_transitions = build_shard_transitions_till_slot( + shard_transitions = get_shard_transitions( spec, state, shard_blocks={shard: shard_blocks}, diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 828cd19d7..33b0beac7 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -9,7 +9,7 @@ from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.shard_block import ( build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, ) from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to @@ -21,7 +21,7 @@ def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, comm shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} - shard_transitions = build_shard_transitions_till_slot(spec, state, shard_blocks) + shard_transitions = get_shard_transitions(spec, state, shard_blocks) attestations = [ get_valid_on_time_attestation( spec, From e03a970eaf88e82ade443ec37ac7451f5903966c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 8 Jun 2020 23:49:24 +0800 Subject: [PATCH 24/25] PR feedback from danny: simplify `verify_shard_block_message` params --- specs/phase1/shard-fork-choice.md | 2 +- specs/phase1/shard-transition.md | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index b1fc3080e..6c431a68d 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -169,7 +169,7 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si ) # Check the block is valid and compute the post-state - assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block, shard_block.slot, shard) + assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block) assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index fe3223933..c62b059ee 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -43,27 +43,26 @@ def compute_shard_transition_digest(beacon_parent_state: BeaconState, ### Shard block verification functions ```python -def verify_shard_block_message(beacon_state: BeaconState, +def verify_shard_block_message(beacon_parent_state: BeaconState, shard_parent_state: ShardState, - block: ShardBlock, - slot: Slot, - shard: Shard) -> bool: + block: ShardBlock) -> bool: # Check `shard_parent_root` field assert block.shard_parent_root == shard_parent_state.latest_block_root # Check `beacon_parent_root` field - beacon_parent_block_header = beacon_state.latest_block_header.copy() + beacon_parent_block_header = beacon_parent_state.latest_block_header.copy() if beacon_parent_block_header.state_root == Root(): - beacon_parent_block_header.state_root = hash_tree_root(beacon_state) + beacon_parent_block_header.state_root = hash_tree_root(beacon_parent_state) beacon_parent_root = hash_tree_root(beacon_parent_block_header) assert block.beacon_parent_root == beacon_parent_root # Check `slot` field - next_slot = Slot(slot + 1) - offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), next_slot) - assert slot in offset_slots + shard = block.shard + next_slot = Slot(block.slot + 1) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_parent_state, shard), next_slot) + assert block.slot in offset_slots # Check `shard` field assert block.shard == shard # Check `proposer_index` field - assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) + assert block.proposer_index == get_shard_proposer_index(beacon_parent_state, block.slot, shard) # Check `body` field assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE return True From 2d895e9388cd7364448fb357a769a9f03d7a5141 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Jun 2020 00:13:27 +0800 Subject: [PATCH 25/25] PR feedback from danny --- specs/phase1/shard-fork-choice.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 61e4cd36f..0607613e8 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -76,18 +76,18 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro ```python def get_shard_head(store: Store, shard_store: ShardStore) -> Root: # Execute the LMD-GHOST fork choice - shard_blocks = shard_store.blocks - head_beacon_root = get_head(store) - head_shard_state = store.block_states[head_beacon_root].shard_states[shard_store.shard] - shard_head_root = head_shard_state.latest_block_root + beacon_head_root = get_head(store) + shard_head_state = store.block_states[beacon_head_root].shard_states[shard_store.shard] + shard_head_root = shard_head_state.latest_block_root + shard_blocks = { + root: shard_block for root, shard_block in shard_store.blocks.items() + if shard_block.slot > shard_head_state.slot + } while True: # Find the valid child block roots children = [ - root for root in shard_store.blocks.keys() - if ( - shard_blocks[root].shard_parent_root == shard_head_root - and shard_blocks[root].slot > head_shard_state.slot - ) + root for root, shard_block in shard_blocks.items() + if shard_block.shard_parent_root == shard_head_root ] if len(children) == 0: return shard_head_root @@ -107,7 +107,7 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: elif block.slot == slot: return root else: - # root is older than queried slot, thus a skip slot. Return earliest root prior to slot + # root is older than queried slot, thus a skip slot. Return most recent root prior to slot return root ``` @@ -116,7 +116,7 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: ```python def get_pending_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: """ - Return the shard blocks branch that from shard head to beacon head. + Return the canonical shard block branch that has not yet been crosslinked. """ shard = shard_store.shard