From dab5a936c4b1fdfae0cc9c9b5c98ac0643097bd7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 Apr 2020 23:55:46 +0800 Subject: [PATCH] wip shard fork choice rule --- setup.py | 1 + specs/phase1/shard-fork-choice.md | 209 ++++++++++++++++++ .../test/fork_choice/test_get_head.py | 30 +-- .../test/fork_choice/test_on_shard_head.py | 97 ++++++++ .../eth2spec/test/helpers/fork_choice.py | 27 +++ 5 files changed, 335 insertions(+), 29 deletions(-) create mode 100644 specs/phase1/shard-fork-choice.md create mode 100644 tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py create mode 100644 tests/core/pyspec/eth2spec/test/helpers/fork_choice.py diff --git a/setup.py b/setup.py index e0d6561dd..316a7d32a 100644 --- a/setup.py +++ b/setup.py @@ -378,6 +378,7 @@ class PySpecCommand(Command): specs/phase1/shard-transition.md specs/phase1/fork-choice.md specs/phase1/phase1-fork.md + specs/phase1/shard-fork-choice.md """ else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md new file mode 100644 index 000000000..46e467e12 --- /dev/null +++ b/specs/phase1/shard-fork-choice.md @@ -0,0 +1,209 @@ +# Ethereum 2.0 Phase 1 -- Beacon Chain + Shard Chain Fork Choice + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [Introduction](#introduction) +- [Fork choice](#fork-choice) + - [Helpers](#helpers) + - [Extended `Store`](#extended-store) + - [Updated `get_forkchoice_store`](#updated-get_forkchoice_store) + - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) + - [`get_shard_head`](#get_shard_head) + - [`get_shard_ancestor`](#get_shard_ancestor) + - [`filter_shard_block_tree`](#filter_shard_block_tree) + - [`get_filtered_block_tree`](#get_filtered_block_tree) + - [Handlers](#handlers) + - [`on_shard_block`](#on_shard_block) + + + +## Introduction + +This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. + +## Fork choice + +### Helpers + +#### Extended `Store` + +```python +@dataclass +class Store(object): + time: uint64 + genesis_time: uint64 + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint + best_justified_checkpoint: Checkpoint + blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) + block_states: Dict[Root, BeaconState] = field(default_factory=dict) + checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) + latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) + # shard chain + shard_init_slots: Dict[Shard, Slot] = field(default_factory=dict) + shard_blocks: Dict[Shard, Dict[Root, ShardBlock]] = field(default_factory=dict) + shard_block_states: Dict[Shard, Dict[Root, ShardState]] = field(default_factory=dict) +``` + +#### Updated `get_forkchoice_store` + +```python +def get_forkchoice_store(anchor_state: BeaconState, + shard_init_slots: Dict[Shard, Slot], + anchor_state_shard_blocks: Dict[Shard, Dict[Root, ShardBlock]]) -> Store: + shard_count = len(anchor_state.shard_states) + anchor_block_header = anchor_state.latest_block_header.copy() + if anchor_block_header.state_root == Bytes32(): + anchor_block_header.state_root = hash_tree_root(anchor_state) + anchor_root = hash_tree_root(anchor_block_header) + anchor_epoch = get_current_epoch(anchor_state) + justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + return Store( + time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, + genesis_time=anchor_state.genesis_time, + justified_checkpoint=justified_checkpoint, + finalized_checkpoint=finalized_checkpoint, + best_justified_checkpoint=justified_checkpoint, + blocks={anchor_root: anchor_block_header}, + block_states={anchor_root: anchor_state.copy()}, + checkpoint_states={justified_checkpoint: anchor_state.copy()}, + # shard chain + shard_init_slots=shard_init_slots, + shard_blocks=anchor_state_shard_blocks, + shard_block_states={ + shard: { + anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard] + } + for shard in map(Shard, range(shard_count)) + }, + ) +``` + +#### `get_shard_latest_attesting_balance` + +```python +def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) -> Gwei: + state = store.checkpoint_states[store.justified_checkpoint] + active_indices = get_active_validator_indices(state, get_current_epoch(state)) + return Gwei(sum( + state.validators[i].effective_balance for i in active_indices + if ( + i in store.latest_messages and get_shard_ancestor( + store, shard, store.latest_messages[i].root, store.shard_blocks[shard][root].slot + ) == root + ) + )) +``` + +#### `get_shard_head` + +```python +def get_shard_head(store: Store, shard: Shard) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_shard_block_tree(store, shard) + + # Execute the LMD-GHOST fork choice + head_beacon_root = get_head(store) + head_shard_root = store.block_states[head_beacon_root].shard_states[shard].latest_block_root + while True: + children = [ + root for root in blocks.keys() + if blocks[root].shard_parent_root == head_shard_root + ] + if len(children) == 0: + return head_shard_root + # Sort by latest attesting balance with ties broken lexicographically + head_shard_root = max(children, key=lambda root: (get_shard_latest_attesting_balance(store, shard, root), root)) +``` + +#### `get_shard_ancestor` + +```python +def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: + block = store.shard_blocks[shard][root] + if block.slot > slot: + return get_shard_ancestor(store, shard, block.shard_parent_root, slot) + elif block.slot == slot: + return root + else: + # root is older than queried slot, thus a skip slot. Return earliest root prior to slot + return root +``` + +#### `filter_shard_block_tree` + +```python +def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: + block = store.shard_blocks[shard][block_root] + children = [ + root for root in store.shard_blocks[shard].keys() + if ( + store.shard_blocks[shard][root].shard_parent_root == block_root + and store.shard_blocks[shard][root].slot != store.shard_init_slots[shard] + ) + ] + + if any(children): + filter_block_tree_result = [filter_shard_block_tree(store, shard, child, blocks) for child in children] + if any(filter_block_tree_result): + blocks[block_root] = block + return True + return False + + return False +``` + +#### `get_filtered_block_tree` + +```python +def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, ShardBlock]: + base_beacon_block_root = get_head(store) + base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root + blocks: Dict[Root, ShardBlock] = {} + filter_shard_block_tree(store, shard, base_shard_block_root, blocks) + return blocks +``` + +### Handlers + +#### `on_shard_block` + +```python +def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: + shard_block = signed_shard_block.message + + # 1. Check shard parent exists + assert shard_block.shard_parent_root in store.shard_block_states[shard] + pre_shard_state = store.shard_block_states[shard][shard_block.shard_parent_root] + + # 2. Check beacon parent exists + assert shard_block.beacon_parent_root in store.block_states + beacon_state = store.block_states[shard_block.beacon_parent_root] + + # 3. Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert shard_block.slot > finalized_slot + + # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + assert ( + shard_block.beacon_parent_root == store.finalized_checkpoint.root + or get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root + ) + + # Add new block to the store + store.shard_blocks[shard][hash_tree_root(shard_block)] = shard_block + + # Check the block is valid and compute the post-state + verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) + verify_shard_block_signature(beacon_state, signed_shard_block) + post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) + # Add new state for this block to the store + store.shard_block_states[shard][hash_tree_root(shard_block)] = post_state +``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py index 17d4f644f..e25aad18f 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -1,41 +1,13 @@ from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.fork_choice import add_attestation_to_store, add_block_to_store, get_anchor_root from eth2spec.test.helpers.state import ( next_epoch, state_transition_and_sign_block, ) -def add_block_to_store(spec, store, signed_block): - pre_state = store.block_states[signed_block.message.parent_root] - block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT - - if store.time < block_time: - spec.on_tick(store, block_time) - - spec.on_block(store, signed_block) - - -def add_attestation_to_store(spec, store, attestation): - parent_block = store.blocks[attestation.data.beacon_block_root] - pre_state = store.block_states[spec.hash_tree_root(parent_block)] - block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT - next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT - - if store.time < next_epoch_time: - spec.on_tick(store, next_epoch_time) - - spec.on_attestation(store, attestation) - - -def get_anchor_root(spec, state): - anchor_block_header = state.latest_block_header.copy() - if anchor_block_header.state_root == spec.Bytes32(): - anchor_block_header.state_root = spec.hash_tree_root(state) - return spec.hash_tree_root(anchor_block_header) - - @with_all_phases @spec_state_test def test_genesis(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py new file mode 100644 index 000000000..8e72e214e --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -0,0 +1,97 @@ +from eth2spec.utils.ssz.ssz_impl import hash_tree_root + +from eth2spec.test.context import spec_state_test, with_all_phases_except, PHASE0 +from eth2spec.test.helpers.shard_block import ( + build_attestation_with_shard_transition, + build_shard_block, + build_shard_transitions_till_slot, +) +from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root +from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block +from eth2spec.test.helpers.block import build_empty_block + + +def run_on_shard_block(spec, store, shard, signed_block, valid=True): + if not valid: + try: + spec.on_shard_block(store, shard, signed_block) + except AssertionError: + return + else: + assert False + + spec.on_shard_block(store, shard, signed_block) + assert store.shard_blocks[shard][hash_tree_root(signed_block.message)] == signed_block.message + + +def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): + store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + + # Create SignedShardBlock + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_blocks = [shard_block] + + # Attester creates `attestation` + # Use temporary next state to get ShardTransition of shard block + shard_transitions = build_shard_transitions_till_slot( + spec, + state, + shards=[shard, ], + shard_blocks={shard: shard_blocks}, + target_len_offset_slot=1, + ) + shard_transition = shard_transitions[shard] + attestation = build_attestation_with_shard_transition( + spec, + state, + slot=state.slot, + index=committee_index, + target_len_offset_slot=1, + shard_transition=shard_transition, + ) + + # Propose beacon block at slot + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) + beacon_block.body.attestations = [attestation] + beacon_block.body.shard_transitions = shard_transitions + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + + run_on_shard_block(spec, store, shard, shard_block) + add_block_to_store(spec, store, signed_beacon_block) + + assert spec.get_head(store) == beacon_block.hash_tree_root() + assert spec.get_shard_head(store, shard) == shard_block.message.hash_tree_root() + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_basic(spec, state): + spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking + state = spec.upgrade_to_phase1(state) + next_slot(spec, state) + + # Initialization + shard_count = len(state.shard_states) + # Genesis shard blocks + anchor_shard_blocks = { + shard: { + state.shard_states[shard].latest_block_root: spec.ShardBlock( + slot=state.slot, + ) + } + for shard in map(spec.Shard, range(shard_count)) + } + shard_init_slots = { + shard: state.slot + for shard in map(spec.Shard, range(shard_count)) + } + store = spec.get_forkchoice_store(state, shard_init_slots, anchor_shard_blocks) + anchor_root = get_anchor_root(spec, state) + assert spec.get_head(store) == anchor_root + + committee_index = spec.CommitteeIndex(0) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + + run_apply_shard_and_beacon(spec, state, store, shard, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard, committee_index) diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py new file mode 100644 index 000000000..04e36ea84 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -0,0 +1,27 @@ +def get_anchor_root(spec, state): + anchor_block_header = state.latest_block_header.copy() + if anchor_block_header.state_root == spec.Bytes32(): + anchor_block_header.state_root = spec.hash_tree_root(state) + return spec.hash_tree_root(anchor_block_header) + + +def add_block_to_store(spec, store, signed_block): + pre_state = store.block_states[signed_block.message.parent_root] + block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT + + if store.time < block_time: + spec.on_tick(store, block_time) + + spec.on_block(store, signed_block) + + +def add_attestation_to_store(spec, store, attestation): + parent_block = store.blocks[attestation.data.beacon_block_root] + pre_state = store.block_states[spec.hash_tree_root(parent_block)] + block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT + next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT + + if store.time < next_epoch_time: + spec.on_tick(store, next_epoch_time) + + spec.on_attestation(store, attestation)