From dab5a936c4b1fdfae0cc9c9b5c98ac0643097bd7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 Apr 2020 23:55:46 +0800 Subject: [PATCH 01/49] wip shard fork choice rule --- setup.py | 1 + specs/phase1/shard-fork-choice.md | 209 ++++++++++++++++++ .../test/fork_choice/test_get_head.py | 30 +-- .../test/fork_choice/test_on_shard_head.py | 97 ++++++++ .../eth2spec/test/helpers/fork_choice.py | 27 +++ 5 files changed, 335 insertions(+), 29 deletions(-) create mode 100644 specs/phase1/shard-fork-choice.md create mode 100644 tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py create mode 100644 tests/core/pyspec/eth2spec/test/helpers/fork_choice.py diff --git a/setup.py b/setup.py index e0d6561dd..316a7d32a 100644 --- a/setup.py +++ b/setup.py @@ -378,6 +378,7 @@ class PySpecCommand(Command): specs/phase1/shard-transition.md specs/phase1/fork-choice.md specs/phase1/phase1-fork.md + specs/phase1/shard-fork-choice.md """ else: raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md new file mode 100644 index 000000000..46e467e12 --- /dev/null +++ b/specs/phase1/shard-fork-choice.md @@ -0,0 +1,209 @@ +# Ethereum 2.0 Phase 1 -- Beacon Chain + Shard Chain Fork Choice + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + +- [Introduction](#introduction) +- [Fork choice](#fork-choice) + - [Helpers](#helpers) + - [Extended `Store`](#extended-store) + - [Updated `get_forkchoice_store`](#updated-get_forkchoice_store) + - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) + - [`get_shard_head`](#get_shard_head) + - [`get_shard_ancestor`](#get_shard_ancestor) + - [`filter_shard_block_tree`](#filter_shard_block_tree) + - [`get_filtered_block_tree`](#get_filtered_block_tree) + - [Handlers](#handlers) + - [`on_shard_block`](#on_shard_block) + + + +## Introduction + +This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. + +## Fork choice + +### Helpers + +#### Extended `Store` + +```python +@dataclass +class Store(object): + time: uint64 + genesis_time: uint64 + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint + best_justified_checkpoint: Checkpoint + blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) + block_states: Dict[Root, BeaconState] = field(default_factory=dict) + checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) + latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) + # shard chain + shard_init_slots: Dict[Shard, Slot] = field(default_factory=dict) + shard_blocks: Dict[Shard, Dict[Root, ShardBlock]] = field(default_factory=dict) + shard_block_states: Dict[Shard, Dict[Root, ShardState]] = field(default_factory=dict) +``` + +#### Updated `get_forkchoice_store` + +```python +def get_forkchoice_store(anchor_state: BeaconState, + shard_init_slots: Dict[Shard, Slot], + anchor_state_shard_blocks: Dict[Shard, Dict[Root, ShardBlock]]) -> Store: + shard_count = len(anchor_state.shard_states) + anchor_block_header = anchor_state.latest_block_header.copy() + if anchor_block_header.state_root == Bytes32(): + anchor_block_header.state_root = hash_tree_root(anchor_state) + anchor_root = hash_tree_root(anchor_block_header) + anchor_epoch = get_current_epoch(anchor_state) + justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + return Store( + time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, + genesis_time=anchor_state.genesis_time, + justified_checkpoint=justified_checkpoint, + finalized_checkpoint=finalized_checkpoint, + best_justified_checkpoint=justified_checkpoint, + blocks={anchor_root: anchor_block_header}, + block_states={anchor_root: anchor_state.copy()}, + checkpoint_states={justified_checkpoint: anchor_state.copy()}, + # shard chain + shard_init_slots=shard_init_slots, + shard_blocks=anchor_state_shard_blocks, + shard_block_states={ + shard: { + anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard] + } + for shard in map(Shard, range(shard_count)) + }, + ) +``` + +#### `get_shard_latest_attesting_balance` + +```python +def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) -> Gwei: + state = store.checkpoint_states[store.justified_checkpoint] + active_indices = get_active_validator_indices(state, get_current_epoch(state)) + return Gwei(sum( + state.validators[i].effective_balance for i in active_indices + if ( + i in store.latest_messages and get_shard_ancestor( + store, shard, store.latest_messages[i].root, store.shard_blocks[shard][root].slot + ) == root + ) + )) +``` + +#### `get_shard_head` + +```python +def get_shard_head(store: Store, shard: Shard) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_shard_block_tree(store, shard) + + # Execute the LMD-GHOST fork choice + head_beacon_root = get_head(store) + head_shard_root = store.block_states[head_beacon_root].shard_states[shard].latest_block_root + while True: + children = [ + root for root in blocks.keys() + if blocks[root].shard_parent_root == head_shard_root + ] + if len(children) == 0: + return head_shard_root + # Sort by latest attesting balance with ties broken lexicographically + head_shard_root = max(children, key=lambda root: (get_shard_latest_attesting_balance(store, shard, root), root)) +``` + +#### `get_shard_ancestor` + +```python +def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: + block = store.shard_blocks[shard][root] + if block.slot > slot: + return get_shard_ancestor(store, shard, block.shard_parent_root, slot) + elif block.slot == slot: + return root + else: + # root is older than queried slot, thus a skip slot. Return earliest root prior to slot + return root +``` + +#### `filter_shard_block_tree` + +```python +def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: + block = store.shard_blocks[shard][block_root] + children = [ + root for root in store.shard_blocks[shard].keys() + if ( + store.shard_blocks[shard][root].shard_parent_root == block_root + and store.shard_blocks[shard][root].slot != store.shard_init_slots[shard] + ) + ] + + if any(children): + filter_block_tree_result = [filter_shard_block_tree(store, shard, child, blocks) for child in children] + if any(filter_block_tree_result): + blocks[block_root] = block + return True + return False + + return False +``` + +#### `get_filtered_block_tree` + +```python +def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, ShardBlock]: + base_beacon_block_root = get_head(store) + base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root + blocks: Dict[Root, ShardBlock] = {} + filter_shard_block_tree(store, shard, base_shard_block_root, blocks) + return blocks +``` + +### Handlers + +#### `on_shard_block` + +```python +def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: + shard_block = signed_shard_block.message + + # 1. Check shard parent exists + assert shard_block.shard_parent_root in store.shard_block_states[shard] + pre_shard_state = store.shard_block_states[shard][shard_block.shard_parent_root] + + # 2. Check beacon parent exists + assert shard_block.beacon_parent_root in store.block_states + beacon_state = store.block_states[shard_block.beacon_parent_root] + + # 3. Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert shard_block.slot > finalized_slot + + # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + assert ( + shard_block.beacon_parent_root == store.finalized_checkpoint.root + or get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root + ) + + # Add new block to the store + store.shard_blocks[shard][hash_tree_root(shard_block)] = shard_block + + # Check the block is valid and compute the post-state + verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) + verify_shard_block_signature(beacon_state, signed_shard_block) + post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) + # Add new state for this block to the store + store.shard_block_states[shard][hash_tree_root(shard_block)] = post_state +``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py index 17d4f644f..e25aad18f 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py @@ -1,41 +1,13 @@ from eth2spec.test.context import with_all_phases, spec_state_test from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.fork_choice import add_attestation_to_store, add_block_to_store, get_anchor_root from eth2spec.test.helpers.state import ( next_epoch, state_transition_and_sign_block, ) -def add_block_to_store(spec, store, signed_block): - pre_state = store.block_states[signed_block.message.parent_root] - block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT - - if store.time < block_time: - spec.on_tick(store, block_time) - - spec.on_block(store, signed_block) - - -def add_attestation_to_store(spec, store, attestation): - parent_block = store.blocks[attestation.data.beacon_block_root] - pre_state = store.block_states[spec.hash_tree_root(parent_block)] - block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT - next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT - - if store.time < next_epoch_time: - spec.on_tick(store, next_epoch_time) - - spec.on_attestation(store, attestation) - - -def get_anchor_root(spec, state): - anchor_block_header = state.latest_block_header.copy() - if anchor_block_header.state_root == spec.Bytes32(): - anchor_block_header.state_root = spec.hash_tree_root(state) - return spec.hash_tree_root(anchor_block_header) - - @with_all_phases @spec_state_test def test_genesis(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py new file mode 100644 index 000000000..8e72e214e --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -0,0 +1,97 @@ +from eth2spec.utils.ssz.ssz_impl import hash_tree_root + +from eth2spec.test.context import spec_state_test, with_all_phases_except, PHASE0 +from eth2spec.test.helpers.shard_block import ( + build_attestation_with_shard_transition, + build_shard_block, + build_shard_transitions_till_slot, +) +from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root +from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block +from eth2spec.test.helpers.block import build_empty_block + + +def run_on_shard_block(spec, store, shard, signed_block, valid=True): + if not valid: + try: + spec.on_shard_block(store, shard, signed_block) + except AssertionError: + return + else: + assert False + + spec.on_shard_block(store, shard, signed_block) + assert store.shard_blocks[shard][hash_tree_root(signed_block.message)] == signed_block.message + + +def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): + store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH + + # Create SignedShardBlock + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_blocks = [shard_block] + + # Attester creates `attestation` + # Use temporary next state to get ShardTransition of shard block + shard_transitions = build_shard_transitions_till_slot( + spec, + state, + shards=[shard, ], + shard_blocks={shard: shard_blocks}, + target_len_offset_slot=1, + ) + shard_transition = shard_transitions[shard] + attestation = build_attestation_with_shard_transition( + spec, + state, + slot=state.slot, + index=committee_index, + target_len_offset_slot=1, + shard_transition=shard_transition, + ) + + # Propose beacon block at slot + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) + beacon_block.body.attestations = [attestation] + beacon_block.body.shard_transitions = shard_transitions + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + + run_on_shard_block(spec, store, shard, shard_block) + add_block_to_store(spec, store, signed_beacon_block) + + assert spec.get_head(store) == beacon_block.hash_tree_root() + assert spec.get_shard_head(store, shard) == shard_block.message.hash_tree_root() + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_basic(spec, state): + spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking + state = spec.upgrade_to_phase1(state) + next_slot(spec, state) + + # Initialization + shard_count = len(state.shard_states) + # Genesis shard blocks + anchor_shard_blocks = { + shard: { + state.shard_states[shard].latest_block_root: spec.ShardBlock( + slot=state.slot, + ) + } + for shard in map(spec.Shard, range(shard_count)) + } + shard_init_slots = { + shard: state.slot + for shard in map(spec.Shard, range(shard_count)) + } + store = spec.get_forkchoice_store(state, shard_init_slots, anchor_shard_blocks) + anchor_root = get_anchor_root(spec, state) + assert spec.get_head(store) == anchor_root + + committee_index = spec.CommitteeIndex(0) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + + run_apply_shard_and_beacon(spec, state, store, shard, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard, committee_index) diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py new file mode 100644 index 000000000..04e36ea84 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_choice.py @@ -0,0 +1,27 @@ +def get_anchor_root(spec, state): + anchor_block_header = state.latest_block_header.copy() + if anchor_block_header.state_root == spec.Bytes32(): + anchor_block_header.state_root = spec.hash_tree_root(state) + return spec.hash_tree_root(anchor_block_header) + + +def add_block_to_store(spec, store, signed_block): + pre_state = store.block_states[signed_block.message.parent_root] + block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT + + if store.time < block_time: + spec.on_tick(store, block_time) + + spec.on_block(store, signed_block) + + +def add_attestation_to_store(spec, store, attestation): + parent_block = store.blocks[attestation.data.beacon_block_root] + pre_state = store.block_states[spec.hash_tree_root(parent_block)] + block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT + next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT + + if store.time < next_epoch_time: + spec.on_tick(store, next_epoch_time) + + spec.on_attestation(store, attestation) From cddf9cf114744c07d8a1ce455c5cf73ff5ad89db Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 Apr 2020 20:06:20 +0800 Subject: [PATCH 02/49] Refactor --- specs/phase1/shard-fork-choice.md | 56 ++++++++++--------- .../test/fork_choice/test_on_shard_head.py | 18 +----- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 46e467e12..e9026b991 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -35,7 +35,13 @@ This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase ```python @dataclass -class Store(object): +class Store: + + @dataclass + class ShardStore: + blocks: Dict[Root, ShardBlock] = field(default_factory=dict) + block_states: Dict[Root, ShardState] = field(default_factory=dict) + time: uint64 genesis_time: uint64 justified_checkpoint: Checkpoint @@ -46,17 +52,13 @@ class Store(object): checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) # shard chain - shard_init_slots: Dict[Shard, Slot] = field(default_factory=dict) - shard_blocks: Dict[Shard, Dict[Root, ShardBlock]] = field(default_factory=dict) - shard_block_states: Dict[Shard, Dict[Root, ShardState]] = field(default_factory=dict) + shards: Dict[Shard, ShardStore] = field(default_factory=dict) # noqa: F821 ``` #### Updated `get_forkchoice_store` ```python -def get_forkchoice_store(anchor_state: BeaconState, - shard_init_slots: Dict[Shard, Slot], - anchor_state_shard_blocks: Dict[Shard, Dict[Root, ShardBlock]]) -> Store: +def get_forkchoice_store(anchor_state: BeaconState) -> Store: shard_count = len(anchor_state.shard_states) anchor_block_header = anchor_state.latest_block_header.copy() if anchor_block_header.state_root == Bytes32(): @@ -65,6 +67,14 @@ def get_forkchoice_store(anchor_state: BeaconState, anchor_epoch = get_current_epoch(anchor_state) justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + + shard_stores = {} + for shard in map(Shard, range(shard_count)): + shard_stores[shard] = Store.ShardStore( + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, + block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, + ) + return Store( time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, genesis_time=anchor_state.genesis_time, @@ -75,14 +85,7 @@ def get_forkchoice_store(anchor_state: BeaconState, block_states={anchor_root: anchor_state.copy()}, checkpoint_states={justified_checkpoint: anchor_state.copy()}, # shard chain - shard_init_slots=shard_init_slots, - shard_blocks=anchor_state_shard_blocks, - shard_block_states={ - shard: { - anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard] - } - for shard in map(Shard, range(shard_count)) - }, + shards=shard_stores, ) ``` @@ -96,7 +99,7 @@ def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) - state.validators[i].effective_balance for i in active_indices if ( i in store.latest_messages and get_shard_ancestor( - store, shard, store.latest_messages[i].root, store.shard_blocks[shard][root].slot + store, shard, store.latest_messages[i].root, store.shards[shard].blocks[root].slot ) == root ) )) @@ -127,7 +130,7 @@ def get_shard_head(store: Store, shard: Shard) -> Root: ```python def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: - block = store.shard_blocks[shard][root] + block = store.shards[shard].blocks[root] if block.slot > slot: return get_shard_ancestor(store, shard, block.shard_parent_root, slot) elif block.slot == slot: @@ -141,13 +144,11 @@ def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Ro ```python def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: - block = store.shard_blocks[shard][block_root] + shard_store = store.shards[shard] + block = shard_store.blocks[block_root] children = [ - root for root in store.shard_blocks[shard].keys() - if ( - store.shard_blocks[shard][root].shard_parent_root == block_root - and store.shard_blocks[shard][root].slot != store.shard_init_slots[shard] - ) + root for root in shard_store.blocks.keys() + if shard_store.blocks[root].shard_parent_root == block_root ] if any(children): @@ -178,10 +179,11 @@ def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, Shar ```python def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: shard_block = signed_shard_block.message + shard_store = store.shards[shard] # 1. Check shard parent exists - assert shard_block.shard_parent_root in store.shard_block_states[shard] - pre_shard_state = store.shard_block_states[shard][shard_block.shard_parent_root] + assert shard_block.shard_parent_root in shard_store.block_states + pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] # 2. Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states @@ -198,12 +200,12 @@ def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBl ) # Add new block to the store - store.shard_blocks[shard][hash_tree_root(shard_block)] = shard_block + shard_store.blocks[hash_tree_root(shard_block)] = shard_block # Check the block is valid and compute the post-state verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) verify_shard_block_signature(beacon_state, signed_shard_block) post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) # Add new state for this block to the store - store.shard_block_states[shard][hash_tree_root(shard_block)] = post_state + shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 8e72e214e..220c510e7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -21,7 +21,7 @@ def run_on_shard_block(spec, store, shard, signed_block, valid=True): assert False spec.on_shard_block(store, shard, signed_block) - assert store.shard_blocks[shard][hash_tree_root(signed_block.message)] == signed_block.message + assert store.shards[shard].blocks[hash_tree_root(signed_block.message)] == signed_block.message def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): @@ -72,21 +72,7 @@ def test_basic(spec, state): next_slot(spec, state) # Initialization - shard_count = len(state.shard_states) - # Genesis shard blocks - anchor_shard_blocks = { - shard: { - state.shard_states[shard].latest_block_root: spec.ShardBlock( - slot=state.slot, - ) - } - for shard in map(spec.Shard, range(shard_count)) - } - shard_init_slots = { - shard: state.slot - for shard in map(spec.Shard, range(shard_count)) - } - store = spec.get_forkchoice_store(state, shard_init_slots, anchor_shard_blocks) + store = spec.get_forkchoice_store(state) anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root From 8fafb6a9e5b17ee3e93fa0c1b3a8795608f7336b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 May 2020 10:25:58 +0800 Subject: [PATCH 03/49] Make `ShardStore` an independent object --- specs/phase1/shard-fork-choice.md | 97 ++++++------------- .../test/fork_choice/test_on_shard_head.py | 20 ++-- 2 files changed, 43 insertions(+), 74 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index e9026b991..4b3f42194 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -31,75 +31,38 @@ This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase ### Helpers -#### Extended `Store` +#### `ShardStore` ```python @dataclass -class Store: - - @dataclass - class ShardStore: - blocks: Dict[Root, ShardBlock] = field(default_factory=dict) - block_states: Dict[Root, ShardState] = field(default_factory=dict) - - time: uint64 - genesis_time: uint64 - justified_checkpoint: Checkpoint - finalized_checkpoint: Checkpoint - best_justified_checkpoint: Checkpoint - blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) - block_states: Dict[Root, BeaconState] = field(default_factory=dict) - checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) - latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) - # shard chain - shards: Dict[Shard, ShardStore] = field(default_factory=dict) # noqa: F821 +class ShardStore: + shard: Shard + blocks: Dict[Root, ShardBlock] = field(default_factory=dict) + block_states: Dict[Root, ShardState] = field(default_factory=dict) ``` -#### Updated `get_forkchoice_store` +#### Updated `get_forkchoice_shard_store` ```python -def get_forkchoice_store(anchor_state: BeaconState) -> Store: - shard_count = len(anchor_state.shard_states) - anchor_block_header = anchor_state.latest_block_header.copy() - if anchor_block_header.state_root == Bytes32(): - anchor_block_header.state_root = hash_tree_root(anchor_state) - anchor_root = hash_tree_root(anchor_block_header) - anchor_epoch = get_current_epoch(anchor_state) - justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - - shard_stores = {} - for shard in map(Shard, range(shard_count)): - shard_stores[shard] = Store.ShardStore( - blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, - block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, - ) - - return Store( - time=anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot, - genesis_time=anchor_state.genesis_time, - justified_checkpoint=justified_checkpoint, - finalized_checkpoint=finalized_checkpoint, - best_justified_checkpoint=justified_checkpoint, - blocks={anchor_root: anchor_block_header}, - block_states={anchor_root: anchor_state.copy()}, - checkpoint_states={justified_checkpoint: anchor_state.copy()}, - # shard chain - shards=shard_stores, +def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: + return ShardStore( + shard=shard, + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, + block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, ) ``` #### `get_shard_latest_attesting_balance` ```python -def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) -> Gwei: +def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, root: Root) -> Gwei: state = store.checkpoint_states[store.justified_checkpoint] active_indices = get_active_validator_indices(state, get_current_epoch(state)) return Gwei(sum( state.validators[i].effective_balance for i in active_indices if ( i in store.latest_messages and get_shard_ancestor( - store, shard, store.latest_messages[i].root, store.shards[shard].blocks[root].slot + store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot ) == root ) )) @@ -108,13 +71,13 @@ def get_shard_latest_attesting_balance(store: Store, shard: Shard, root: Root) - #### `get_shard_head` ```python -def get_shard_head(store: Store, shard: Shard) -> Root: +def get_shard_head(store: Store, shard_store: ShardStore) -> Root: # Get filtered block tree that only includes viable branches - blocks = get_filtered_shard_block_tree(store, shard) + blocks = get_filtered_shard_block_tree(store, shard_store) # Execute the LMD-GHOST fork choice head_beacon_root = get_head(store) - head_shard_root = store.block_states[head_beacon_root].shard_states[shard].latest_block_root + head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root while True: children = [ root for root in blocks.keys() @@ -123,16 +86,18 @@ def get_shard_head(store: Store, shard: Shard) -> Root: if len(children) == 0: return head_shard_root # Sort by latest attesting balance with ties broken lexicographically - head_shard_root = max(children, key=lambda root: (get_shard_latest_attesting_balance(store, shard, root), root)) + head_shard_root = max( + children, key=lambda root: (get_shard_latest_attesting_balance(store, shard_store, root), root) + ) ``` #### `get_shard_ancestor` ```python -def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Root: - block = store.shards[shard].blocks[root] +def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: Slot) -> Root: + block = shard_store.blocks[root] if block.slot > slot: - return get_shard_ancestor(store, shard, block.shard_parent_root, slot) + return get_shard_ancestor(store, shard_store, block.shard_parent_root, slot) elif block.slot == slot: return root else: @@ -143,8 +108,10 @@ def get_shard_ancestor(store: Store, shard: Shard, root: Root, slot: Slot) -> Ro #### `filter_shard_block_tree` ```python -def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks: Dict[Root, ShardBlock]) -> bool: - shard_store = store.shards[shard] +def filter_shard_block_tree(store: Store, + shard_store: ShardStore, + block_root: Root, + blocks: Dict[Root, ShardBlock]) -> bool: block = shard_store.blocks[block_root] children = [ root for root in shard_store.blocks.keys() @@ -152,7 +119,7 @@ def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks ] if any(children): - filter_block_tree_result = [filter_shard_block_tree(store, shard, child, blocks) for child in children] + filter_block_tree_result = [filter_shard_block_tree(store, shard_store, child, blocks) for child in children] if any(filter_block_tree_result): blocks[block_root] = block return True @@ -164,11 +131,12 @@ def filter_shard_block_tree(store: Store, shard: Shard, block_root: Root, blocks #### `get_filtered_block_tree` ```python -def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, ShardBlock]: +def get_filtered_shard_block_tree(store: Store, shard_store: ShardStore) -> Dict[Root, ShardBlock]: + shard = shard_store.shard base_beacon_block_root = get_head(store) base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root blocks: Dict[Root, ShardBlock] = {} - filter_shard_block_tree(store, shard, base_shard_block_root, blocks) + filter_shard_block_tree(store, shard_store, base_shard_block_root, blocks) return blocks ``` @@ -177,10 +145,9 @@ def get_filtered_shard_block_tree(store: Store, shard: Shard) -> Dict[Root, Shar #### `on_shard_block` ```python -def on_shard_block(store: Store, shard: Shard, signed_shard_block: SignedShardBlock) -> None: +def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: SignedShardBlock) -> None: shard_block = signed_shard_block.message - shard_store = store.shards[shard] - + shard = shard_store.shard # 1. Check shard parent exists assert shard_block.shard_parent_root in shard_store.block_states pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 220c510e7..f4b883f06 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -11,20 +11,21 @@ from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_blo from eth2spec.test.helpers.block import build_empty_block -def run_on_shard_block(spec, store, shard, signed_block, valid=True): +def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): if not valid: try: - spec.on_shard_block(store, shard, signed_block) + spec.on_shard_block(store, shard_store, signed_block) except AssertionError: return else: assert False - spec.on_shard_block(store, shard, signed_block) - assert store.shards[shard].blocks[hash_tree_root(signed_block.message)] == signed_block.message + spec.on_shard_block(store, shard_store, signed_block) + assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): +def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index): + shard = shard_store.shard store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH # Create SignedShardBlock @@ -57,11 +58,11 @@ def run_apply_shard_and_beacon(spec, state, store, shard, committee_index): beacon_block.body.shard_transitions = shard_transitions signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) - run_on_shard_block(spec, store, shard, shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) add_block_to_store(spec, store, signed_beacon_block) assert spec.get_head(store) == beacon_block.hash_tree_root() - assert spec.get_shard_head(store, shard) == shard_block.message.hash_tree_root() + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() @with_all_phases_except([PHASE0]) @@ -78,6 +79,7 @@ def test_basic(spec, state): committee_index = spec.CommitteeIndex(0) shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard_store = spec.get_forkchoice_shard_store(state, shard) - run_apply_shard_and_beacon(spec, state, store, shard, committee_index) - run_apply_shard_and_beacon(spec, state, store, shard, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) + run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) From fca1bbccb948b132fef22a9732c2f0a6aef0b2a4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 May 2020 10:33:23 +0800 Subject: [PATCH 04/49] Remove `get_filtered_shard_block_tree` --- specs/phase1/shard-fork-choice.md | 50 ++++--------------------------- 1 file changed, 5 insertions(+), 45 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 4b3f42194..b026aabfc 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -11,13 +11,11 @@ - [Introduction](#introduction) - [Fork choice](#fork-choice) - [Helpers](#helpers) - - [Extended `Store`](#extended-store) - - [Updated `get_forkchoice_store`](#updated-get_forkchoice_store) + - [`ShardStore`](#shardstore) + - [`get_forkchoice_shard_store`](#get_forkchoice_shard_store) - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) - [`get_shard_head`](#get_shard_head) - [`get_shard_ancestor`](#get_shard_ancestor) - - [`filter_shard_block_tree`](#filter_shard_block_tree) - - [`get_filtered_block_tree`](#get_filtered_block_tree) - [Handlers](#handlers) - [`on_shard_block`](#on_shard_block) @@ -41,7 +39,7 @@ class ShardStore: block_states: Dict[Root, ShardState] = field(default_factory=dict) ``` -#### Updated `get_forkchoice_shard_store` +#### `get_forkchoice_shard_store` ```python def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: @@ -72,16 +70,13 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro ```python def get_shard_head(store: Store, shard_store: ShardStore) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_shard_block_tree(store, shard_store) - # Execute the LMD-GHOST fork choice head_beacon_root = get_head(store) head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root while True: children = [ - root for root in blocks.keys() - if blocks[root].shard_parent_root == head_shard_root + root for root in shard_store.blocks.keys() + if shard_store.blocks[root].shard_parent_root == head_shard_root ] if len(children) == 0: return head_shard_root @@ -105,41 +100,6 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: return root ``` -#### `filter_shard_block_tree` - -```python -def filter_shard_block_tree(store: Store, - shard_store: ShardStore, - block_root: Root, - blocks: Dict[Root, ShardBlock]) -> bool: - block = shard_store.blocks[block_root] - children = [ - root for root in shard_store.blocks.keys() - if shard_store.blocks[root].shard_parent_root == block_root - ] - - if any(children): - filter_block_tree_result = [filter_shard_block_tree(store, shard_store, child, blocks) for child in children] - if any(filter_block_tree_result): - blocks[block_root] = block - return True - return False - - return False -``` - -#### `get_filtered_block_tree` - -```python -def get_filtered_shard_block_tree(store: Store, shard_store: ShardStore) -> Dict[Root, ShardBlock]: - shard = shard_store.shard - base_beacon_block_root = get_head(store) - base_shard_block_root = store.block_states[base_beacon_block_root].shard_states[shard].latest_block_root - blocks: Dict[Root, ShardBlock] = {} - filter_shard_block_tree(store, shard_store, base_shard_block_root, blocks) - return blocks -``` - ### Handlers #### `on_shard_block` From 79b1b4bdbe0c4822d4a2e197a7bf59930ffe69c1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 1 May 2020 10:52:45 +0800 Subject: [PATCH 05/49] Add `(shard, shard_root)` to `LatestMessage` --- specs/phase1/fork-choice.md | 32 ++++++++++++++++++- .../test/fork_choice/test_on_attestation.py | 13 ++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md index d8bf7fa09..f4e771ddb 100644 --- a/specs/phase1/fork-choice.md +++ b/specs/phase1/fork-choice.md @@ -10,6 +10,9 @@ - [Introduction](#introduction) - [Fork choice](#fork-choice) + - [Helpers](#helpers) + - [Extended `LatestMessage`](#extended-latestmessage) + - [Updated `update_latest_messages`](#updated-update_latest_messages) - [Handlers](#handlers) @@ -25,6 +28,33 @@ Due to the changes in the structure of `IndexedAttestation` in Phase 1, `on_atte The rest of the fork choice remains stable. +### Helpers + +#### Extended `LatestMessage` + +```python +@dataclass(eq=True, frozen=True) +class LatestMessage(object): + epoch: Epoch + root: Root + shard: Shard + shard_root: Root +``` + +#### Updated `update_latest_messages` + +```python +def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: + target = attestation.data.target + beacon_block_root = attestation.data.beacon_block_root + shard = get_shard(store.block_states[beacon_block_root], attestation) + for i in attesting_indices: + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage( + epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.head_shard_root + ) +``` + ### Handlers ```python @@ -49,4 +79,4 @@ def on_attestation(store: Store, attestation: Attestation) -> None: if attestation.aggregation_bits[i] ] update_latest_messages(store, attesting_indices, attestation) -``` \ No newline at end of file +``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 360c18ccd..1bce42ca3 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -18,18 +18,25 @@ def run_on_attestation(spec, state, store, attestation, valid=True): if spec.fork == PHASE0: sample_index = indexed_attestation.attesting_indices[0] + latest_message = spec.LatestMessage( + epoch=attestation.data.target.epoch, + root=attestation.data.beacon_block_root, + ) else: attesting_indices = [ index for i, index in enumerate(indexed_attestation.committee) if attestation.aggregation_bits[i] ] sample_index = attesting_indices[0] - assert ( - store.latest_messages[sample_index] == - spec.LatestMessage( + latest_message = spec.LatestMessage( epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, + shard=spec.get_shard(state, attestation), + shard_root=attestation.data.head_shard_root, ) + + assert ( + store.latest_messages[sample_index] == latest_message ) From 56309342c05d324e359591b902abc355fba82fb1 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 21 May 2020 15:33:47 +0200 Subject: [PATCH 06/49] Use `Bytes32` for `error_message` `ErrorMessage.error_message` is a leftover from an older version of SSZ that was able to encode unbounded lists. This is no longer the case - all collection types now have a fixed upper bound on length. In general, the `error_message`, just like the `graffitti` field, should not be interpreted in any particular way except for debugging and vanity - as such, using the same type, a `Bytes32`, seems reasonable. An alternative would be `List[byte, 256]` which maybe could be "reasonably backwards compatible" with whatever clients are are doing now - depending on how they are dealing with this field type that no longer exists in the SSZ spec :) It would however be the only place where `List[uintN, N]` is used in the current spec. As an exercise, this could be considered a security issue since it's essentially unbounded and undefined behaviour. --- specs/phase0/p2p-interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 118be8839..84db361c9 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -391,11 +391,11 @@ The `ErrorMessage` schema is: ``` ( - error_message: String + error_message: Bytes32 ) ``` -*Note*: The String type is encoded as UTF-8 bytes without NULL terminator when SSZ-encoded. As the `ErrorMessage` is not an SSZ-container, only the UTF-8 bytes will be sent when SSZ-encoded. +*Note*: By convention, the `error_message` is a sequence of bytes that can be interpreted as a UTF-8 string up to 32 bytes - a 0 byte shortens the string in this interpretation. Clients MUST treat as valid any bytes. ### Encoding strategies From 2dc041807af913f16e54cf44a46fef07f80dbe2e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 29 May 2020 23:50:18 +0800 Subject: [PATCH 07/49] Implement `get_start_shard` --- setup.py | 15 ++++- specs/phase1/beacon-chain.md | 61 ++++++++++++++++++- specs/phase1/phase1-fork.md | 1 + .../test_process_crosslink.py | 2 +- .../test/phase_1/sanity/test_blocks.py | 4 +- 5 files changed, 74 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 4f3f2b872..d0f063b33 100644 --- a/setup.py +++ b/setup.py @@ -140,7 +140,7 @@ SUNDRY_CONSTANTS_FUNCTIONS = ''' def ceillog2(x: uint64) -> int: return (x - 1).bit_length() ''' -SUNDRY_FUNCTIONS = ''' +PHASE0_SUNDRY_FUNCTIONS = ''' # Monkey patch hash cache _hash = hash hash_cache: Dict[bytes, Bytes32] = {} @@ -220,6 +220,13 @@ get_attesting_indices = cache_this( _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)''' +PHASE1_SUNDRY_FUNCTIONS = ''' +_get_start_shard = get_start_shard +get_start_shard = cache_this( + lambda state, slot: (state.validators.hash_tree_root(), slot), + _get_start_shard, lru_size=SLOTS_PER_EPOCH * 3)''' + + def objects_to_spec(spec_object: SpecObject, imports: str, fork: str) -> str: """ Given all the objects that constitute a spec, combine them into a single pyfile. @@ -250,9 +257,11 @@ def objects_to_spec(spec_object: SpecObject, imports: str, fork: str) -> str: + '\n\n' + CONFIG_LOADER + '\n\n' + ssz_objects_instantiation_spec + '\n\n' + functions_spec - + '\n' + SUNDRY_FUNCTIONS - + '\n' + + '\n' + PHASE0_SUNDRY_FUNCTIONS ) + if fork == 'phase1': + spec += '\n' + PHASE1_SUNDRY_FUNCTIONS + spec += '\n' return spec diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 39a73c0aa..124442b59 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -47,6 +47,7 @@ - [`get_light_client_committee`](#get_light_client_committee) - [`get_shard_proposer_index`](#get_shard_proposer_index) - [`get_indexed_attestation`](#get_indexed_attestation) + - [`get_committee_count_delta`](#get_committee_count_delta) - [`get_start_shard`](#get_start_shard) - [`get_shard`](#get_shard) - [`get_latest_slot_for_shard`](#get_latest_slot_for_shard) @@ -69,6 +70,7 @@ - [Shard transition false positives](#shard-transition-false-positives) - [Light client processing](#light-client-processing) - [Epoch transition](#epoch-transition) + - [Phase 1 final updates](#phase-1-final-updates) - [Custody game updates](#custody-game-updates) - [Online-tracking](#online-tracking) - [Light client committee updates](#light-client-committee-updates) @@ -280,6 +282,7 @@ class BeaconState(Container): current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint # Phase 1 + current_epoch_start_shard: Shard shard_states: List[ShardState, MAX_SHARDS] online_countdown: List[OnlineEpochs, VALIDATOR_REGISTRY_LIMIT] # not a raw byte array, considered its large size. current_light_committee: CompactCommittee @@ -530,18 +533,53 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) ) ``` +#### `get_committee_count_delta` + +```python +def get_committee_count_delta(state: BeaconState, start_slot: Slot, stop_slot: Slot) -> uint64: + """ + Return the sum of committee counts between ``[start_slot, stop_slot)``. + """ + committee_sum = 0 + for slot in range(start_slot, stop_slot): + count = get_committee_count_at_slot(state, Slot(slot)) + committee_sum += count + return committee_sum +``` + #### `get_start_shard` ```python def get_start_shard(state: BeaconState, slot: Slot) -> Shard: - # TODO: implement start shard logic - return Shard(0) + """ + Return the start shard at ``slot``. + """ + current_epoch_start_slot = compute_start_slot_at_epoch(get_current_epoch(state)) + active_shard_count = get_active_shard_count(state) + if current_epoch_start_slot == slot: + return state.current_epoch_start_shard + elif current_epoch_start_slot > slot: + # Current epoch or the next epoch lookahead + shard_delta = get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) + return Shard((state.current_epoch_start_shard + shard_delta) % active_shard_count) + else: + # Previous epoch + shard_delta = get_committee_count_delta(state, start_slot=slot, stop_slot=current_epoch_start_slot) + max_committees_per_epoch = MAX_COMMITTEES_PER_SLOT * SLOTS_PER_EPOCH + return Shard( + # Ensure positive + (state.current_epoch_start_shard + max_committees_per_epoch * active_shard_count - shard_delta) + % active_shard_count + ) ``` #### `get_shard` ```python def get_shard(state: BeaconState, attestation: Attestation) -> Shard: + """ + Return the shard that the given attestation is attesting. + """ return compute_shard_from_committee_index(state, attestation.data.index, attestation.data.slot) ``` @@ -549,6 +587,9 @@ def get_shard(state: BeaconState, attestation: Attestation) -> Shard: ```python def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: + """ + Return the latest slot number of the given shard. + """ return state.shard_states[shard].slot ``` @@ -556,7 +597,11 @@ def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: ```python def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]: - return compute_offset_slots(state.shard_states[shard].slot, state.slot) + """ + Return the offset slots of the given shard. + The offset slot are after the latest slot and before current slot. + """ + return compute_offset_slots(get_latest_slot_for_shard(state, shard), state.slot) ``` ### Predicates @@ -993,9 +1038,19 @@ def process_epoch(state: BeaconState) -> None: process_reveal_deadlines(state) process_slashings(state) process_final_updates(state) + process_phase_1_final_updates(state) +``` + +#### Phase 1 final updates + +```python +def process_phase_1_final_updates(state: BeaconState) -> None: process_custody_final_updates(state) process_online_tracking(state) process_light_client_committee_updates(state) + + # Update current_epoch_start_shard + state.current_epoch_start_shard = get_start_shard(state, state.slot) ``` #### Custody game updates diff --git a/specs/phase1/phase1-fork.md b/specs/phase1/phase1-fork.md index cc7d8f33e..d362ed633 100644 --- a/specs/phase1/phase1-fork.md +++ b/specs/phase1/phase1-fork.md @@ -99,6 +99,7 @@ def upgrade_to_phase1(pre: phase0.BeaconState) -> BeaconState: current_justified_checkpoint=pre.current_justified_checkpoint, finalized_checkpoint=pre.finalized_checkpoint, # Phase 1 + current_epoch_start_shard=Shard(0), shard_states=List[ShardState, MAX_SHARDS]( ShardState( slot=pre.slot, diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py index 1f066b344..af0ff1a90 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_crosslink.py @@ -18,7 +18,7 @@ def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): # At the beginning, let `x = state.slot`, `state.shard_states[shard].slot == x - 1` slot_x = state.slot committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) assert state.shard_states[shard].slot == slot_x - 1 # Create SignedShardBlock diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 60af35d45..1499d34cd 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -67,7 +67,7 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) assert state.shard_states[shard].slot == state.slot - 1 pre_gasprice = state.shard_states[shard].gasprice @@ -93,7 +93,7 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) assert state.shard_states[shard].slot == state.slot - 1 # No new shard block From 870ad8b921705cb658ebb0b88ff4e08bdeb54609 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 30 May 2020 03:56:56 +0800 Subject: [PATCH 08/49] Fix test --- .../pyspec/eth2spec/test/fork_choice/test_on_shard_head.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index f4b883f06..5b4205e91 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -30,6 +30,7 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + target_len_offset_slot = 1 shard_block = build_shard_block(spec, state, shard, body=body, signed=True) shard_blocks = [shard_block] @@ -38,17 +39,15 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) shard_transitions = build_shard_transitions_till_slot( spec, state, - shards=[shard, ], shard_blocks={shard: shard_blocks}, - target_len_offset_slot=1, + on_time_slot=state.slot + target_len_offset_slot, ) shard_transition = shard_transitions[shard] attestation = build_attestation_with_shard_transition( spec, state, - slot=state.slot, index=committee_index, - target_len_offset_slot=1, + on_time_slot=state.slot + target_len_offset_slot, shard_transition=shard_transition, ) From 33e115f8c65407357852f1605399bfaf79f933cb Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Sat, 30 May 2020 20:59:12 +0200 Subject: [PATCH 09/49] Clean up remaining `[]` List syntax ..and use List[byte, 256] for error message --- specs/phase0/p2p-interface.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 84db361c9..a77f1851a 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -150,6 +150,7 @@ This section outlines constants that are used in this spec. | Name | Value | Description | |---|---|---| | `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum allowed size of uncompressed gossip messages. | +| `MAX_REQUEST_BLOCKS` | `2**10` (= 1024) | Maximum number of blocks in a single request | | `MAX_CHUNK_SIZE` | `2**20` (1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | | `TTFB_TIMEOUT` | `5s` | The maximum time to wait for first byte of request response (time-to-first-byte). | | `RESP_TIMEOUT` | `10s` | The maximum time for complete response transfer. | @@ -391,11 +392,11 @@ The `ErrorMessage` schema is: ``` ( - error_message: Bytes32 + error_message: List[byte, 256] ) ``` -*Note*: By convention, the `error_message` is a sequence of bytes that can be interpreted as a UTF-8 string up to 32 bytes - a 0 byte shortens the string in this interpretation. Clients MUST treat as valid any bytes. +*Note*: By convention, the `error_message` is a sequence of bytes that MAY be interpreted as a UTF-8 string (for debugging purposes). Clients MUST treat as valid any byte sequences. ### Encoding strategies @@ -443,9 +444,9 @@ In case of an invalid input (header or payload), a reader MUST: All messages that contain only a single field MUST be encoded directly as the type of that field and MUST NOT be encoded as an SSZ container. -Responses that are SSZ-lists (for example `[]SignedBeaconBlock`) send their +Responses that are SSZ-lists (for example `List[SignedBeaconBlock, ...]`) send their constituents individually as `response_chunk`s. For example, the -`[]SignedBeaconBlock` response type sends zero or more `response_chunk`s. Each _successful_ `response_chunk` contains a single `SignedBeaconBlock` payload. +`List[SignedBeaconBlock, ...]` response type sends zero or more `response_chunk`s. Each _successful_ `response_chunk` contains a single `SignedBeaconBlock` payload. ### Messages @@ -528,7 +529,7 @@ Request Content: Response Content: ``` ( - []SignedBeaconBlock + List[SignedBeaconBlock, MAX_REQUEST_BLOCKS] ) ``` @@ -545,7 +546,7 @@ The response MUST consist of zero or more `response_chunk`. Each _successful_ `r Clients MUST keep a record of signed blocks seen since the since the start of the weak subjectivity period and MUST support serving requests of blocks up to their own `head_block_root`. -Clients MUST respond with at least the first block that exists in the range, if they have it. +Clients MUST respond with at least the first block that exists in the range, if they have it, and no more than `MAX_REQUEST_BLOCKS` blocks. The following blocks, where they exist, MUST be send in consecutive order. @@ -568,7 +569,7 @@ Request Content: ``` ( - []Root + List[Root, MAX_REQUEST_BLOCKS] ) ``` @@ -576,12 +577,14 @@ Response Content: ``` ( - []SignedBeaconBlock + List[SignedBeaconBlock, MAX_REQUEST_BLOCKS] ) ``` Requests blocks by block root (= `hash_tree_root(SignedBeaconBlock.message)`). The response is a list of `SignedBeaconBlock` whose length is less than or equal to the number of requested blocks. It may be less in the case that the responding peer is missing blocks. +No more than `MAX_REQUEST_BLOCKS` may be requested at a time. + `BeaconBlocksByRoot` is primarily used to recover recent blocks (e.g. when receiving a block or attestation whose parent is unknown). The request MUST be encoded as an SSZ-field. From 92db6da50854071eecea138576e9771a5004a596 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jun 2020 17:56:22 +0800 Subject: [PATCH 10/49] Apply suggestions from Terence Co-authored-by: terence tsao --- specs/phase1/beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 124442b59..1367d029b 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -578,7 +578,7 @@ def get_start_shard(state: BeaconState, slot: Slot) -> Shard: ```python def get_shard(state: BeaconState, attestation: Attestation) -> Shard: """ - Return the shard that the given attestation is attesting. + Return the shard that the given ``attestation`` is attesting. """ return compute_shard_from_committee_index(state, attestation.data.index, attestation.data.slot) ``` @@ -588,7 +588,7 @@ def get_shard(state: BeaconState, attestation: Attestation) -> Shard: ```python def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: """ - Return the latest slot number of the given shard. + Return the latest slot number of the given ``shard``. """ return state.shard_states[shard].slot ``` @@ -598,7 +598,7 @@ def get_latest_slot_for_shard(state: BeaconState, shard: Shard) -> Slot: ```python def get_offset_slots(state: BeaconState, shard: Shard) -> Sequence[Slot]: """ - Return the offset slots of the given shard. + Return the offset slots of the given ``shard``. The offset slot are after the latest slot and before current slot. """ return compute_offset_slots(get_latest_slot_for_shard(state, shard), state.slot) From 30f72dd69646d037a06943d026e2d91ef2b9c2c5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jun 2020 23:15:16 +0800 Subject: [PATCH 11/49] Fix `get_shard` and `compute_shard_from_committee_index` calls --- specs/phase1/beacon-chain.md | 25 ++++++------- .../eth2spec/test/helpers/attestations.py | 37 +++++++++---------- .../eth2spec/test/helpers/shard_block.py | 2 +- .../test/helpers/shard_transitions.py | 15 ++++++++ .../test/phase_0/sanity/test_blocks.py | 20 +++++++--- .../test_process_shard_transition.py | 9 ++--- .../test/phase_1/sanity/test_blocks.py | 4 +- 7 files changed, 67 insertions(+), 45 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 228cd888e..52b6f2afb 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -578,7 +578,7 @@ def get_start_shard(state: BeaconState, slot: Slot) -> Shard: active_shard_count = get_active_shard_count(state) if current_epoch_start_slot == slot: return state.current_epoch_start_shard - elif current_epoch_start_slot > slot: + elif slot > current_epoch_start_slot: # Current epoch or the next epoch lookahead shard_delta = get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) return Shard((state.current_epoch_start_shard + shard_delta) % active_shard_count) @@ -693,8 +693,7 @@ def is_on_time_attestation(state: BeaconState, """ Check if the given attestation is on-time. """ - # TODO: MIN_ATTESTATION_INCLUSION_DELAY should always be 1 - return attestation.data.slot + MIN_ATTESTATION_INCLUSION_DELAY == state.slot + return attestation.data.slot == compute_previous_slot(state.slot) ``` #### `is_winning_attestation` @@ -803,20 +802,19 @@ def validate_attestation(state: BeaconState, attestation: Attestation) -> None: else: assert attestation.data.source == state.previous_justified_checkpoint - shard = get_shard(state, attestation) - # Type 1: on-time attestations, the custody bits should be non-empty. if attestation.custody_bits_blocks != []: # Ensure on-time attestation assert is_on_time_attestation(state, attestation) # Correct data root count + shard = get_shard(state, attestation) assert len(attestation.custody_bits_blocks) == len(get_offset_slots(state, shard)) # Correct parent block root assert data.beacon_block_root == get_block_root_at_slot(state, compute_previous_slot(state.slot)) # Type 2: no shard transition, no custody bits else: # Ensure delayed attestation - assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY < state.slot + assert data.slot < compute_previous_slot(state.slot) # Late attestations cannot have a shard transition root assert data.shard_transition_root == Root() @@ -911,9 +909,10 @@ def process_crosslink_for_shard(state: BeaconState, committee_index: CommitteeIndex, shard_transition: ShardTransition, attestations: Sequence[Attestation]) -> Root: - committee = get_beacon_committee(state, state.slot, committee_index) + on_time_attestation_slot = compute_previous_slot(state.slot) + committee = get_beacon_committee(state, on_time_attestation_slot, committee_index) online_indices = get_online_validator_indices(state) - shard = compute_shard_from_committee_index(state, committee_index, state.slot) + shard = compute_shard_from_committee_index(state, committee_index, on_time_attestation_slot) # Loop over all shard transition roots shard_transition_roots = set([a.data.shard_transition_root for a in attestations]) @@ -968,15 +967,15 @@ def process_crosslink_for_shard(state: BeaconState, def process_crosslinks(state: BeaconState, shard_transitions: Sequence[ShardTransition], attestations: Sequence[Attestation]) -> None: - committee_count = get_committee_count_at_slot(state, state.slot) + on_time_attestation_slot = compute_previous_slot(state.slot) + committee_count = get_committee_count_at_slot(state, on_time_attestation_slot) for committee_index in map(CommitteeIndex, range(committee_count)): - shard = compute_shard_from_committee_index(state, committee_index, state.slot) # All attestations in the block for this committee/shard and current slot shard_attestations = [ attestation for attestation in attestations if is_on_time_attestation(state, attestation) and attestation.data.index == committee_index ] - + shard = compute_shard_from_committee_index(state, committee_index, on_time_attestation_slot) winning_root = process_crosslink_for_shard(state, committee_index, shard_transitions[shard], shard_attestations) if winning_root != Root(): # Mark relevant pending attestations as creating a successful crosslink @@ -1083,7 +1082,7 @@ def process_epoch(state: BeaconState) -> None: process_registry_updates(state) process_reveal_deadlines(state) process_slashings(state) - process_final_updates(state) + process_final_updates(state) # phase 0 final updates process_phase_1_final_updates(state) ``` @@ -1096,7 +1095,7 @@ def process_phase_1_final_updates(state: BeaconState) -> None: process_light_client_committee_updates(state) # Update current_epoch_start_shard - state.current_epoch_start_shard = get_start_shard(state, state.slot) + state.current_epoch_start_shard = get_start_shard(state, Slot(state.slot + 1)) ``` #### Custody game updates diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index c533182ef..1372b0654 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -1,8 +1,9 @@ from typing import List from eth2spec.test.context import expect_assertion_error, PHASE0, PHASE1 -from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot, transition_to +from eth2spec.test.helpers.state import state_transition_and_sign_block, next_epoch, next_slot from eth2spec.test.helpers.block import build_empty_block_for_next_slot +from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.ssz.ssz_typing import Bitlist @@ -81,12 +82,12 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() else: - # No shard transition + # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: temp_state = state.copy() next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, []) + shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[]) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() @@ -180,7 +181,7 @@ def get_valid_attestation(spec, fill_aggregate_attestation(spec, state, attestation, signed=signed, filter_participant_set=filter_participant_set) if spec.fork == PHASE1 and on_time: - attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed) + attestation = convert_to_valid_on_time_attestation(spec, state, attestation, signed=signed) return attestation @@ -317,7 +318,19 @@ def next_epoch_with_attestations(spec, committees_per_slot = spec.get_committee_count_at_slot(state, slot_to_attest) if slot_to_attest >= spec.compute_start_slot_at_epoch(spec.get_current_epoch(post_state)): for index in range(committees_per_slot): - cur_attestation = get_valid_attestation(spec, post_state, slot_to_attest, index=index, signed=True) + if spec.fork == PHASE1: + shard = spec.compute_shard_from_committee_index(post_state, index, slot_to_attest) + shard_transition = get_shard_transition_of_committee( + spec, post_state, index, slot=slot_to_attest + ) + block.body.shard_transitions[shard] = shard_transition + else: + shard_transition = None + + cur_attestation = get_valid_attestation( + spec, post_state, slot_to_attest, + shard_transition=shard_transition, index=index, signed=True, on_time=True + ) block.body.attestations.append(cur_attestation) if fill_prev_epoch: @@ -328,9 +341,6 @@ def next_epoch_with_attestations(spec, spec, post_state, slot_to_attest, index=index, signed=True, on_time=False) block.body.attestations.append(prev_attestation) - if spec.fork == PHASE1: - fill_block_shard_transitions_by_attestations(spec, post_state, block) - signed_block = state_transition_and_sign_block(spec, post_state, block) signed_blocks.append(signed_block) @@ -396,14 +406,3 @@ def cached_prepare_state_with_attestations(spec, state): # Put the LRU cache result into the state view, as if we transitioned the original view state.set_backing(_prep_state_cache_dict[key]) - - -def fill_block_shard_transitions_by_attestations(spec, state, block): - block.body.shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS - for attestation in block.body.attestations: - shard = spec.get_shard(state, attestation) - if attestation.data.slot == state.slot: - temp_state = state.copy() - transition_to(spec, temp_state, slot=block.slot) - shard_transition = spec.get_shard_transition(temp_state, shard, []) - block.body.shard_transitions[shard] = shard_transition diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 805b955f7..58efada83 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -70,7 +70,7 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): return shard_transitions -def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition=None): +def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition): temp_state = state.copy() transition_to(spec, temp_state, on_time_slot - 1) attestation = get_valid_on_time_attestation( diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index 4ac0ddcfb..abb5e7278 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -1,4 +1,5 @@ from eth2spec.test.context import expect_assertion_error +from eth2spec.test.helpers.state import transition_to def run_shard_transitions_processing(spec, state, shard_transitions, attestations, valid=True): @@ -26,3 +27,17 @@ def run_shard_transitions_processing(spec, state, shard_transitions, attestation # yield post-state yield 'post', state + + +def get_shard_transition_of_committee(spec, state, committee_index, slot=None, shard_blocks=None): + if shard_blocks is None: + shard_blocks = [] + + if slot is None: + slot = state.slot + + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + temp_state = state.copy() + transition_to(spec, temp_state, slot + 1) + shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=shard_blocks) + return shard_transition diff --git a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py index f0cfc462e..1e057e5a9 100644 --- a/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_0/sanity/test_blocks.py @@ -16,8 +16,9 @@ from eth2spec.test.helpers.attester_slashings import ( get_indexed_attestation_participants, ) from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing, check_proposer_slashing_effect -from eth2spec.test.helpers.attestations import get_valid_attestation, fill_block_shard_transitions_by_attestations +from eth2spec.test.helpers.attestations import get_valid_attestation from eth2spec.test.helpers.deposits import prepare_state_and_deposit +from eth2spec.test.helpers.shard_transitions import get_shard_transition_of_committee from eth2spec.test.context import ( spec_state_test, with_all_phases, expect_assertion_error, always_bls, with_phases, @@ -687,14 +688,23 @@ def test_attestation(spec, state): yield 'pre', state - attestation = get_valid_attestation(spec, state, signed=True, on_time=True) + attestation_block = build_empty_block(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) + + index = 0 + if spec.fork == PHASE1: + shard = spec.compute_shard_from_committee_index(state, index, state.slot) + shard_transition = get_shard_transition_of_committee(spec, state, index) + attestation_block.body.shard_transitions[shard] = shard_transition + else: + shard_transition = None + + attestation = get_valid_attestation( + spec, state, shard_transition=shard_transition, index=index, signed=True, on_time=True + ) # Add to state via block transition pre_current_attestations_len = len(state.current_epoch_attestations) - attestation_block = build_empty_block(spec, state, state.slot + spec.MIN_ATTESTATION_INCLUSION_DELAY) attestation_block.body.attestations.append(attestation) - if spec.fork == PHASE1: - fill_block_shard_transitions_by_attestations(spec, state, attestation_block) signed_attestation_block = state_transition_and_sign_block(spec, state, attestation_block) assert len(state.current_epoch_attestations) == pre_current_attestations_len + 1 diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index ba408cd48..00ffbe0a8 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -15,11 +15,10 @@ from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): state = transition_to_valid_shard_slot(spec, state) - # At the beginning, let `x = state.slot`, `state.shard_states[shard].slot == x - 1` - slot_x = state.slot + init_slot = state.slot committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) - assert state.shard_states[shard].slot == slot_x - 1 + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) + assert state.shard_states[shard].slot == state.slot - 1 # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE @@ -50,7 +49,7 @@ def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): if valid: # After state transition, - assert state.slot == slot_x + target_len_offset_slot + assert state.slot == init_slot + target_len_offset_slot shard_state = state.shard_states[shard] assert shard_state != pre_shard_state assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 1499d34cd..0175bd40d 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -67,7 +67,7 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 pre_gasprice = state.shard_states[shard].gasprice @@ -93,7 +93,7 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): target_len_offset_slot = 1 committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot) + shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 # No new shard block From 5f10ac13bf91c181152a5469562dfb9d89383926 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 1 Jun 2020 23:22:59 +0800 Subject: [PATCH 12/49] PR feedback from Terence and Danny: refactor `get_committee_count_delta` --- specs/phase1/beacon-chain.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 52b6f2afb..484b24cf1 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -558,13 +558,9 @@ def get_indexed_attestation(beacon_state: BeaconState, attestation: Attestation) ```python def get_committee_count_delta(state: BeaconState, start_slot: Slot, stop_slot: Slot) -> uint64: """ - Return the sum of committee counts between ``[start_slot, stop_slot)``. + Return the sum of committee counts in range ``[start_slot, stop_slot)``. """ - committee_sum = 0 - for slot in range(start_slot, stop_slot): - count = get_committee_count_at_slot(state, Slot(slot)) - committee_sum += count - return committee_sum + return sum(get_committee_count_at_slot(state, Slot(slot)) for slot in range(start_slot, stop_slot)) ``` #### `get_start_shard` From 2a218520a1338aca2601cd90ffe2d31f64475495 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Mon, 1 Jun 2020 14:31:45 -0700 Subject: [PATCH 13/49] Tidying up `shard_state_transition` --- specs/phase1/shard-transition.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index ce6d289bd..f9de34b31 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -71,21 +71,22 @@ def verify_shard_block_signature(beacon_state: BeaconState, def shard_state_transition(beacon_state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: - # Update shard state + """ + Update ``shard_state`` with shard ``block`` and ``beacon_state`. + """ + shard_state.slot = block.slot prev_gasprice = shard_state.gasprice - if len(block.body) == 0: - latest_block_root = shard_state.latest_block_root - else: - latest_block_root = hash_tree_root(block) - + shard_state.gasprice = compute_updated_gasprice(prev_gasprice, len(block.body)) shard_state.transition_digest = compute_shard_transition_digest( beacon_state, shard_state, block.beacon_parent_root, hash_tree_root(block.body), ) - shard_state.gasprice = compute_updated_gasprice(prev_gasprice, len(block.body)) - shard_state.slot = block.slot + if len(block.body) == 0: + latest_block_root = shard_state.latest_block_root + else: + latest_block_root = hash_tree_root(block) shard_state.latest_block_root = latest_block_root ``` From c6aac1650600407a832fa9646362873ebb785d72 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 2 Jun 2020 17:16:25 +1000 Subject: [PATCH 14/49] Reword fork choice comment --- specs/phase0/fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index baf7b7b7b..3f1c20c0a 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -285,7 +285,7 @@ def validate_on_attestation(store: Store, attestation: Attestation) -> None: # Attestations must not be for blocks in the future. If not, the attestation should not be considered assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot - # FFG and LMD vote must be consistent with each other + # LMD vote must be consistent with FFG target target_slot = compute_start_slot_at_epoch(target.epoch) assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) From 2a125e8497c37b8685ba5478be254105df2542d3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 2 Jun 2020 17:22:33 +1000 Subject: [PATCH 15/49] Update fork-choice.md --- specs/phase0/fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 3f1c20c0a..b9d8ecd3c 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -285,7 +285,7 @@ def validate_on_attestation(store: Store, attestation: Attestation) -> None: # Attestations must not be for blocks in the future. If not, the attestation should not be considered assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot - # LMD vote must be consistent with FFG target + # LMD vote must be consistent with FFG vote target target_slot = compute_start_slot_at_epoch(target.epoch) assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) From 142ba17451e05b0e6478796fe9bea9eb2de4c488 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 2 Jun 2020 18:08:28 +0800 Subject: [PATCH 16/49] PR review from Danny --- specs/phase1/shard-fork-choice.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index b026aabfc..867730203 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -59,7 +59,9 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro return Gwei(sum( state.validators[i].effective_balance for i in active_indices if ( - i in store.latest_messages and get_shard_ancestor( + i in store.latest_messages and + store.latest_messages[i].shard == shard_store.shard and + get_shard_ancestor( store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot ) == root ) @@ -71,12 +73,18 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro ```python def get_shard_head(store: Store, shard_store: ShardStore) -> Root: # Execute the LMD-GHOST fork choice + shard_blocks = shard_store.blocks head_beacon_root = get_head(store) - head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root + head_shard_state = store.block_states[head_beacon_root].shard_states[shard_store.shard] + head_shard_root = head_shard_state.latest_block_root while True: + # Find the valid child block roots children = [ root for root in shard_store.blocks.keys() - if shard_store.blocks[root].shard_parent_root == head_shard_root + if ( + shard_blocks[root].shard_parent_root == head_shard_root + and shard_blocks[root].slot > head_shard_state.slot + ) ] if len(children) == 0: return head_shard_root @@ -116,14 +124,15 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si assert shard_block.beacon_parent_root in store.block_states beacon_state = store.block_states[shard_block.beacon_parent_root] - # 3. Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert shard_block.slot > finalized_slot + # 3. Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) + finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] + finalized_shard_state = finalized_beacon_state.shard_states[shard] + assert shard_block.slot > finalized_shard_state.slot # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert ( - shard_block.beacon_parent_root == store.finalized_checkpoint.root - or get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root + get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root ) # Add new block to the store From 24427947b17060623047fca2fbc0b49d8b636fe7 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Tue, 2 Jun 2020 08:09:43 -0700 Subject: [PATCH 17/49] Moved transition_digest to last --- specs/phase1/shard-transition.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index f9de34b31..e6221a980 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -77,17 +77,17 @@ def shard_state_transition(beacon_state: BeaconState, shard_state.slot = block.slot prev_gasprice = shard_state.gasprice shard_state.gasprice = compute_updated_gasprice(prev_gasprice, len(block.body)) + if len(block.body) == 0: + latest_block_root = shard_state.latest_block_root + else: + latest_block_root = hash_tree_root(block) + shard_state.latest_block_root = latest_block_root shard_state.transition_digest = compute_shard_transition_digest( beacon_state, shard_state, block.beacon_parent_root, hash_tree_root(block.body), ) - if len(block.body) == 0: - latest_block_root = shard_state.latest_block_root - else: - latest_block_root = hash_tree_root(block) - shard_state.latest_block_root = latest_block_root ``` We have a pure function `get_post_shard_state` for describing the fraud proof verification and honest validator behavior. From 34412130c5a8df907680649adbae58ae45f3c0bc Mon Sep 17 00:00:00 2001 From: Raw Pong Ghmoa <58883403+q9f@users.noreply.github.com> Date: Mon, 1 Jun 2020 13:59:03 +0200 Subject: [PATCH 18/49] phase0: enable a tunable genesis time --- specs/phase0/beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/phase0/beacon-chain.md b/specs/phase0/beacon-chain.md index 1be32da44..e4de2d38b 100644 --- a/specs/phase0/beacon-chain.md +++ b/specs/phase0/beacon-chain.md @@ -218,7 +218,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `MIN_GENESIS_DELAY` | `86400` | seconds | 1 day | +| `GENESIS_DELAY` | `172800` | seconds | 2 days | | `SECONDS_PER_SLOT` | `12` | seconds | 12 seconds | | `SECONDS_PER_ETH1_BLOCK` | `14` | seconds | 14 seconds | | `MIN_ATTESTATION_INCLUSION_DELAY` | `2**0` (= 1) | slots | 12 seconds | @@ -1137,7 +1137,7 @@ Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 b - `eth1_timestamp` is the Unix timestamp corresponding to `eth1_block_hash` - `deposits` is the sequence of all deposits, ordered chronologically, up to (and including) the block with hash `eth1_block_hash` -Eth1 blocks must only be considered once they are at least `SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE` seconds old (i.e. `eth1_timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= current_unix_time`). Due to this constraint, if `MIN_GENESIS_DELAY < SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE`, then the `genesis_time` can happen before the time/state is first known. Values should be configured to avoid this case. +Eth1 blocks must only be considered once they are at least `SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE` seconds old (i.e. `eth1_timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= current_unix_time`). Due to this constraint, if `GENESIS_DELAY < SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE`, then the `genesis_time` can happen before the time/state is first known. Values should be configured to avoid this case. ```python def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, @@ -1149,7 +1149,7 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, epoch=GENESIS_EPOCH, ) state = BeaconState( - genesis_time=eth1_timestamp - eth1_timestamp % MIN_GENESIS_DELAY + 2 * MIN_GENESIS_DELAY, + genesis_time=eth1_timestamp + GENESIS_DELAY, fork=fork, eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=len(deposits)), latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), From b3f2d81ad5063f728c4620ed5a9435b5dfbe8853 Mon Sep 17 00:00:00 2001 From: Raw Pong Ghmoa <58883403+q9f@users.noreply.github.com> Date: Mon, 1 Jun 2020 14:02:29 +0200 Subject: [PATCH 19/49] reflect changes in mainnet.yaml --- configs/mainnet.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 42845c235..3631b7045 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -76,8 +76,8 @@ BLS_WITHDRAWAL_PREFIX: 0x00 # Time parameters # --------------------------------------------------------------- -# 86400 seconds (1 day) -MIN_GENESIS_DELAY: 86400 +# 172800 seconds (2 days) +GENESIS_DELAY: 172800 # 12 seconds SECONDS_PER_SLOT: 12 # 2**0 (= 1) slots 12 seconds From d5ed78e974aa97e22c4695d25f82165a5cc46310 Mon Sep 17 00:00:00 2001 From: Raw Pong Ghmoa <58883403+q9f@users.noreply.github.com> Date: Mon, 1 Jun 2020 14:03:07 +0200 Subject: [PATCH 20/49] reflect changes in minimal.yaml --- configs/minimal.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index d8e346ffa..5abf6c93c 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -77,7 +77,7 @@ BLS_WITHDRAWAL_PREFIX: 0x00 # Time parameters # --------------------------------------------------------------- # [customized] Faster to spin up testnets, but does not give validator reasonable warning time for genesis -MIN_GENESIS_DELAY: 300 +GENESIS_DELAY: 300 # [customized] Faster for testing purposes SECONDS_PER_SLOT: 6 # 2**0 (= 1) slots 6 seconds From 671fae6efe83e0fe7e0e1256d2c0199daf7c42ed Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 2 Jun 2020 11:09:42 -0600 Subject: [PATCH 21/49] change note about genesis delay in p2p spec to match new GENESIS_DELAY config value; fix tests --- specs/phase0/p2p-interface.md | 2 +- .../core/pyspec/eth2spec/test/genesis/test_initialization.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index a994b0c66..e47fafed0 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -1052,7 +1052,7 @@ discv5 uses ENRs and we will presumably need to: Although client software might very well be running locally prior to the solidification of the eth2 genesis state and block, clients cannot form valid ENRs prior to this point. ENRs contain `fork_digest` which utilizes the `genesis_validators_root` for a cleaner separation between chains so prior to knowing genesis, we cannot use `fork_digest` to cleanly find peers on our intended chain. Once genesis data is known, we can then form ENRs and safely find peers. -When using an eth1 deposit contract for deposits, `fork_digest` will be known at least `MIN_GENESIS_DELAY` (24 hours in mainnet configuration) before `genesis_time`, providing ample time to find peers and form initial connections and gossip subnets prior to genesis. +When using an eth1 deposit contract for deposits, `fork_digest` will be known `GENESIS_DELAY` (48hours in mainnet configuration) before `genesis_time`, providing ample time to find peers and form initial connections and gossip subnets prior to genesis. ## Compression/Encoding diff --git a/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py b/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py index 882821337..faade2d17 100644 --- a/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py +++ b/tests/core/pyspec/eth2spec/test/genesis/test_initialization.py @@ -21,7 +21,7 @@ def test_initialize_beacon_state_from_eth1(spec): # initialize beacon_state state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits) - assert state.genesis_time == eth1_timestamp - eth1_timestamp % spec.MIN_GENESIS_DELAY + 2 * spec.MIN_GENESIS_DELAY + assert state.genesis_time == eth1_timestamp + spec.GENESIS_DELAY assert len(state.validators) == deposit_count assert state.eth1_data.deposit_root == deposit_root assert state.eth1_data.deposit_count == deposit_count @@ -57,7 +57,7 @@ def test_initialize_beacon_state_some_small_balances(spec): # initialize beacon_state state = spec.initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits) - assert state.genesis_time == eth1_timestamp - eth1_timestamp % spec.MIN_GENESIS_DELAY + 2 * spec.MIN_GENESIS_DELAY + assert state.genesis_time == eth1_timestamp + spec.GENESIS_DELAY assert len(state.validators) == small_deposit_count assert state.eth1_data.deposit_root == deposit_root assert state.eth1_data.deposit_count == len(deposits) From 06dfff022bf05f63e703e1f0d1031d530edf4fb4 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 2 Jun 2020 11:21:00 -0600 Subject: [PATCH 22/49] add comment about default of 0x00 for genesis finalized checkpoint in p2p spec --- specs/phase0/p2p-interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index a994b0c66..c7d61d8f1 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -468,9 +468,9 @@ The fields are, as seen by the client at the time of sending the message: - `fork_digest`: The node's `ForkDigest` (`compute_fork_digest(current_fork_version, genesis_validators_root)`) where - `current_fork_version` is the fork version at the node's current epoch defined by the wall-clock time (not necessarily the epoch to which the node is sync) - `genesis_validators_root` is the static `Root` found in `state.genesis_validators_root` -- `finalized_root`: `state.finalized_checkpoint.root` for the state corresponding to the head block. +- `finalized_root`: `state.finalized_checkpoint.root` for the state corresponding to the head block (Note this defaults to `Root(b'\x00' * 32)` for the genesis finalized checkpoint). - `finalized_epoch`: `state.finalized_checkpoint.epoch` for the state corresponding to the head block. -- `head_root`: The hash_tree_root root of the current head block. +- `head_root`: The `hash_tree_root` root of the current head block. - `head_slot`: The slot of the block corresponding to the `head_root`. The dialing client MUST send a `Status` request upon connection. From cf7b9993b5ea1124ddd87579dc27a78ebf092e92 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 2 Jun 2020 13:51:43 -0600 Subject: [PATCH 23/49] clarify `head_root` is for a `BeaconBlock` Co-authored-by: Diederik Loerakker --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index c7d61d8f1..e98070b6f 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -470,7 +470,7 @@ The fields are, as seen by the client at the time of sending the message: - `genesis_validators_root` is the static `Root` found in `state.genesis_validators_root` - `finalized_root`: `state.finalized_checkpoint.root` for the state corresponding to the head block (Note this defaults to `Root(b'\x00' * 32)` for the genesis finalized checkpoint). - `finalized_epoch`: `state.finalized_checkpoint.epoch` for the state corresponding to the head block. -- `head_root`: The `hash_tree_root` root of the current head block. +- `head_root`: The `hash_tree_root` root of the current head block (`BeaconBlock`). - `head_slot`: The slot of the block corresponding to the `head_root`. The dialing client MUST send a `Status` request upon connection. From 314dea97a5018740c5166e4884777c936f5d5150 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 2 Jun 2020 17:26:26 -0600 Subject: [PATCH 24/49] bump VERSION.txt to 0.12.1 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index d33c3a212..aac2dacab 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -0.12.0 \ No newline at end of file +0.12.1 \ No newline at end of file From 5c5cedd60d1c08583d8627e93db1b1b179f6b7f4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 3 Jun 2020 22:31:16 +0800 Subject: [PATCH 25/49] Apply PR feedback from Danny and Terence --- specs/phase1/shard-fork-choice.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 867730203..3b6bc5ac9 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -6,7 +6,7 @@ -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* + - [Introduction](#introduction) - [Fork choice](#fork-choice) @@ -23,7 +23,7 @@ ## Introduction -This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. +This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1. It assumes the [beacon chain fork choice spec](./fork-choice.md). ## Fork choice @@ -59,9 +59,9 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro return Gwei(sum( state.validators[i].effective_balance for i in active_indices if ( - i in store.latest_messages and - store.latest_messages[i].shard == shard_store.shard and - get_shard_ancestor( + i in store.latest_messages + and store.latest_messages[i].shard == shard_store.shard + and get_shard_ancestor( store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot ) == root ) @@ -116,20 +116,25 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: SignedShardBlock) -> None: shard_block = signed_shard_block.message shard = shard_store.shard - # 1. Check shard parent exists + + # Check shard + # TODO: check it in networking spec + assert shard_block.shard == shard + + # Check shard parent exists assert shard_block.shard_parent_root in shard_store.block_states pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] - # 2. Check beacon parent exists + # Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states beacon_state = store.block_states[shard_block.beacon_parent_root] - # 3. Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) + # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] finalized_shard_state = finalized_beacon_state.shard_states[shard] assert shard_block.slot > finalized_shard_state.slot - # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert ( get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root From 68e934bf1552517346c1b31f1fb9e181b51d82cb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 3 Jun 2020 23:08:38 +0800 Subject: [PATCH 26/49] Add `get_start_shard` unittests and update minimal config 1. Add unittests for testing `get_start_shard` with better granularity 2. Change `INITIAL_ACTIVE_SHARDS` from `4` to `2` for tight crosslinking --- configs/minimal.yaml | 2 +- .../test/phase_1/unittests/__init__.py | 0 .../phase_1/unittests/test_get_start_shard.py | 71 +++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 tests/core/pyspec/eth2spec/test/phase_1/unittests/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py diff --git a/configs/minimal.yaml b/configs/minimal.yaml index d8e346ffa..bb46294f5 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -165,7 +165,7 @@ PHASE_1_FORK_VERSION: 0x01000001 # [customized] for testing PHASE_1_GENESIS_SLOT: 8 # [customized] reduced for testing -INITIAL_ACTIVE_SHARDS: 4 +INITIAL_ACTIVE_SHARDS: 2 # Phase 1: General diff --git a/tests/core/pyspec/eth2spec/test/phase_1/unittests/__init__.py b/tests/core/pyspec/eth2spec/test/phase_1/unittests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py new file mode 100644 index 000000000..f9f605d9f --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py @@ -0,0 +1,71 @@ +from eth2spec.test.context import ( + PHASE0, + with_all_phases_except, + spec_state_test, +) +from eth2spec.test.helpers.state import next_epoch + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_committee_count_delta(spec, state): + assert spec.get_committee_count_delta(state, 0, 0) == 0 + assert spec.get_committee_count_at_slot(state, 0) != 0 + assert spec.get_committee_count_delta(state, 0, 1) == spec.get_committee_count_at_slot(state, 0) + assert spec.get_committee_count_delta(state, 1, 2) == spec.get_committee_count_at_slot(state, 1) + assert spec.get_committee_count_delta(state, 0, 2) == ( + spec.get_committee_count_at_slot(state, 0) + spec.get_committee_count_at_slot(state, 1) + ) + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_start_shard_current_epoch_start(spec, state): + assert state.current_epoch_start_shard == 0 + next_epoch(spec, state) + active_shard_count = spec.get_active_shard_count(state) + assert state.current_epoch_start_shard == ( + spec.get_committee_count_delta(state, 0, spec.SLOTS_PER_EPOCH) % active_shard_count + ) + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + + slot = current_epoch_start_slot + start_shard = spec.get_start_shard(state, slot) + assert start_shard == state.current_epoch_start_shard + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_start_shard_next_slot(spec, state): + next_epoch(spec, state) + active_shard_count = spec.get_active_shard_count(state) + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + + slot = current_epoch_start_slot + 1 + start_shard = spec.get_start_shard(state, slot) + + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + expected_start_shard = ( + state.current_epoch_start_shard + + spec.get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) + ) % active_shard_count + assert start_shard == expected_start_shard + + +@with_all_phases_except([PHASE0]) +@spec_state_test +def test_get_start_shard_previous_slot(spec, state): + next_epoch(spec, state) + active_shard_count = spec.get_active_shard_count(state) + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + + slot = current_epoch_start_slot - 1 + start_shard = spec.get_start_shard(state, slot) + + current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) + expected_start_shard = ( + state.current_epoch_start_shard + + spec.MAX_COMMITTEES_PER_SLOT * spec.SLOTS_PER_EPOCH * active_shard_count + - spec.get_committee_count_delta(state, start_slot=slot, stop_slot=current_epoch_start_slot) + ) % active_shard_count + assert start_shard == expected_start_shard From a685be3bbecb18d7bf336af911a638d68e6b64f9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 00:42:08 +0800 Subject: [PATCH 27/49] PR feedback from Danny Co-authored-by: Danny Ryan --- .../eth2spec/test/phase_1/unittests/test_get_start_shard.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py index f9f605d9f..27afd4a4e 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/unittests/test_get_start_shard.py @@ -46,8 +46,8 @@ def test_get_start_shard_next_slot(spec, state): current_epoch_start_slot = spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)) expected_start_shard = ( - state.current_epoch_start_shard + - spec.get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) + state.current_epoch_start_shard + + spec.get_committee_count_delta(state, start_slot=current_epoch_start_slot, stop_slot=slot) ) % active_shard_count assert start_shard == expected_start_shard From e1981a7bfdeb2baf296cbfdb1f74169b56341883 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 01:00:52 +0800 Subject: [PATCH 28/49] `head_shard_root` -> `shard_head_root` --- specs/phase1/fork-choice.md | 2 +- specs/phase1/shard-fork-choice.md | 8 ++++---- .../eth2spec/test/fork_choice/test_on_attestation.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/phase1/fork-choice.md b/specs/phase1/fork-choice.md index f4e771ddb..3f9fbdbfb 100644 --- a/specs/phase1/fork-choice.md +++ b/specs/phase1/fork-choice.md @@ -51,7 +51,7 @@ def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIn for i in attesting_indices: if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: store.latest_messages[i] = LatestMessage( - epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.head_shard_root + epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.shard_head_root ) ``` diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 3b6bc5ac9..fb98893ac 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -76,20 +76,20 @@ def get_shard_head(store: Store, shard_store: ShardStore) -> Root: shard_blocks = shard_store.blocks head_beacon_root = get_head(store) head_shard_state = store.block_states[head_beacon_root].shard_states[shard_store.shard] - head_shard_root = head_shard_state.latest_block_root + shard_head_root = head_shard_state.latest_block_root while True: # Find the valid child block roots children = [ root for root in shard_store.blocks.keys() if ( - shard_blocks[root].shard_parent_root == head_shard_root + shard_blocks[root].shard_parent_root == shard_head_root and shard_blocks[root].slot > head_shard_state.slot ) ] if len(children) == 0: - return head_shard_root + return shard_head_root # Sort by latest attesting balance with ties broken lexicographically - head_shard_root = max( + shard_head_root = max( children, key=lambda root: (get_shard_latest_attesting_balance(store, shard_store, root), root) ) ``` diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py index 04d2588d9..a5334c5c7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py @@ -32,7 +32,7 @@ def run_on_attestation(spec, state, store, attestation, valid=True): epoch=attestation.data.target.epoch, root=attestation.data.beacon_block_root, shard=spec.get_shard(state, attestation), - shard_root=attestation.data.head_shard_root, + shard_root=attestation.data.shard_head_root, ) assert ( From c0108afe7713ac10542c478b8d07514bdad42c3b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:06:04 +0800 Subject: [PATCH 29/49] Use shard_block.slot to get seed for proposer selection --- specs/phase1/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 484b24cf1..fccb93f55 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -538,7 +538,8 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) - r = bytes_to_int(get_seed(beacon_state, get_current_epoch(beacon_state), DOMAIN_SHARD_COMMITTEE)[:8]) + epoch = compute_epoch_at_slot(slot) + r = bytes_to_int(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE)[:8]) return committee[r % len(committee)] ``` From d3445217416ad3a8f73dbf45e4669970d063682a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:30:13 +0800 Subject: [PATCH 30/49] Bugfix: should set `shard` for empty proposal --- specs/phase1/shard-transition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index e6221a980..5e8616568 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -221,7 +221,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, validate_signature=validate_signature, ) if len(choices) == 0: - block = ShardBlock(slot=slot) + block = ShardBlock(slot=slot, shard=shard) proposal = SignedShardBlock(message=block) elif len(choices) == 1: proposal = choices[0] From 26aae40941aac98d731655a475e3085e777c0110 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:31:53 +0800 Subject: [PATCH 31/49] Use epoch of the shard_block.slot for generating seed --- specs/phase1/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index 484b24cf1..fccb93f55 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -538,7 +538,8 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) - r = bytes_to_int(get_seed(beacon_state, get_current_epoch(beacon_state), DOMAIN_SHARD_COMMITTEE)[:8]) + epoch = compute_epoch_at_slot(slot) + r = bytes_to_int(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE)[:8]) return committee[r % len(committee)] ``` From 376c83619024caee0dfd5f1c7da552ca78cedc10 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 4 Jun 2020 12:49:22 +1000 Subject: [PATCH 32/49] Clarify proposer slashing gossip conditions --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 9949db66f..767748e89 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -263,7 +263,7 @@ Additional global topics are used to propagate lower frequency validator message - _[IGNORE]_ The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. - _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. The following validations MUST pass before forwarding the `proposer_slashing` on to the network - - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.index`. + - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.header_1.proposer_index`. - _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. - _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). From 8afb93f5a36adac5ef312ee3fa4678632df2b6c4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 11:19:04 +0800 Subject: [PATCH 33/49] Add `shard_block.slot` to seed --- specs/phase1/beacon-chain.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/specs/phase1/beacon-chain.md b/specs/phase1/beacon-chain.md index fccb93f55..4719d2e6f 100644 --- a/specs/phase1/beacon-chain.md +++ b/specs/phase1/beacon-chain.md @@ -537,9 +537,13 @@ def get_light_client_committee(beacon_state: BeaconState, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard) -> ValidatorIndex: - committee = get_shard_committee(beacon_state, compute_epoch_at_slot(slot), shard) + """ + Return the proposer's index of shard block at ``slot``. + """ epoch = compute_epoch_at_slot(slot) - r = bytes_to_int(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE)[:8]) + committee = get_shard_committee(beacon_state, epoch, shard) + seed = hash(get_seed(beacon_state, epoch, DOMAIN_SHARD_COMMITTEE) + int_to_bytes(slot, length=8)) + r = bytes_to_int(seed[:8]) return committee[r % len(committee)] ``` From c2b7ff7422e449dc2486261adad6196259015579 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 4 Jun 2020 16:13:49 +1000 Subject: [PATCH 34/49] Re-clarify proposer slashing check Fixes a typo from #1871 --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 767748e89..bdf0919e0 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -263,7 +263,7 @@ Additional global topics are used to propagate lower frequency validator message - _[IGNORE]_ The voluntary exit is the first valid voluntary exit received for the validator with index `signed_voluntary_exit.message.validator_index`. - _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. - `proposer_slashing` - This topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. The following validations MUST pass before forwarding the `proposer_slashing` on to the network - - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.header_1.proposer_index`. + - _[IGNORE]_ The proposer slashing is the first valid proposer slashing received for the proposer with index `proposer_slashing.signed_header_1.message.proposer_index`. - _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. - `attester_slashing` - This topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. - _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation has not yet been seen in any prior `attester_slashing` (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). From 7e44456be543193b3c4848faead8ea9f7baa1583 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 5 Jun 2020 09:50:49 -0600 Subject: [PATCH 35/49] mod compute_subnet_for_attestation to be usable for lookahead --- specs/phase0/p2p-interface.md | 4 ++-- specs/phase0/validator.md | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index bdf0919e0..6c4cdd2a6 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -275,7 +275,7 @@ Additional global topics are used to propagate lower frequency validator message Attestation subnets are used to propagate unaggregated attestations to subsections of the network. Their `Name`s are: - `beacon_attestation_{subnet_id}` - These topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. The following validations MUST pass before forwarding the `attestation` on the subnet. - - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation) == subnet_id`). + - _[REJECT]_ The attestation is for the correct subnet (i.e. `compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.index) == subnet_id`). - _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot). - _[REJECT]_ The attestation is unaggregated -- that is, it has exactly one participating validator (`len(get_attesting_indices(state, attestation.data, attestation.aggregation_bits)) == 1`). - _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index. @@ -286,7 +286,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. The correct subnet for an attestation can be calculated with `compute_subnet_for_attestation`. `beacon_attestation_{subnet_id}` topics, are rotated through throughout the epoch in a similar fashion to rotating through shards in committees in Phase 1. -Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` as `Attestation`s. +Unaggregated attestations are sent to the subnet topic, `beacon_attestation_{compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.index)}` as `Attestation`s. Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `AggregateAndProof`s. diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 5e8ddc977..35c52f346 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -199,8 +199,8 @@ The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahe Specifically a validator should: * Call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. -* Find peers of the pubsub topic `committee_index{committee_index % ATTESTATION_SUBNET_COUNT}_beacon_attestation`. - * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][committee_index % ATTESTATION_SUBNET_COUNT] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field. +* Find peers of the pubsub topic `beacon_attestation_{compute_subnet_for_attestation(state, slot, committee_index)}`. + * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][compute_subnet_for_attestation(state, slot, committee_index)] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field. * If the validator is assigned to be an aggregator for the slot (see `is_aggregator()`), then subscribe to the topic. *Note*: If the validator is _not_ assigned to be an aggregator, the validator only needs sufficient number of peers on the topic to be able to publish messages. The validator does not need to _subscribe_ and listen to all messages on the topic. @@ -425,18 +425,18 @@ def get_attestation_signature(state: BeaconState, attestation_data: AttestationD #### Broadcast attestation -Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation)}` pubsub topic. +Finally, the validator broadcasts `attestation` to the associated attestation subnet -- the `beacon_attestation_{compute_subnet_for_attestation(state, attestation.data.slot, attestation.data.committee_index)}` pubsub topic. ```python -def compute_subnet_for_attestation(state: BeaconState, attestation: Attestation) -> uint64: +def compute_subnet_for_attestation(state: BeaconState, slot: Slot, committee_index: CommitteeIndex) -> uint64: """ Compute the correct subnet for an attestation for Phase 0. Note, this mimics expected Phase 1 behavior where attestations will be mapped to their shard subnet. """ - slots_since_epoch_start = attestation.data.slot % SLOTS_PER_EPOCH - committees_since_epoch_start = get_committee_count_at_slot(state, attestation.data.slot) * slots_since_epoch_start + slots_since_epoch_start = slot % SLOTS_PER_EPOCH + committees_since_epoch_start = get_committee_count_at_slot(state, slot) * slots_since_epoch_start - return (committees_since_epoch_start + attestation.data.index) % ATTESTATION_SUBNET_COUNT + return (committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT ``` ### Attestation aggregation From c9a53b8039cb2eada12cbb09ca1cd74509356549 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 05:33:01 +0800 Subject: [PATCH 36/49] WIP test case --- .../test/fork_choice/test_on_shard_head.py | 91 ++++++++++++------- .../eth2spec/test/helpers/shard_block.py | 26 ++++-- 2 files changed, 78 insertions(+), 39 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 5b4205e91..1151d18d7 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -5,6 +5,7 @@ from eth2spec.test.helpers.shard_block import ( build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, + get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block @@ -24,44 +25,65 @@ def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index): +def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): shard = shard_store.shard + committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) + has_shard_committee = committee_index is not None store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH # Create SignedShardBlock - body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - target_len_offset_slot = 1 - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) - shard_blocks = [shard_block] + # Check offsets + temp_state = state.copy() + next_slot(spec, temp_state) + offset_slots = spec.get_offset_slots(temp_state, shard) + if state.slot in offset_slots: + # Build block + body = b'\x56' * 4 + shard_head_root = spec.get_shard_head(store, shard_store) + shard_parent_state = shard_store.block_states[shard_head_root] + assert shard_parent_state.slot != state.slot + shard_block = build_shard_block( + spec, state, shard, + shard_parent_state=shard_parent_state, slot=state.slot, body=body, signed=True + ) + shard_blocks_buffer.append(shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) # Attester creates `attestation` - # Use temporary next state to get ShardTransition of shard block - shard_transitions = build_shard_transitions_till_slot( - spec, - state, - shard_blocks={shard: shard_blocks}, - on_time_slot=state.slot + target_len_offset_slot, - ) - shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( - spec, - state, - index=committee_index, - on_time_slot=state.slot + target_len_offset_slot, - shard_transition=shard_transition, - ) + if has_shard_committee and len(shard_blocks_buffer) > 0: + # Use temporary next state to get ShardTransition of shard block + shard_transitions = build_shard_transitions_till_slot( + spec, + state, + shard_blocks={shard: shard_blocks_buffer}, + on_time_slot=state.slot + 1, + ) + shard_transition = shard_transitions[shard] + + attestation = build_attestation_with_shard_transition( + spec, + state, + index=committee_index, + on_time_slot=state.slot + 1, + shard_transition=shard_transition, + ) + assert attestation.data.slot == state.slot + assert spec.get_shard(state, attestation) == shard + beacon_block.body.attestations = [attestation] + beacon_block.body.shard_transitions = shard_transitions - # Propose beacon block at slot - beacon_block = build_empty_block(spec, state, slot=state.slot + 1) - beacon_block.body.attestations = [attestation] - beacon_block.body.shard_transitions = shard_transitions signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) - run_on_shard_block(spec, store, shard_store, shard_block) add_block_to_store(spec, store, signed_beacon_block) - assert spec.get_head(store) == beacon_block.hash_tree_root() - assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + if has_shard_committee: + shard_blocks_buffer = [] # clear buffer + + return has_shard_committee, shard_blocks_buffer @with_all_phases_except([PHASE0]) @@ -69,16 +91,19 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) def test_basic(spec, state): spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking state = spec.upgrade_to_phase1(state) - next_slot(spec, state) # Initialization store = spec.get_forkchoice_store(state) anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root - committee_index = spec.CommitteeIndex(0) - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) + shard = spec.Shard(1) shard_store = spec.get_forkchoice_shard_store(state, shard) - - run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) - run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index) + shard_block_count = 2 + shard_blocks_buffer = [] + while shard_block_count > 0: + has_shard_committee, shard_blocks_buffer = run_apply_shard_and_beacon( + spec, state, store, shard_store, shard_blocks_buffer + ) + if has_shard_committee: + shard_block_count -= 1 diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 58efada83..410213edd 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -23,19 +23,24 @@ def build_shard_block(spec, shard, slot=None, body=None, + shard_parent_state=None, signed=False): - shard_state = beacon_state.shard_states[shard] + if shard_parent_state is None: + shard_parent_state = beacon_state.shard_states[shard] + if slot is None: - slot = shard_state.slot + 1 + slot = shard_parent_state.slot + 1 if body is None: body = b'\x56' * 128 - proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) + temp_state = beacon_state.copy() + transition_to(spec, temp_state, slot) beacon_state, beacon_parent_root = get_state_and_beacon_parent_root_at_slot(spec, beacon_state, slot) - + assert beacon_state == temp_state + proposer_index = spec.get_shard_proposer_index(temp_state, slot, shard) block = spec.ShardBlock( - shard_parent_root=shard_state.latest_block_root, + shard_parent_root=shard_parent_state.latest_block_root, beacon_parent_root=beacon_parent_root, slot=slot, shard=shard, @@ -59,7 +64,6 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): for shard, blocks in shard_blocks.items(): offset_slots = spec.get_offset_slots(temp_state, shard) len_offset_slots = len(offset_slots) - assert len_offset_slots == on_time_slot - state.shard_states[shard].slot - 1 shard_transition = spec.get_shard_transition(temp_state, shard, blocks) if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() @@ -84,3 +88,13 @@ def build_attestation_with_shard_transition(spec, state, index, on_time_slot, sh if shard_transition is not None: assert attestation.data.shard_transition_root == shard_transition.hash_tree_root() return attestation + + +def get_committee_index_of_shard(spec, state, slot, shard): # Optional[CommitteeIndex] + active_shard_count = spec.get_active_shard_count(state) + committee_count = spec.get_committee_count_at_slot(state, slot) + start_shard = spec.get_start_shard(state, slot) + for committee_index in range(committee_count): + if (start_shard + committee_index) % active_shard_count == shard: + return committee_index + return None From 727353c054f259aa81e7738099fc055722e9734c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 18:39:27 +0800 Subject: [PATCH 37/49] Verify shard_block.slot fits the expected offset_slots --- specs/phase1/shard-fork-choice.md | 9 +++++---- specs/phase1/shard-transition.md | 3 +++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index fb98893ac..b60edd948 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -127,7 +127,6 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si # Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states - beacon_state = store.block_states[shard_block.beacon_parent_root] # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] @@ -144,9 +143,11 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si shard_store.blocks[hash_tree_root(shard_block)] = shard_block # Check the block is valid and compute the post-state - verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard) - verify_shard_block_signature(beacon_state, signed_shard_block) - post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block) + beacon_head_root = get_head(store) + beacon_head_state = store.block_states[beacon_head_root] + assert verify_shard_block_message(beacon_head_state, pre_shard_state, shard_block, shard_block.slot, shard) + assert verify_shard_block_signature(beacon_head_state, signed_shard_block) + post_state = get_post_shard_state(beacon_head_state, pre_shard_state, shard_block) # Add new state for this block to the store shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 5e8616568..6c4f652d2 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -50,6 +50,9 @@ def verify_shard_block_message(beacon_state: BeaconState, shard: Shard) -> bool: assert block.shard_parent_root == shard_state.latest_block_root assert block.slot == slot + next_slot = Slot(beacon_state.slot + 1) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), next_slot) + assert slot in offset_slots assert block.shard == shard assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE From f8597d296545c6711d1c7cf8fe25d5dbb5b1987b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 20:31:54 +0800 Subject: [PATCH 38/49] Add `get_pendings_shard_blocks` --- specs/phase1/shard-fork-choice.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index b60edd948..5bc2cbe4e 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -16,6 +16,7 @@ - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) - [`get_shard_head`](#get_shard_head) - [`get_shard_ancestor`](#get_shard_ancestor) + - [`get_pendings_shard_blocks`](#get_pendings_shard_blocks) - [Handlers](#handlers) - [`on_shard_block`](#on_shard_block) @@ -108,6 +109,31 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: return root ``` +#### `get_pendings_shard_blocks` + +```python +def get_pendings_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: + """ + Return the shard blocks branch that from shard head to beacon head. + """ + shard = shard_store.shard + + beacon_head_root = get_head(store) + beacon_head_state = store.block_states[beacon_head_root] + latest_shard_block_root = beacon_head_state.shard_states[shard].latest_block_root + + shard_head_root = get_shard_head(store, shard_store) + root = shard_head_root + shard_blocks = [] + while root != latest_shard_block_root: + shard_block = shard_store.blocks[root] + shard_blocks.append(shard_block) + root = shard_block.shard_parent_root + + shard_blocks.reverse() + return shard_blocks +``` + ### Handlers #### `on_shard_block` From ab42eee4c04818d58fbcea5ba83d1c5890d0c3b8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 20:32:31 +0800 Subject: [PATCH 39/49] Update shard fork choice rule to be able to handle mainnet config --- .../test/fork_choice/test_on_shard_head.py | 97 ++++++++++++------- 1 file changed, 62 insertions(+), 35 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 1151d18d7..7e94ddd8e 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -1,6 +1,6 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root -from eth2spec.test.context import spec_state_test, with_all_phases_except, PHASE0 +from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls from eth2spec.test.helpers.shard_block import ( build_attestation_with_shard_transition, build_shard_block, @@ -8,7 +8,7 @@ from eth2spec.test.helpers.shard_block import ( get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root -from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block +from eth2spec.test.helpers.state import state_transition_and_sign_block from eth2spec.test.helpers.block import build_empty_block @@ -25,35 +25,51 @@ def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): +def apply_shard_block(spec, store, shard_store, beacon_head_state, shard_blocks_buffer): shard = shard_store.shard - committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) - has_shard_committee = committee_index is not None + body = b'\x56' * 4 + shard_head_root = spec.get_shard_head(store, shard_store) + shard_parent_state = shard_store.block_states[shard_head_root] + assert shard_parent_state.slot != beacon_head_state.slot + shard_block = build_shard_block( + spec, beacon_head_state, shard, + shard_parent_state=shard_parent_state, slot=beacon_head_state.slot, body=body, signed=True + ) + shard_blocks_buffer.append(shard_block) + run_on_shard_block(spec, store, shard_store, shard_block) + assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + + +def check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer): + pending_shard_blocks = [ + spec.SignedShardBlock(message=b) + for b in spec.get_pendings_shard_blocks(store, shard_store) + ] + assert pending_shard_blocks == shard_blocks_buffer + + +def is_in_offset_sets(spec, beacon_head_state, shard): + offset_slots = spec.compute_offset_slots( + beacon_head_state.shard_states[shard].slot, beacon_head_state.slot + 1 + ) + return beacon_head_state.slot in offset_slots + + +def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer): store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH - # Create SignedShardBlock - # Check offsets - temp_state = state.copy() - next_slot(spec, temp_state) - offset_slots = spec.get_offset_slots(temp_state, shard) - if state.slot in offset_slots: - # Build block - body = b'\x56' * 4 - shard_head_root = spec.get_shard_head(store, shard_store) - shard_parent_state = shard_store.block_states[shard_head_root] - assert shard_parent_state.slot != state.slot - shard_block = build_shard_block( - spec, state, shard, - shard_parent_state=shard_parent_state, slot=state.slot, body=body, signed=True - ) - shard_blocks_buffer.append(shard_block) - run_on_shard_block(spec, store, shard_store, shard_block) - assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root() + shard = shard_store.shard + committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) + has_shard_committee = committee_index is not None # has committee of `shard` at this slot + # On beacon blocks at `state.slot + 1` beacon_block = build_empty_block(spec, state, slot=state.slot + 1) - # Attester creates `attestation` + # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block if has_shard_committee and len(shard_blocks_buffer) > 0: + # Sanity check `get_pendings_shard_blocks` function + check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) + # Use temporary next state to get ShardTransition of shard block shard_transitions = build_shard_transitions_till_slot( spec, @@ -62,7 +78,6 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buf on_time_slot=state.slot + 1, ) shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( spec, state, @@ -75,35 +90,47 @@ def run_apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buf beacon_block.body.attestations = [attestation] beacon_block.body.shard_transitions = shard_transitions - signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + # Clear buffer + shard_blocks_buffer.clear() + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) add_block_to_store(spec, store, signed_beacon_block) assert spec.get_head(store) == beacon_block.hash_tree_root() - if has_shard_committee: - shard_blocks_buffer = [] # clear buffer + # On shard block at updated `state.slot` + if is_in_offset_sets(spec, state, shard): + # The created shard block would be appended to `shard_blocks_buffer` + apply_shard_block(spec, store, shard_store, state, shard_blocks_buffer) - return has_shard_committee, shard_blocks_buffer + return has_shard_committee @with_all_phases_except([PHASE0]) @spec_state_test +@never_bls # Set to never_bls for testing `check_pending_shard_blocks` def test_basic(spec, state): - spec.PHASE_1_GENESIS_SLOT = 0 # FIXME: remove mocking + spec.PHASE_1_GENESIS_SLOT = 0 # NOTE: mock genesis slot here state = spec.upgrade_to_phase1(state) + shard = spec.Shard(1) # Initialization store = spec.get_forkchoice_store(state) anchor_root = get_anchor_root(spec, state) assert spec.get_head(store) == anchor_root - shard = spec.Shard(1) shard_store = spec.get_forkchoice_shard_store(state, shard) - shard_block_count = 2 + shard_head_root = spec.get_shard_head(store, shard_store) + assert shard_head_root == state.shard_states[shard].latest_block_root + assert shard_store.block_states[shard_head_root].slot == 1 + assert shard_store.block_states[shard_head_root] == state.shard_states[shard] + + # For mainnet config, it's possible that only one committee of `shard` per epoch. + # we set this counter to test more rounds. + shard_committee_counter = 2 shard_blocks_buffer = [] - while shard_block_count > 0: - has_shard_committee, shard_blocks_buffer = run_apply_shard_and_beacon( + while shard_committee_counter > 0: + has_shard_committee = apply_shard_and_beacon( spec, state, store, shard_store, shard_blocks_buffer ) if has_shard_committee: - shard_block_count -= 1 + shard_committee_counter -= 1 From 6f9c290bfb97240a9f41370b026d066ed8df6917 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 20:40:09 +0800 Subject: [PATCH 40/49] Add TODO flag of latest message --- specs/phase1/shard-fork-choice.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 5bc2cbe4e..a474c8214 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -61,6 +61,8 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro state.validators[i].effective_balance for i in active_indices if ( i in store.latest_messages + # TODO: check the latest message logic: currently, validator's previous vote of another shard + # would be ignored once their newer vote is accepted. Check if it makes sense. and store.latest_messages[i].shard == shard_store.shard and get_shard_ancestor( store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot From a154d0c22b9a3175e74383e4df53e5b09f6df0fd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Jun 2020 21:24:17 +0800 Subject: [PATCH 41/49] Fix typo --- specs/phase1/shard-fork-choice.md | 6 +++--- .../pyspec/eth2spec/test/fork_choice/test_on_shard_head.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index a474c8214..427b72784 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -16,7 +16,7 @@ - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance) - [`get_shard_head`](#get_shard_head) - [`get_shard_ancestor`](#get_shard_ancestor) - - [`get_pendings_shard_blocks`](#get_pendings_shard_blocks) + - [`get_pending_shard_blocks`](#get_pending_shard_blocks) - [Handlers](#handlers) - [`on_shard_block`](#on_shard_block) @@ -111,10 +111,10 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: return root ``` -#### `get_pendings_shard_blocks` +#### `get_pending_shard_blocks` ```python -def get_pendings_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: +def get_pending_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: """ Return the shard blocks branch that from shard head to beacon head. """ diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 7e94ddd8e..1ba15968f 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -43,7 +43,7 @@ def apply_shard_block(spec, store, shard_store, beacon_head_state, shard_blocks_ def check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer): pending_shard_blocks = [ spec.SignedShardBlock(message=b) - for b in spec.get_pendings_shard_blocks(store, shard_store) + for b in spec.get_pending_shard_blocks(store, shard_store) ] assert pending_shard_blocks == shard_blocks_buffer @@ -67,7 +67,7 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block if has_shard_committee and len(shard_blocks_buffer) > 0: - # Sanity check `get_pendings_shard_blocks` function + # Sanity check `get_pending_shard_blocks` function check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) # Use temporary next state to get ShardTransition of shard block @@ -129,6 +129,7 @@ def test_basic(spec, state): shard_committee_counter = 2 shard_blocks_buffer = [] while shard_committee_counter > 0: + print(f'state.slot', state.slot) has_shard_committee = apply_shard_and_beacon( spec, state, store, shard_store, shard_blocks_buffer ) From 2d4788fe7d032f473fe5d60d732c40505ad8f485 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 5 Jun 2020 16:19:25 +0800 Subject: [PATCH 42/49] Fix `verify_shard_block_message` Add check for `block.beacon_parent_root` per Terence's suggestion Update `get_shard_transition` 1. Disable verification: it will be fix in v-guide 2. Use `on_time_slot` to compute offset_slots Rework tests --- specs/phase1/shard-fork-choice.md | 11 ++-- specs/phase1/shard-transition.md | 66 +++++++++++-------- .../test/fork_choice/test_on_shard_head.py | 19 ++---- .../eth2spec/test/helpers/attestations.py | 2 +- .../eth2spec/test/helpers/shard_block.py | 36 +++------- .../test/helpers/shard_transitions.py | 5 +- .../test_process_shard_transition.py | 31 ++++----- .../test/phase_1/sanity/test_blocks.py | 56 ++++++---------- 8 files changed, 94 insertions(+), 132 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 427b72784..844dbfb86 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -151,10 +151,11 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si # Check shard parent exists assert shard_block.shard_parent_root in shard_store.block_states - pre_shard_state = shard_store.block_states[shard_block.shard_parent_root] + shard_parent_state = shard_store.block_states[shard_block.shard_parent_root] # Check beacon parent exists assert shard_block.beacon_parent_root in store.block_states + beacon_parent_state = store.block_states[shard_block.beacon_parent_root] # Check that block is later than the finalized shard state slot (optimization to reduce calls to get_ancestor) finalized_beacon_state = store.block_states[store.finalized_checkpoint.root] @@ -171,11 +172,9 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si shard_store.blocks[hash_tree_root(shard_block)] = shard_block # Check the block is valid and compute the post-state - beacon_head_root = get_head(store) - beacon_head_state = store.block_states[beacon_head_root] - assert verify_shard_block_message(beacon_head_state, pre_shard_state, shard_block, shard_block.slot, shard) - assert verify_shard_block_signature(beacon_head_state, signed_shard_block) - post_state = get_post_shard_state(beacon_head_state, pre_shard_state, shard_block) + assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block, shard_block.slot, shard) + assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) + post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) # Add new state for this block to the store shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 6c4f652d2..8d75879f5 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -30,7 +30,7 @@ This document describes the shard transition function and fraud proofs as part o ### Misc ```python -def compute_shard_transition_digest(beacon_state: BeaconState, +def compute_shard_transition_digest(beacon_parent_state: BeaconState, shard_state: ShardState, beacon_parent_root: Root, shard_body_root: Root) -> Bytes32: @@ -44,17 +44,33 @@ def compute_shard_transition_digest(beacon_state: BeaconState, ```python def verify_shard_block_message(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, block: ShardBlock, slot: Slot, - shard: Shard) -> bool: - assert block.shard_parent_root == shard_state.latest_block_root - assert block.slot == slot - next_slot = Slot(beacon_state.slot + 1) + shard: Shard, + beacon_parent_slot: Slot=None) -> bool: + # Check `shard_parent_root` field + assert block.shard_parent_root == shard_parent_state.latest_block_root + # Check `beacon_parent_root` field + if beacon_parent_slot is None: + beacon_parent_slot = beacon_state.slot + if beacon_parent_slot == beacon_state.slot: + beacon_parent_block_header = beacon_state.latest_block_header.copy() + if beacon_parent_block_header.state_root == Root(): + beacon_parent_block_header.state_root = hash_tree_root(beacon_state) + beacon_parent_root = hash_tree_root(beacon_parent_block_header) + else: + beacon_parent_root = get_block_root_at_slot(beacon_state, beacon_parent_slot) + assert block.beacon_parent_root == beacon_parent_root + # Check `slot` field + next_slot = Slot(slot + 1) offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), next_slot) assert slot in offset_slots + # Check `shard` field assert block.shard == shard + # Check `proposer_index` field assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) + # Check `body` field assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE return True ``` @@ -177,7 +193,7 @@ def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[ ```python def get_proposal_choices_at_slot(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, slot: Slot, shard: Shard, shard_blocks: Sequence[SignedShardBlock], @@ -188,25 +204,16 @@ def get_proposal_choices_at_slot(beacon_state: BeaconState, """ choices = [] shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] + shard_state = shard_parent_state.copy() for block in shard_blocks_at_slot: - try: - # Verify block message and signature - # TODO these validations should have been checked upon receiving shard blocks. - assert verify_shard_block_message(beacon_state, shard_state, block.message, slot, shard) - if validate_signature: - assert verify_shard_block_signature(beacon_state, block) - - shard_state = get_post_shard_state(beacon_state, shard_state, block.message) - except Exception: - pass # TODO: throw error in the test helper - else: - choices.append(block) + shard_state = get_post_shard_state(beacon_state, shard_state, block.message) + choices.append(block) return choices ``` ```python def get_proposal_at_slot(beacon_state: BeaconState, - shard_state: ShardState, + shard_parent_state: ShardState, slot: Shard, shard: Shard, shard_blocks: Sequence[SignedShardBlock], @@ -217,7 +224,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, """ choices = get_proposal_choices_at_slot( beacon_state=beacon_state, - shard_state=shard_state, + shard_parent_state=shard_parent_state, slot=slot, shard=shard, shard_blocks=shard_blocks, @@ -232,7 +239,7 @@ def get_proposal_at_slot(beacon_state: BeaconState, proposal = get_winning_proposal(beacon_state, choices) # Apply state transition - shard_state = get_post_shard_state(beacon_state, shard_state, proposal.message) + shard_state = get_post_shard_state(beacon_state, shard_parent_state, proposal.message) return proposal, shard_state ``` @@ -242,15 +249,17 @@ def get_shard_state_transition_result( beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock], + on_time_slot: Slot, validate_signature: bool=True, ) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: proposals = [] shard_states = [] shard_state = beacon_state.shard_states[shard] - for slot in get_offset_slots(beacon_state, shard): + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) + for slot in offset_slots: proposal, shard_state = get_proposal_at_slot( beacon_state=beacon_state, - shard_state=shard_state, + shard_parent_state=shard_state, slot=slot, shard=shard, shard_blocks=shard_blocks, @@ -271,9 +280,12 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y ```python def get_shard_transition(beacon_state: BeaconState, shard: Shard, - shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: - offset_slots = get_offset_slots(beacon_state, shard) - proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks) + shard_blocks: Sequence[SignedShardBlock], + on_time_slot: Slot) -> ShardTransition: + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) + proposals, shard_states, shard_data_roots = get_shard_state_transition_result( + beacon_state, shard, shard_blocks, on_time_slot + ) shard_block_lengths = [] proposer_signatures = [] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 1ba15968f..ca79cfd23 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -1,8 +1,8 @@ from eth2spec.utils.ssz.ssz_impl import hash_tree_root from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_except, never_bls +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, get_committee_index_of_shard, @@ -25,15 +25,15 @@ def run_on_shard_block(spec, store, shard_store, signed_block, valid=True): assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message -def apply_shard_block(spec, store, shard_store, beacon_head_state, shard_blocks_buffer): +def apply_shard_block(spec, store, shard_store, beacon_parent_state, shard_blocks_buffer): shard = shard_store.shard body = b'\x56' * 4 shard_head_root = spec.get_shard_head(store, shard_store) shard_parent_state = shard_store.block_states[shard_head_root] - assert shard_parent_state.slot != beacon_head_state.slot + assert shard_parent_state.slot != beacon_parent_state.slot shard_block = build_shard_block( - spec, beacon_head_state, shard, - shard_parent_state=shard_parent_state, slot=beacon_head_state.slot, body=body, signed=True + spec, beacon_parent_state, shard, + shard_parent_state=shard_parent_state, slot=beacon_parent_state.slot, body=body, signed=True ) shard_blocks_buffer.append(shard_block) run_on_shard_block(spec, store, shard_store, shard_block) @@ -62,30 +62,26 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) committee_index = get_committee_index_of_shard(spec, state, state.slot, shard) has_shard_committee = committee_index is not None # has committee of `shard` at this slot - # On beacon blocks at `state.slot + 1` beacon_block = build_empty_block(spec, state, slot=state.slot + 1) # If next slot has committee of `shard`, add `shard_transtion` to the proposing beacon block if has_shard_committee and len(shard_blocks_buffer) > 0: # Sanity check `get_pending_shard_blocks` function check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) - # Use temporary next state to get ShardTransition of shard block shard_transitions = build_shard_transitions_till_slot( spec, state, shard_blocks={shard: shard_blocks_buffer}, - on_time_slot=state.slot + 1, ) shard_transition = shard_transitions[shard] - attestation = build_attestation_with_shard_transition( + attestation = get_valid_on_time_attestation( spec, state, index=committee_index, - on_time_slot=state.slot + 1, shard_transition=shard_transition, + signed=False, ) - assert attestation.data.slot == state.slot assert spec.get_shard(state, attestation) == shard beacon_block.body.attestations = [attestation] beacon_block.body.shard_transitions = shard_transitions @@ -129,7 +125,6 @@ def test_basic(spec, state): shard_committee_counter = 2 shard_blocks_buffer = [] while shard_committee_counter > 0: - print(f'state.slot', state.slot) has_shard_committee = apply_shard_and_beacon( spec, state, store, shard_store, shard_blocks_buffer ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 1372b0654..106069dd6 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -87,7 +87,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t if on_time: temp_state = state.copy() next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[]) + shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[], on_time_slot=slot + 1) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 410213edd..0a2ed67d2 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -1,6 +1,4 @@ -from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import get_state_and_beacon_parent_root_at_slot -from eth2spec.test.helpers.state import transition_to from eth2spec.test.helpers.keys import privkeys from eth2spec.utils import bls from eth2spec.utils.bls import only_with_bls @@ -34,11 +32,8 @@ def build_shard_block(spec, if body is None: body = b'\x56' * 128 - temp_state = beacon_state.copy() - transition_to(spec, temp_state, slot) beacon_state, beacon_parent_root = get_state_and_beacon_parent_root_at_slot(spec, beacon_state, slot) - assert beacon_state == temp_state - proposer_index = spec.get_shard_proposer_index(temp_state, slot, shard) + proposer_index = spec.get_shard_proposer_index(beacon_state, slot, shard) block = spec.ShardBlock( shard_parent_root=shard_parent_state.latest_block_root, beacon_parent_root=beacon_parent_root, @@ -57,14 +52,17 @@ def build_shard_block(spec, return signed_block -def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot) +def build_shard_transitions_till_slot(spec, parent_beacon_state, shard_blocks): shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS + on_time_slot = parent_beacon_state.slot + 1 for shard, blocks in shard_blocks.items(): - offset_slots = spec.get_offset_slots(temp_state, shard) + offset_slots = spec.compute_offset_slots( + spec.get_latest_slot_for_shard(parent_beacon_state, shard), + on_time_slot, + ) len_offset_slots = len(offset_slots) - shard_transition = spec.get_shard_transition(temp_state, shard, blocks) + shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks, on_time_slot) + if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() assert shard_transition.shard_states[len_offset_slots - 1].latest_block_root == shard_block_root @@ -74,22 +72,6 @@ def build_shard_transitions_till_slot(spec, state, shard_blocks, on_time_slot): return shard_transitions -def build_attestation_with_shard_transition(spec, state, index, on_time_slot, shard_transition): - temp_state = state.copy() - transition_to(spec, temp_state, on_time_slot - 1) - attestation = get_valid_on_time_attestation( - spec, - temp_state, - index=index, - shard_transition=shard_transition, - signed=True, - ) - assert attestation.data.slot == temp_state.slot - if shard_transition is not None: - assert attestation.data.shard_transition_root == shard_transition.hash_tree_root() - return attestation - - def get_committee_index_of_shard(spec, state, slot, shard): # Optional[CommitteeIndex] active_shard_count = spec.get_active_shard_count(state) committee_count = spec.get_committee_count_at_slot(state, slot) diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index abb5e7278..8e62b2f27 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -1,5 +1,4 @@ from eth2spec.test.context import expect_assertion_error -from eth2spec.test.helpers.state import transition_to def run_shard_transitions_processing(spec, state, shard_transitions, attestations, valid=True): @@ -37,7 +36,5 @@ def get_shard_transition_of_committee(spec, state, committee_index, slot=None, s slot = state.slot shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - temp_state = state.copy() - transition_to(spec, temp_state, slot + 1) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=shard_blocks) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks, on_time_slot=slot + 1) return shard_transition diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index 00ffbe0a8..dab4973da 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -2,71 +2,64 @@ from eth2spec.test.context import ( PHASE0, with_all_phases_except, spec_state_test, - always_bls, ) +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_transitions import run_shard_transitions_processing from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, ) -from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot +from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot, next_slot def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): state = transition_to_valid_shard_slot(spec, state) - init_slot = state.slot committee_index = spec.CommitteeIndex(0) shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 + transition_to(spec, state, state.slot + target_len_offset_slot) + assert state.shard_states[shard].slot == state.slot - target_len_offset_slot - 1 # Create SignedShardBlock body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks = [shard_block] - # Create a shard_transitions that would be included at beacon block `state.slot + target_len_offset_slot` shard_transitions = build_shard_transitions_till_slot( spec, state, shard_blocks={shard: shard_blocks}, - on_time_slot=state.slot + target_len_offset_slot, ) shard_transition = shard_transitions[shard] - # Create an attestation that would be included at beacon block `state.slot + target_len_offset_slot` - attestation = build_attestation_with_shard_transition( + attestation = get_valid_on_time_attestation( spec, state, index=committee_index, - on_time_slot=state.slot + target_len_offset_slot, shard_transition=shard_transition, + signed=False, ) + next_slot(spec, state) pre_gasprice = state.shard_states[shard].gasprice - - transition_to(spec, state, state.slot + target_len_offset_slot) pre_shard_state = state.shard_states[shard] - yield from run_shard_transitions_processing(spec, state, shard_transitions, [attestation], valid=valid) if valid: - # After state transition, - assert state.slot == init_slot + target_len_offset_slot shard_state = state.shard_states[shard] assert shard_state != pre_shard_state assert shard_state == shard_transition.shard_states[len(shard_transition.shard_states) - 1] - + assert shard_state.latest_block_root == shard_block.message.hash_tree_root() if target_len_offset_slot == 1: assert shard_state.gasprice > pre_gasprice @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_basic_crosslinks(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=1, valid=True) @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_multiple_offset_slots(spec, state): - yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=3, valid=True) + # NOTE: this test is only for full crosslink (minimal config), not for mainnet + yield from run_basic_crosslink_tests(spec, state, target_len_offset_slot=2, valid=True) diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 0175bd40d..828cd19d7 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -4,37 +4,40 @@ from eth2spec.test.context import ( PHASE0, with_all_phases_except, spec_state_test, - always_bls, ) +from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.shard_block import ( - build_attestation_with_shard_transition, build_shard_block, build_shard_transitions_till_slot, ) -from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot +from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to -def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index, valid=True): - shard_transitions = build_shard_transitions_till_slot( - spec, state, shard_blocks, on_time_slot=state.slot + target_len_offset_slot - ) +def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard, valid=True): + transition_to(spec, state, state.slot + target_len_offset_slot) + + body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE + shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) + shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} + + shard_transitions = build_shard_transitions_till_slot(spec, state, shard_blocks) attestations = [ - build_attestation_with_shard_transition( + get_valid_on_time_attestation( spec, state, - on_time_slot=state.slot + target_len_offset_slot, index=committee_index, shard_transition=shard_transitions[shard], + signed=True, ) for shard in shard_blocks.keys() ] - # Propose beacon block at slot `x + 1` - beacon_block = build_empty_block(spec, state, slot=state.slot + target_len_offset_slot) + beacon_block = build_empty_block(spec, state, slot=state.slot + 1) beacon_block.body.attestations = attestations beacon_block.body.shard_transitions = shard_transitions + pre_gasprice = state.shard_states[shard].gasprice pre_shard_states = state.shard_states.copy() yield 'pre', state.copy() yield 'block', beacon_block @@ -52,17 +55,18 @@ def run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_off assert post_shard_state == shard_transitions[shard].shard_states[ len(shard_transitions[shard].shard_states) - 1 ] - assert beacon_block.slot == shard_transitions[shard].shard_states[0].slot + target_len_offset_slot assert post_shard_state.slot == state.slot - 1 if len(shard_blocks[shard]) == 0: # `latest_block_root` is the same assert post_shard_state.latest_block_root == pre_shard_states[shard].latest_block_root + if target_len_offset_slot == 1 and len(shard_blocks) > 0: + assert post_shard_state.gasprice > pre_gasprice @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_process_beacon_block_with_normal_shard_transition(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 @@ -70,25 +74,13 @@ def test_process_beacon_block_with_normal_shard_transition(spec, state): shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 - pre_gasprice = state.shard_states[shard].gasprice - - # Create SignedShardBlock at slot `shard_state.slot + 1` - body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE - shard_block = build_shard_block(spec, state, shard, body=body, signed=True) - shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} - - yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) - - shard_state = state.shard_states[shard] - - if target_len_offset_slot == 1 and len(shard_blocks) > 0: - assert shard_state.gasprice > pre_gasprice + yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) @with_all_phases_except([PHASE0]) @spec_state_test -@always_bls def test_process_beacon_block_with_empty_proposal_transition(spec, state): + # NOTE: this test is only for full crosslink (minimal config), not for mainnet state = transition_to_valid_shard_slot(spec, state) target_len_offset_slot = 1 @@ -96,12 +88,4 @@ def test_process_beacon_block_with_empty_proposal_transition(spec, state): shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot + target_len_offset_slot - 1) assert state.shard_states[shard].slot == state.slot - 1 - # No new shard block - shard_blocks = {} - - pre_gasprice = state.shard_states[shard].gasprice - - yield from run_beacon_block_with_shard_blocks(spec, state, shard_blocks, target_len_offset_slot, committee_index) - - if target_len_offset_slot == 1 and len(shard_blocks) > 0: - assert state.shard_states[shard].gasprice > pre_gasprice + yield from run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, committee_index, shard) From 2afa315cb3b1d8abcef3af2ef7df9046eb4f8f23 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 5 Jun 2020 21:49:50 +0800 Subject: [PATCH 43/49] clean leftover --- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 106069dd6..ef90a71aa 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -85,9 +85,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: - temp_state = state.copy() - next_slot(spec, temp_state) - shard_transition = spec.get_shard_transition(temp_state, shard, shard_blocks=[], on_time_slot=slot + 1) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[], on_time_slot=slot + 1) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() From a71c0a5ccc4c74f5ae8c3c071aa47506aa5cc03a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 6 Jun 2020 02:35:46 +0800 Subject: [PATCH 44/49] Per #1704 discussion, remove `on_time_slot`: the given `beacon_state` should be transitioned. --- specs/phase1/shard-transition.md | 4 ++-- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 6 ++---- tests/core/pyspec/eth2spec/test/helpers/shard_block.py | 2 +- .../core/pyspec/eth2spec/test/helpers/shard_transitions.py | 7 ++----- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 8d75879f5..191dbd1aa 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -280,8 +280,8 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y ```python def get_shard_transition(beacon_state: BeaconState, shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - on_time_slot: Slot) -> ShardTransition: + shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: + on_time_slot = Slot(beacon_state.slot + 1) offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) proposals, shard_states, shard_data_roots = get_shard_state_transition_result( beacon_state, shard, shard_blocks, on_time_slot diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index ef90a71aa..1e0560405 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -85,7 +85,7 @@ def build_attestation_data(spec, state, slot, index, shard_transition=None, on_t # No shard transition -> no shard block shard = spec.get_shard(state, spec.Attestation(data=attestation_data)) if on_time: - shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[], on_time_slot=slot + 1) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=[]) lastest_shard_data_root_index = len(shard_transition.shard_data_roots) - 1 attestation_data.shard_head_root = shard_transition.shard_data_roots[lastest_shard_data_root_index] attestation_data.shard_transition_root = shard_transition.hash_tree_root() @@ -318,9 +318,7 @@ def next_epoch_with_attestations(spec, for index in range(committees_per_slot): if spec.fork == PHASE1: shard = spec.compute_shard_from_committee_index(post_state, index, slot_to_attest) - shard_transition = get_shard_transition_of_committee( - spec, post_state, index, slot=slot_to_attest - ) + shard_transition = get_shard_transition_of_committee(spec, post_state, index) block.body.shard_transitions[shard] = shard_transition else: shard_transition = None diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index 0a2ed67d2..e63096b92 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -61,7 +61,7 @@ def build_shard_transitions_till_slot(spec, parent_beacon_state, shard_blocks): on_time_slot, ) len_offset_slots = len(offset_slots) - shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks, on_time_slot) + shard_transition = spec.get_shard_transition(parent_beacon_state, shard, blocks) if len(blocks) > 0: shard_block_root = blocks[-1].message.hash_tree_root() diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py index 8e62b2f27..d10d1ee7b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_transitions.py @@ -28,13 +28,10 @@ def run_shard_transitions_processing(spec, state, shard_transitions, attestation yield 'post', state -def get_shard_transition_of_committee(spec, state, committee_index, slot=None, shard_blocks=None): +def get_shard_transition_of_committee(spec, state, committee_index, shard_blocks=None): if shard_blocks is None: shard_blocks = [] - if slot is None: - slot = state.slot - shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot) - shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks, on_time_slot=slot + 1) + shard_transition = spec.get_shard_transition(state, shard, shard_blocks=shard_blocks) return shard_transition From a4cc189f2b34b9e99d53df32dc24b383d3b4d3bd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 6 Jun 2020 05:19:46 +0800 Subject: [PATCH 45/49] Apply PR feedback from Danny --- specs/phase1/shard-fork-choice.md | 10 +-- specs/phase1/shard-transition.md | 63 ++++--------------- .../test/fork_choice/test_on_shard_head.py | 4 +- 3 files changed, 21 insertions(+), 56 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 844dbfb86..b1fc3080e 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -46,7 +46,7 @@ class ShardStore: def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore: return ShardStore( shard=shard, - blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)}, + blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot, shard=shard)}, block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]}, ) ``` @@ -168,13 +168,15 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root ) - # Add new block to the store - shard_store.blocks[hash_tree_root(shard_block)] = shard_block - # Check the block is valid and compute the post-state assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block, shard_block.slot, shard) assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) + post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) + + # Add new block to the store + shard_store.blocks[hash_tree_root(shard_block)] = shard_block + # Add new state for this block to the store shard_store.block_states[hash_tree_root(shard_block)] = post_state ``` diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index 191dbd1aa..fe3223933 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -47,20 +47,14 @@ def verify_shard_block_message(beacon_state: BeaconState, shard_parent_state: ShardState, block: ShardBlock, slot: Slot, - shard: Shard, - beacon_parent_slot: Slot=None) -> bool: + shard: Shard) -> bool: # Check `shard_parent_root` field assert block.shard_parent_root == shard_parent_state.latest_block_root # Check `beacon_parent_root` field - if beacon_parent_slot is None: - beacon_parent_slot = beacon_state.slot - if beacon_parent_slot == beacon_state.slot: - beacon_parent_block_header = beacon_state.latest_block_header.copy() - if beacon_parent_block_header.state_root == Root(): - beacon_parent_block_header.state_root = hash_tree_root(beacon_state) - beacon_parent_root = hash_tree_root(beacon_parent_block_header) - else: - beacon_parent_root = get_block_root_at_slot(beacon_state, beacon_parent_slot) + beacon_parent_block_header = beacon_state.latest_block_header.copy() + if beacon_parent_block_header.state_root == Root(): + beacon_parent_block_header.state_root = hash_tree_root(beacon_state) + beacon_parent_root = hash_tree_root(beacon_parent_block_header) assert block.beacon_parent_root == beacon_parent_root # Check `slot` field next_slot = Slot(slot + 1) @@ -191,26 +185,6 @@ def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[ return [hash_tree_root(proposal.message.body) for proposal in proposals] ``` -```python -def get_proposal_choices_at_slot(beacon_state: BeaconState, - shard_parent_state: ShardState, - slot: Slot, - shard: Shard, - shard_blocks: Sequence[SignedShardBlock], - validate_signature: bool=True) -> Sequence[SignedShardBlock]: - """ - Return the valid shard blocks at the given ``slot``. - Note that this function doesn't change the state. - """ - choices = [] - shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot] - shard_state = shard_parent_state.copy() - for block in shard_blocks_at_slot: - shard_state = get_post_shard_state(beacon_state, shard_state, block.message) - choices.append(block) - return choices -``` - ```python def get_proposal_at_slot(beacon_state: BeaconState, shard_parent_state: ShardState, @@ -222,21 +196,14 @@ def get_proposal_at_slot(beacon_state: BeaconState, Return ``proposal``, ``shard_state`` of the given ``slot``. Note that this function doesn't change the state. """ - choices = get_proposal_choices_at_slot( - beacon_state=beacon_state, - shard_parent_state=shard_parent_state, - slot=slot, - shard=shard, - shard_blocks=shard_blocks, - validate_signature=validate_signature, - ) - if len(choices) == 0: + shard_blocks = [block for block in shard_blocks if block.message.slot == slot] + if len(shard_blocks) == 0: block = ShardBlock(slot=slot, shard=shard) proposal = SignedShardBlock(message=block) - elif len(choices) == 1: - proposal = choices[0] + elif len(shard_blocks) == 1: + proposal = shard_blocks[0] else: - proposal = get_winning_proposal(beacon_state, choices) + proposal = get_winning_proposal(beacon_state, shard_blocks) # Apply state transition shard_state = get_post_shard_state(beacon_state, shard_parent_state, proposal.message) @@ -249,13 +216,12 @@ def get_shard_state_transition_result( beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock], - on_time_slot: Slot, validate_signature: bool=True, ) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]: proposals = [] shard_states = [] shard_state = beacon_state.shard_states[shard] - offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), Slot(beacon_state.slot + 1)) for slot in offset_slots: proposal, shard_state = get_proposal_at_slot( beacon_state=beacon_state, @@ -281,11 +247,8 @@ Suppose you are a committee member on shard `shard` at slot `current_slot` and y def get_shard_transition(beacon_state: BeaconState, shard: Shard, shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition: - on_time_slot = Slot(beacon_state.slot + 1) - offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), on_time_slot) - proposals, shard_states, shard_data_roots = get_shard_state_transition_result( - beacon_state, shard, shard_blocks, on_time_slot - ) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), Slot(beacon_state.slot + 1)) + proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks) shard_block_lengths = [] proposer_signatures = [] diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index ca79cfd23..1a9042960 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -89,11 +89,11 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) # Clear buffer shard_blocks_buffer.clear() - signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) + signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block) # transition! add_block_to_store(spec, store, signed_beacon_block) assert spec.get_head(store) == beacon_block.hash_tree_root() - # On shard block at updated `state.slot` + # On shard block at transitioned `state.slot` if is_in_offset_sets(spec, state, shard): # The created shard block would be appended to `shard_blocks_buffer` apply_shard_block(spec, store, shard_store, state, shard_blocks_buffer) From 435505746cd13f336c0bad1c71829df7fd322ca2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 8 Jun 2020 17:12:46 +0800 Subject: [PATCH 46/49] PR feedback from Terence: fix `get_shard_latest_attesting_balance` Co-authored-by: terence tsao --- specs/phase1/shard-fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 427b72784..5fef81868 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -65,7 +65,7 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro # would be ignored once their newer vote is accepted. Check if it makes sense. and store.latest_messages[i].shard == shard_store.shard and get_shard_ancestor( - store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot + store, shard_store, store.latest_messages[i].shard_root, shard_store.blocks[root].slot ) == root ) )) From 7e67aaeb35c6d8ffc4d8ced61f274161b6c04660 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 8 Jun 2020 18:15:14 +0800 Subject: [PATCH 47/49] Rename `build_shard_transitions_till_slot` to `get_shard_transitions` --- .../pyspec/eth2spec/test/fork_choice/test_on_shard_head.py | 4 ++-- tests/core/pyspec/eth2spec/test/helpers/shard_block.py | 2 +- .../phase_1/block_processing/test_process_shard_transition.py | 4 ++-- tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py index 1a9042960..24eeaedbe 100644 --- a/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py +++ b/tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py @@ -4,7 +4,7 @@ from eth2spec.test.context import PHASE0, spec_state_test, with_all_phases_excep from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_block import ( build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, get_committee_index_of_shard, ) from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root @@ -69,7 +69,7 @@ def apply_shard_and_beacon(spec, state, store, shard_store, shard_blocks_buffer) # Sanity check `get_pending_shard_blocks` function check_pending_shard_blocks(spec, store, shard_store, shard_blocks_buffer) # Use temporary next state to get ShardTransition of shard block - shard_transitions = build_shard_transitions_till_slot( + shard_transitions = get_shard_transitions( spec, state, shard_blocks={shard: shard_blocks_buffer}, diff --git a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py index e63096b92..f8b4a155f 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/shard_block.py +++ b/tests/core/pyspec/eth2spec/test/helpers/shard_block.py @@ -52,7 +52,7 @@ def build_shard_block(spec, return signed_block -def build_shard_transitions_till_slot(spec, parent_beacon_state, shard_blocks): +def get_shard_transitions(spec, parent_beacon_state, shard_blocks): shard_transitions = [spec.ShardTransition()] * spec.MAX_SHARDS on_time_slot = parent_beacon_state.slot + 1 for shard, blocks in shard_blocks.items(): diff --git a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py index dab4973da..e97cc90a8 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/block_processing/test_process_shard_transition.py @@ -7,7 +7,7 @@ from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.shard_transitions import run_shard_transitions_processing from eth2spec.test.helpers.shard_block import ( build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, ) from eth2spec.test.helpers.state import transition_to, transition_to_valid_shard_slot, next_slot @@ -24,7 +24,7 @@ def run_basic_crosslink_tests(spec, state, target_len_offset_slot, valid=True): body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks = [shard_block] - shard_transitions = build_shard_transitions_till_slot( + shard_transitions = get_shard_transitions( spec, state, shard_blocks={shard: shard_blocks}, diff --git a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py index 828cd19d7..33b0beac7 100644 --- a/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase_1/sanity/test_blocks.py @@ -9,7 +9,7 @@ from eth2spec.test.helpers.attestations import get_valid_on_time_attestation from eth2spec.test.helpers.block import build_empty_block from eth2spec.test.helpers.shard_block import ( build_shard_block, - build_shard_transitions_till_slot, + get_shard_transitions, ) from eth2spec.test.helpers.state import state_transition_and_sign_block, transition_to_valid_shard_slot, transition_to @@ -21,7 +21,7 @@ def run_beacon_block_with_shard_blocks(spec, state, target_len_offset_slot, comm shard_block = build_shard_block(spec, state, shard, body=body, slot=state.slot, signed=True) shard_blocks: Dict[spec.Shard, Sequence[spec.SignedShardBlock]] = {shard: [shard_block]} - shard_transitions = build_shard_transitions_till_slot(spec, state, shard_blocks) + shard_transitions = get_shard_transitions(spec, state, shard_blocks) attestations = [ get_valid_on_time_attestation( spec, From e03a970eaf88e82ade443ec37ac7451f5903966c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 8 Jun 2020 23:49:24 +0800 Subject: [PATCH 48/49] PR feedback from danny: simplify `verify_shard_block_message` params --- specs/phase1/shard-fork-choice.md | 2 +- specs/phase1/shard-transition.md | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index b1fc3080e..6c431a68d 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -169,7 +169,7 @@ def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: Si ) # Check the block is valid and compute the post-state - assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block, shard_block.slot, shard) + assert verify_shard_block_message(beacon_parent_state, shard_parent_state, shard_block) assert verify_shard_block_signature(beacon_parent_state, signed_shard_block) post_state = get_post_shard_state(beacon_parent_state, shard_parent_state, shard_block) diff --git a/specs/phase1/shard-transition.md b/specs/phase1/shard-transition.md index fe3223933..c62b059ee 100644 --- a/specs/phase1/shard-transition.md +++ b/specs/phase1/shard-transition.md @@ -43,27 +43,26 @@ def compute_shard_transition_digest(beacon_parent_state: BeaconState, ### Shard block verification functions ```python -def verify_shard_block_message(beacon_state: BeaconState, +def verify_shard_block_message(beacon_parent_state: BeaconState, shard_parent_state: ShardState, - block: ShardBlock, - slot: Slot, - shard: Shard) -> bool: + block: ShardBlock) -> bool: # Check `shard_parent_root` field assert block.shard_parent_root == shard_parent_state.latest_block_root # Check `beacon_parent_root` field - beacon_parent_block_header = beacon_state.latest_block_header.copy() + beacon_parent_block_header = beacon_parent_state.latest_block_header.copy() if beacon_parent_block_header.state_root == Root(): - beacon_parent_block_header.state_root = hash_tree_root(beacon_state) + beacon_parent_block_header.state_root = hash_tree_root(beacon_parent_state) beacon_parent_root = hash_tree_root(beacon_parent_block_header) assert block.beacon_parent_root == beacon_parent_root # Check `slot` field - next_slot = Slot(slot + 1) - offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_state, shard), next_slot) - assert slot in offset_slots + shard = block.shard + next_slot = Slot(block.slot + 1) + offset_slots = compute_offset_slots(get_latest_slot_for_shard(beacon_parent_state, shard), next_slot) + assert block.slot in offset_slots # Check `shard` field assert block.shard == shard # Check `proposer_index` field - assert block.proposer_index == get_shard_proposer_index(beacon_state, slot, shard) + assert block.proposer_index == get_shard_proposer_index(beacon_parent_state, block.slot, shard) # Check `body` field assert 0 < len(block.body) <= MAX_SHARD_BLOCK_SIZE return True From 2d895e9388cd7364448fb357a769a9f03d7a5141 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Jun 2020 00:13:27 +0800 Subject: [PATCH 49/49] PR feedback from danny --- specs/phase1/shard-fork-choice.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/specs/phase1/shard-fork-choice.md b/specs/phase1/shard-fork-choice.md index 61e4cd36f..0607613e8 100644 --- a/specs/phase1/shard-fork-choice.md +++ b/specs/phase1/shard-fork-choice.md @@ -76,18 +76,18 @@ def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, ro ```python def get_shard_head(store: Store, shard_store: ShardStore) -> Root: # Execute the LMD-GHOST fork choice - shard_blocks = shard_store.blocks - head_beacon_root = get_head(store) - head_shard_state = store.block_states[head_beacon_root].shard_states[shard_store.shard] - shard_head_root = head_shard_state.latest_block_root + beacon_head_root = get_head(store) + shard_head_state = store.block_states[beacon_head_root].shard_states[shard_store.shard] + shard_head_root = shard_head_state.latest_block_root + shard_blocks = { + root: shard_block for root, shard_block in shard_store.blocks.items() + if shard_block.slot > shard_head_state.slot + } while True: # Find the valid child block roots children = [ - root for root in shard_store.blocks.keys() - if ( - shard_blocks[root].shard_parent_root == shard_head_root - and shard_blocks[root].slot > head_shard_state.slot - ) + root for root, shard_block in shard_blocks.items() + if shard_block.shard_parent_root == shard_head_root ] if len(children) == 0: return shard_head_root @@ -107,7 +107,7 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: elif block.slot == slot: return root else: - # root is older than queried slot, thus a skip slot. Return earliest root prior to slot + # root is older than queried slot, thus a skip slot. Return most recent root prior to slot return root ``` @@ -116,7 +116,7 @@ def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: ```python def get_pending_shard_blocks(store: Store, shard_store: ShardStore) -> Sequence[ShardBlock]: """ - Return the shard blocks branch that from shard head to beacon head. + Return the canonical shard block branch that has not yet been crosslinked. """ shard = shard_store.shard