From 334d6c6bc73eee2daf45f75c53a62a8cd5a4066a Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 27 Aug 2019 14:58:28 +0100 Subject: [PATCH 01/28] Revamp 1_shard-data-chains.md WIP! * Significant simplifications * A few bug fixes * Lots of cleanups and reorganising (making it consistent with `0_beacon-chain.md`) * Likely a few bugs introduced --- specs/core/1_shard-data-chains.md | 641 ++++++++++++++---------------- 1 file changed, 295 insertions(+), 346 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 8e1532f17..9335c1803 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -17,31 +17,37 @@ - [State list lengths](#state-list-lengths) - [Rewards and penalties](#rewards-and-penalties) - [Signature domain types](#signature-domain-types) - - [TODO PLACEHOLDER](#todo-placeholder) - - [Data structures](#data-structures) - - [`ShardBlockHeader`](#shardblockheader) - - [`ShardBlock`](#shardblock) + - [Containers](#containers) - [`ShardBlockSignatures`](#shardblocksignatures) - - [`ShardBlockCore`](#shardblockcore) - - [`ExtendedShardBlockCore`](#extendedshardblockcore) + - [`ShardBlock`](#shardblock) + - [`ShardBlockHeader`](#shardblockheader) - [`ShardState`](#shardstate) - [`ShardReceiptDelta`](#shardreceiptdelta) - [Helper functions](#helper-functions) - - [`compute_slot_of_shard_slot`](#compute_slot_of_shard_slot) - - [`compute_epoch_of_shard_slot`](#compute_epoch_of_shard_slot) - - [`get_shard_period_start_epoch`](#get_shard_period_start_epoch) - - [`get_period_committee`](#get_period_committee) - - [`get_persistent_committee`](#get_persistent_committee) - - [`get_shard_block_proposer_index`](#get_shard_block_proposer_index) - - [`get_shard_header`](#get_shard_header) - - [`pad`](#pad) - - [`flatten_shard_header`](#flatten_shard_header) - - [`compute_crosslink_data_root`](#compute_crosslink_data_root) - - [`get_default_shard_state`](#get_default_shard_state) + - [Misc](#misc-1) + - [`pad`](#pad) + - [`compute_slot_of_shard_slot`](#compute_slot_of_shard_slot) + - [`compute_epoch_of_shard_slot`](#compute_epoch_of_shard_slot) + - [`compute_period_start_epoch`](#compute_period_start_epoch) + - [`compute_flat_shard_header`](#compute_flat_shard_header) + - [`compute_crosslink_data_root`](#compute_crosslink_data_root) + - [State accessors](#state-accessors) + - [`get_period_committee`](#get_period_committee) + - [`get_persistent_committee`](#get_persistent_committee) + - [`get_shard_proposer_index`](#get_shard_proposer_index) + - [`get_default_shard_state`](#get_default_shard_state) + - [`get_shard_base_reward`](#get_shard_base_reward) + - [State mutators](#state-mutators) + - [`add_fee`](#add_fee) + - [`add_reward`](#add_reward) + - [Shard state transition function](#shard-state-transition-function) + - [Period processing](#period-processing) + - [Block processing](#block-processing) + - [Block header](#block-header) + - [Attestations](#attestations) + - [Block data fees](#block-data-fees) - [Object validity](#object-validity) - [Shard block validation: preliminary](#shard-block-validation-preliminary) - - [Shard state transition function helpers](#shard-state-transition-function-helpers) - - [Shard state transition function](#shard-state-transition-function) - [Beacon attestations](#beacon-attestations) - [Shard fork choice rule](#shard-fork-choice-rule) @@ -53,11 +59,9 @@ This document describes the shard data layer and the shard fork choice rule in P ## Custom types -We define the following Python custom types for type hinting and readability: - | Name | SSZ equivalent | Description | | - | - | - | -| `ShardSlot` | `uint64` | a slot number in shard chain | +| `ShardSlot` | `uint64` | a shard slot number | ## Configuration @@ -65,7 +69,7 @@ We define the following Python custom types for type hinting and readability: | Name | Value | | - | - | -| `SHARD_SLOTS_PER_BEACON_SLOT` | `2**1` (= 2) | +| `SHARD_SLOTS_PER_EPOCH` | `2**7` (= 128) | | `TARGET_PERSISTENT_COMMITTEE_SIZE` | `2**7` (= 128) | | `SHARD_HEADER_SIZE` | `2**9` (= 512) | | `SHARD_BLOCK_SIZE_TARGET` | `2**14` (= 16,384) | @@ -76,7 +80,6 @@ We define the following Python custom types for type hinting and readability: | Name | Value | | - | - | | `PHASE_1_FORK_EPOCH` | **TBD** | -| `PHASE_1_FORK_SLOT` | **TBD** | ### Time parameters @@ -96,75 +99,53 @@ We define the following Python custom types for type hinting and readability: | Name | Value | | - | - | | `BASEFEE_ADJUSTMENT_FACTOR` | `2**3` (= 8) | -| `REWARD_COEFFICIENT_BASE` | `2**20` ( = 1,048,576) | +| `REWARD_COEFFICIENT_BASE` | `2**20` (= 1,048,576) | ### Signature domain types -The following types are defined, mapping into `DomainType` (little endian): - | Name | Value | | - | - | | `DOMAIN_SHARD_PROPOSER` | `128` | | `DOMAIN_SHARD_ATTESTER` | `129` | -### TODO PLACEHOLDER +## Containers -| Name | Value | -| - | - | -| `PLACEHOLDER` | `2**3` | - -## Data structures - -_Note: the shard block header structure is carefully designed so that all of the values have the same depth in a hash tree implementation, so `hash_tree_root(SSZ_partial(x)) == hash_tree_root(x)` (using the "left-to-right leaves" scheme [here](https://github.com/ethereum/eth2.0-specs/issues/1303)), which allows shard block headers to look like an SSZ object when in the crosslink structure. This is done by balancing it so that 7 or 8 items are on the left side (the "core") and two 96-byte (ie. 3*2 = 6 chunk) items are on the right side. Change with care._ - -### `ShardBlockHeader` +### `ShardBlockSignatures` ```python -class ShardBlockHeader(Container): - core: ShardBlockCore - signatures: ShardBlockSignatures +class ShardBlockSignatures(Container): + attesters: BLSSignature + proposer: BLSSignature ``` ### `ShardBlock` ```python class ShardBlock(Container): - core: ExtendedShardBlockCore + slot: ShardSlot + beacon_chain_root: Hash + parent_root: Hash + state_root: Hash + aggregation_bits: Bitvector[TARGET_PERSISTENT_COMMITTEE_SIZE * 2] + total_bytes: uint64 + body: Bytes[SHARD_BLOCK_SIZE_LIMIT - SHARD_HEADER_SIZE] + padding: Bytes[32] signatures: ShardBlockSignatures ``` -### `ShardBlockSignatures` +### `ShardBlockHeader` ```python -class ShardBlockSignatures(Container): - attestation_signature: BLSSignature - proposer_signature: BLSSignature -``` - -### `ShardBlockCore` - -```python -class ShardBlockCore(Container): +class ShardBlockHeader(Container): slot: ShardSlot beacon_chain_root: Hash parent_root: Hash - data_root: Hash state_root: Hash + aggregation_bits: Bitvector[TARGET_PERSISTENT_COMMITTEE_SIZE * 2] total_bytes: uint64 - attester_bitfield: Bitvector[TARGET_PERSISTENT_COMMITTEE_SIZE * 2] -``` - -### `ExtendedShardBlockCore` - -```python -class ExtendedShardBlockCore(Container): - slot: ShardSlot - beacon_chain_root: Hash - parent_root: Hash - data: Bytes[SHARD_BLOCK_SIZE_LIMIT - SHARD_HEADER_SIZE] - state_root: Hash - total_bytes: uint64 - attester_bitfield: Bitvector[TARGET_PERSISTENT_COMMITTEE_SIZE * 2] + body_root: Hash + padding: Bytes[32] + signatures: ShardBlockSignatures ``` ### `ShardState` @@ -179,7 +160,7 @@ class ShardState(Container): basefee: Gwei slot: ShardSlot shard: Shard - most_recent_block_core: ShardBlockCore + latest_block_header: ShardBlockHeader receipt_root: Hash total_bytes: uint64 ``` @@ -195,34 +176,67 @@ class ShardReceiptDelta(Container): ## Helper functions -### `compute_slot_of_shard_slot` +### Misc + +#### `pad` ```python -def compute_slot_of_shard_slot(slot: ShardSlot) -> Epoch: - return Epoch(slot // SHARD_SLOTS_PER_BEACON_SLOT) +def pad(x: bytes, length: uint64) -> bytes: + assert len(x) <= length + return x + b'\x00' * (length - len(x)) ``` -### `compute_epoch_of_shard_slot` +#### `compute_epoch_of_shard_slot` ```python def compute_epoch_of_shard_slot(slot: ShardSlot) -> Epoch: - return Epoch(slot // SHARD_SLOTS_PER_BEACON_SLOT // SLOTS_PER_EPOCH) + return compute_epoch_of_slot(compute_slot_of_shard_slot(slot)) ``` -### `get_shard_period_start_epoch` +#### `compute_period_start_epoch` ```python -def get_shard_period_start_epoch(epoch: Epoch, lookback: int=0) -> Epoch: +def compute_period_start_epoch(epoch: Epoch, lookback: Epoch=0) -> Epoch: return Epoch(epoch - (epoch % EPOCHS_PER_SHARD_PERIOD) - lookback * EPOCHS_PER_SHARD_PERIOD) ``` -### `get_period_committee` +#### `compute_flat_shard_header` + +```python +def compute_flat_shard_header(block: ShardBlock) -> Bytes[SHARD_HEADER_SIZE]: + """ + Return a flat serialisation of the ``block`` header which preserves hash tree root. + """ + return ( + pad(int_to_bytes(block.slot, length=8), 32) + + block.beacon_chain_root + + block.parent_root + + hash_tree_root(block.body) + + block.state_root + + pad(int_to_bytes(block.total_bytes, length=8), 32) + + bytes([sum([block.aggregation_bits[i + j] << j for j in range(8)]) for i in range(0, 256, 8)]) + + block.padding + + pad(block.signatures.attesters, 128) + + pad(block.signatures.proposer, 128) + ) +``` + +#### `compute_crosslink_data_root` + +```python +def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Hash: + headers = b''.join([compute_flat_shard_header(block) for block in blocks]) + bodies = b''.join([block.body for block in blocks]) + MAX_SIZE = SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK + return hash_tree_root(BytesN[MAX_SIZE](pad(headers + bodies, MAX_SIZE))) +``` + +### State accessors + +#### `get_period_committee` ```python def get_period_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]: - """ - Return committee for a period. Used to construct persistent committees. - """ full_committee = compute_committee( indices=get_active_validator_indices(state, epoch), seed=get_seed(state, epoch), @@ -233,19 +247,12 @@ def get_period_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Sequ return full_committee[:TARGET_PERSISTENT_COMMITTEE_SIZE] ``` -### `get_persistent_committee` +#### `get_persistent_committee` ```python -def get_persistent_committee(state: BeaconState, - shard: Shard, - slot: ShardSlot) -> Sequence[ValidatorIndex]: - """ - Return the persistent committee for the given ``shard`` at the given ``slot``. - """ - epoch = compute_epoch_of_shard_slot(slot) - - earlier_committee = get_period_committee(state, get_shard_period_start_epoch(epoch, lookback=2), shard) - later_committee = get_period_committee(state, get_shard_period_start_epoch(epoch, lookback=1), shard) +def get_persistent_committee(state: BeaconState, shard: Shard, epoch: Epoch) -> Sequence[ValidatorIndex]: + earlier_committee = get_period_committee(state, compute_period_start_epoch(epoch, lookback=2), shard) + later_committee = get_period_committee(state, compute_period_start_epoch(epoch, lookback=1), shard) # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from # later committee; return a sorted list of the union of the two, deduplicated @@ -255,22 +262,17 @@ def get_persistent_committee(state: BeaconState, )) ``` -### `get_shard_block_proposer_index` +#### `get_shard_proposer_index` ```python -def get_shard_block_proposer_index(state: BeaconState, - shard: Shard, - slot: ShardSlot) -> Optional[ValidatorIndex]: - # Randomly shift persistent committee - persistent_committee = list(get_persistent_committee(state, shard, slot)) - current_epoch = get_current_epoch(state) - - active_indices = [i for i in persistent_committee if is_active_validator(state.validators[i], current_epoch)] - if not any(active_indices): - return None +def get_shard_proposer_index(state: BeaconState, shard: Shard, slot: ShardSlot) -> ValidatorIndex: + epoch = get_current_epoch(state) + persistent_committee = list(get_persistent_committee(state, shard, epoch)) + active_indices = [i for i in persistent_committee if is_active_validator(state.validators[i], epoch)] + assert len(active_indices) > 0 MAX_RANDOM_BYTE = 2**8 - 1 - seed = hash(get_seed(state, current_epoch) + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) + seed = hash(get_seed(state, epoch) + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) i = 0 while True: candidate_index = active_indices[(slot + i) % len(active_indices)] @@ -281,68 +283,7 @@ def get_shard_block_proposer_index(state: BeaconState, i += 1 ``` -### `get_shard_header` - -```python -def get_shard_header(block: ShardBlock) -> ShardBlockHeader: - return ShardBlockHeader( - core=ShardBlockCore( - slot=block.core.slot, - beacon_chain_root=block.core.beacon_chain_root, - parent_root=block.core.parent_root, - data_root=hash_tree_root(block.core.data), - state_root=block.core.state_root, - total_bytes=block.core.total_bytes, - attester_bitfield=block.core.attester_bitfield, - ), - signatures=block.signatures, - ) -``` - -### `pad` - -```python -def pad(x: bytes, length: int) -> bytes: - assert len(x) <= length - return x + b'\x00' * (length - len(x)) -``` - -### `flatten_shard_header` - -```python -def flatten_shard_header(header: ShardBlockHeader) -> Bytes[SHARD_HEADER_SIZE]: - """ - Converts a shard block header into a flat object with the same hash tree root. Used - in the crosslink construction. - """ - committee_size = len(header.core.attester_bitfield) - attester_bits = [header.core.attester_bitfield[i] if i < committee_size else 0 for i in range(256)] - attester_bytes = bytes([sum([attester_bits[i + j] << j for j in range(8)]) for i in range(0, 256, 8)]) - return ( - pad(int_to_bytes(header.core.slot, length=8), 32) + - header.core.beacon_chain_root + - header.core.parent_root + - header.core.data_root + - header.core.state_root + - pad(int_to_bytes(header.core.total_bytes, length=8), 32) + - attester_bytes + - b'\x00' * 32 + - pad(header.signatures.attestation_signature, 128) + - pad(header.signatures.proposer_signature, 128) - ) -``` - -### `compute_crosslink_data_root` - -```python -def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Hash: - header = b''.join([flatten_shard_header(get_shard_header(block)) for block in blocks]) - footer = b''.join([block.core.data for block in blocks]) - MAX_SIZE = SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK - return hash_tree_root(BytesN[MAX_SIZE](pad(header + footer, MAX_SIZE))) -``` - -### `get_default_shard_state` +#### `get_default_shard_state` ```python def get_default_shard_state(beacon_state: BeaconState, shard: Shard) -> ShardState: @@ -359,7 +300,7 @@ def get_default_shard_state(beacon_state: BeaconState, shard: Shard) -> ShardSta return ShardState( basefee=1, shard=shard, - slot=PHASE_1_FORK_SLOT, + slot=ShardSlot(PHASE_1_FORK_EPOCH * SHARD_SLOTS_PER_EPOCH), earlier_committee_rewards=[REWARD_COEFFICIENT_BASE for _ in range(len(earlier_committee))], later_committee_rewards=[REWARD_COEFFICIENT_BASE for _ in range(len(later_committee))], earlier_committee_fees=[Gwei(0) for _ in range(len(earlier_committee))], @@ -367,195 +308,206 @@ def get_default_shard_state(beacon_state: BeaconState, shard: Shard) -> ShardSta ) ``` +#### `get_shard_base_reward` + +```python +def get_shard_base_reward(beacon_state: BeaconState) -> Gwei: + total_balance_root = integer_squareroot(get_total_active_balance(beacon_state)) + return Gwei(REWARD_COEFFICIENT_BASE * BASE_REWARD_FACTOR // total_balance_root // BASE_REWARDS_PER_EPOCH) +``` + +### State mutators + +#### `add_reward` + +```python +def add_reward(state: ShardState, beacon_state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: + epoch = compute_epoch_of_shard_slot(state.slot) + earlier_committee = get_period_committee(beacon_state, compute_period_start_epoch(epoch, lookback=2), state.shard) + later_committee = get_period_committee(beacon_state, compute_period_start_epoch(epoch, lookback=1), state.shard) + if index in earlier_committee: + state.earlier_committee_rewards[earlier_committee.index(index)] += delta + elif index in later_committee: + state.later_committee_rewards[later_committee.index(index)] += delta +``` + +#### `add_fee` + +```python +def add_fee(state: ShardState, beacon_state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: + epoch = compute_epoch_of_shard_slot(state.slot) + earlier_committee = get_period_committee(beacon_state, compute_period_start_epoch(epoch, lookback=2), state.shard) + later_committee = get_period_committee(beacon_state, compute_period_start_epoch(epoch, lookback=1), state.shard) + if index in earlier_committee: + state.earlier_committee_fees[earlier_committee.index(index)] += delta + elif index in later_committee: + state.later_committee_fees[later_committee.index(index)] += delta +``` + +## Shard state transition function + +The post-state corresponding to a pre-state `state`, a beacon state `beacon_state`, and a block `block` is defined as `shard_state_transition(state, beacon_state, block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. + +```python +def shard_state_transition(state: ShardState, + beacon_state: BeaconState, + block: ShardBlock, + validate_state_root: bool=False) -> ShardState: + # Process slots (including those with no blocks) since block + process_shard_slots(state, beacon_state, block.slot) + # Process block + process_shard_block(state, beacon_state, block) + # Validate state root (`validate_state_root == True` in production) + if validate_state_root: + assert block.state_root == hash_tree_root(state) + # Return post-state + return state +``` + +```python +def process_shard_slots(state: ShardState, beacon_state: BeaconState, slot: ShardSlot) -> None: + assert state.slot <= slot + while state.slot < slot: + process_shard_slot(state) + # Process period on the start slot of the next period + if (state.slot + 1) % (SHARD_SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD) == 0: + process_shard_period(state) + state.slot += ShardSlot(1) +``` + +```python +def process_shard_slot(state: ShardState, beacon_state: BeaconState, slot: ShardSlot) -> None: + # Cache state root + if state.latest_block_header.state_root == Hash(): + state.latest_block_header.state_root = hash_tree_root(state) + # Save state roots in history accumulator + depth = 0 + state_root = hash_tree_root(state) + while state.slot % 2**depth == 0 and depth < HISTORY_ACCUMULATOR_VECTOR: + state.history_accumulator[depth] = state_root + depth += 1 +``` + +### Period processing + +```python +def process_shard_period(state: ShardState, beacon_state: BeaconState) -> None: + epoch = compute_epoch_of_shard_slot(state.slot) + earlier_committee = get_period_committee( + beacon_state, + compute_period_start_epoch(epoch, lookback=2), + state.shard, + ) + later_committee = get_period_committee( + beacon_state, + compute_period_start_epoch(epoch, lookback=1), + state.shard, + ) + + state.receipt_root = hash_tree_root(List[ShardReceiptDelta, TARGET_PERSISTENT_COMMITTEE_SIZE]([ + ShardReceiptDelta(validator_index, state.earlier_committee_rewards[i], state.earlier_committee_fees[i]) + for i, validator_index in enumerate(earlier_committee) + ])) + state.earlier_committee_rewards = state.later_committee_rewards + state.earlier_committee_fees = state.later_committee_fees + state.later_committee_rewards = [REWARD_COEFFICIENT_BASE for _ in range(len(later_committee))] + state.later_committee_fees = [Gwei(0) for _ in range(len(later_committee))] +``` + +### Block processing + +```python +def process_shard_block(state: ShardState, beacon_state: BeaconState, block: ShardBlock) -> None: + process_shard_block_header(state, beacon_state, block) + process_shard_attestations(state, beacon_state, block + process_shard_block_data_fees(state, beacon_state, block) +``` + +#### Block header + +```python +def process_shard_block_header(state: ShardState, beacon_state: BeaconState, block: ShardBlock) -> None: + # Verify that the slots match + assert block.slot == state.slot + # Verify that the parent matches + if block.parent_root != Hash(): + assert block.parent_root == signing_root(state.latest_block_header) + # Save current block as the new latest block + state.latest_block_header = ShardBlockHeader( + slot=block.slot, + beacon_chain_root=block.beacon_chain_root, + parent_root=block.parent_root, + # `state_root` is zeroed and overwritten in the next `process_shard_slot` call + aggregation_bits=block.aggregation_bits, + total_bytes=block.total_bytes, + body_root=hash_tree_root(block.body), + # `signatures` is zeroed + ) + # Verify proposer signature + proposer_index = get_shard_proposer_index(beacon_state, state.shard, block.slot) + pubkey = beacon_state.validators[proposer_index].pubkey + domain = get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.slot)) + assert bls_verify(pubkey, signing_root(block), block.signatures.proposer, domain) + # Verify total bytes count + state.total_bytes += len(block.body) + assert block.total_bytes == state.total_bytes +``` + +#### Attestations + +```python +def process_shard_attestations(state: ShardState, beacon_state: BeaconState, block: ShardBlock) -> None: + persistent_committee = get_persistent_committee(beacon_state, state.shard, block.slot) + pubkeys = [] + attestation_count = 0 + base_reward = get_shard_base_reward(beacon_state) + for i, validator_index in enumerate(persistent_committee): + if block.aggregation_bits[i]: + pubkeys.append(beacon_state.validators[validator_index].pubkey) + add_reward(state, beacon_state, validator_index, base_reward) + attestation_count += 1 + for i in range(len(persistent_committee), TARGET_PERSISTENT_COMMITTEE_SIZE): + assert block.aggregation_bits[i] is False or block.aggregation_bits[i] == 0 # TODO: Fix Bitvector + # Verify aggregate signature + domain = get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(block.slot)) + assert bls_verify(bls_aggregate_pubkeys(pubkeys), block.parent_root, block.signatures.attesters, domain) + # Proposer micro-rewards + add_reward(state, beacon_state, proposer_index, attestation_count * get_shard_base_reward(beacon_state) // PROPOSER_REWARD_QUOTIENT) +``` + +#### Block data fees + +```python +def process_shard_block_data_fees(state: ShardState, beacon_state: BeaconState, block: ShardBlock) -> None: + base_reward = get_shard_base_reward(beacon_state) + add_fee(state, beacon_state, proposer_index, state.basefee * len(block.body) // SHARD_BLOCK_SIZE_LIMIT) + QUOTIENT = SHARD_BLOCK_SIZE_LIMIT * BASEFEE_ADJUSTMENT_FACTOR + if len(block.body) > SHARD_BLOCK_SIZE_TARGET: + state.basefee += Gwei(max(1, state.basefee * (len(block.body) - SHARD_BLOCK_SIZE_TARGET) // QUOTIENT)) + elif len(block.body) < SHARD_BLOCK_SIZE_TARGET: + state.basefee -= Gwei(max(1, state.basefee * (len(block.body) - SHARD_BLOCK_SIZE_TARGET) // QUOTIENT)) + state.basefee = Gwei(max(1, min( EFFECTIVE_BALANCE_INCREMENT // EPOCHS_PER_SHARD_PERIOD // SHARD_SLOTS_PER_EPOCH, + state.basefee, + ))) +``` + ## Object validity ### Shard block validation: preliminary Accept a shard block `block` only if all of the following are correct: -* Either `block.core.parent_root == Hash()` or a block `parent` such that `hash_tree_root(parent.core) == block.core.parent_root` has already been accepted. -* `block.core.beacon_chain_root == get_block_root(head_beacon_state, compute_epoch_of_shard_slot(parent.core.slot))` where `head_beacon_state` is the current beacon chain head state. Alternatively phrased, a beacon chain block `beacon_ref` such that `signing_root(beacon_ref) == block.core.beacon_chain_root` has already been accepted and is part of the canonical chain, and no block with slot `beacon_ref.slot < slot <= compute_start_slot_of_epoch(compute_epoch_of_shard_slot(parent.core.slot))` is part of the canonical chain. -* Let `beacon_state` be the state where `beacon_ref.state_root == hash_tree_root(beacon_state)`. Let `prev_state` be the post-state of the `parent` if the `parent` exists, otherwise let it be `get_default_shard_state(beacon_state, shard)` (defined below). `block.core.state_root` must equal the `hash_tree_root` of the state after applying `shard_state_transition(prev_state, beacon_state, block)`. +* Either `block.parent_root == Hash()` or a block `parent` such that `signing_root(parent) == block.parent_root` has already been accepted. +* `block.beacon_chain_root == get_block_root(head_beacon_state, compute_epoch_of_shard_slot(parent.slot))` where `head_beacon_state` is the current beacon chain head state. Alternatively phrased, a beacon chain block `beacon_ref` such that `signing_root(beacon_ref) == block.beacon_chain_root` has already been accepted and is part of the canonical chain, and no block with slot `beacon_ref.slot < slot <= compute_start_slot_of_epoch(compute_epoch_of_shard_slot(parent.slot))` is part of the canonical chain. +* Let `beacon_state` be the state where `beacon_ref.state_root == hash_tree_root(beacon_state)`. Let `prev_state` be the post-state of the `parent` if the `parent` exists, otherwise let it be `get_default_shard_state(beacon_state, shard)` (defined below). `block.state_root` must equal the `hash_tree_root` of the state after applying `shard_state_transition(prev_state, beacon_state, block)`. Note that these acceptance conditions depend on the canonical beacon chain; when the canonical beacon chain reorganizes, the eligibility of shard blocks should be re-evaluated. -### Shard state transition function helpers - -```python -def add_reward(state: ShardState, beacon_state: BeaconState, index: ValidatorIndex, delta: int) -> None: - epoch = compute_epoch_of_shard_slot(state.slot) - earlier_committee = get_period_committee( - beacon_state, - get_shard_period_start_epoch(epoch, lookback=2), - state.shard, - ) - later_committee = get_period_committee(beacon_state, get_shard_period_start_epoch(epoch, lookback=1), state.shard) - if index in earlier_committee: - state.earlier_committee_rewards[earlier_committee.index(index)] += delta - elif index in later_committee: - state.later_committee_rewards[later_committee.index(index)] += delta - else: - raise Exception("Should never be here") -``` - -```python -def add_fee(state: ShardState, beacon_state: BeaconState, index: ValidatorIndex, delta: int) -> None: - epoch = compute_epoch_of_shard_slot(state.slot) - earlier_committee = get_period_committee(beacon_state, get_shard_period_start_epoch(epoch, lookback=2), state.shard) - later_committee = get_period_committee(beacon_state, get_shard_period_start_epoch(epoch, lookback=1), state.shard) - if index in earlier_committee: - state.earlier_committee_fees[earlier_committee.index(index)] += delta - elif index in later_committee: - state.later_committee_fees[later_committee.index(index)] += delta - else: - raise Exception("Should never be here") -``` - -### Shard state transition function - -```python -def shard_state_transition(state: ShardState, - beacon_state: BeaconState, - block: ShardBlock, - validate_state_root: bool=False) -> None: - assert block.core.slot > state.slot - for slot in range(state.slot, block.core.slot): - shard_slot_transition(state, beacon_state) - shard_block_transition(state, beacon_state, block, validate_state_root=validate_state_root) -``` - -```python -def shard_slot_transition(state: ShardState, beacon_state: BeaconState) -> None: - # Correct saved state root - if state.most_recent_block_core.state_root == Hash(): - state.most_recent_block_core.state_root = hash_tree_root(state) - - # Save states in history accumulator - depth = 0 - h = hash_tree_root(state) - while state.slot % 2**depth == 0 and depth < HISTORY_ACCUMULATOR_VECTOR: - state.history_accumulator[depth] = h - depth += 1 - - # Period transitions - if (state.slot + 1) % (SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD) == 0: - epoch = compute_epoch_of_shard_slot(state.slot) - earlier_committee = get_period_committee( - beacon_state, - get_shard_period_start_epoch(epoch, lookback=2), - state.shard, - ) - later_committee = get_period_committee( - beacon_state, - get_shard_period_start_epoch(epoch, lookback=1), - state.shard, - ) - state.receipt_root = hash_tree_root(List[ShardReceiptDelta, PLACEHOLDER]([ - ShardReceiptDelta( - index=validator_index, - reward_coefficient=state.earlier_committee_rewards[i], - block_fee=state.earlier_committee_fees[i], - ) - for i, validator_index in enumerate(earlier_committee) - ])) - state.earlier_committee_rewards = state.later_committee_rewards - state.earlier_committee_fees = state.later_committee_fees - state.later_committee_rewards = [REWARD_COEFFICIENT_BASE for _ in range(len(later_committee))], - state.later_committee_fees = [Gwei(0) for _ in range(len(later_committee))], - else: - state.receipt_root = Hash() - state.slot += ShardSlot(1) -``` - -```python -def shard_block_transition(state: ShardState, - beacon_state: BeaconState, - block: ShardBlock, - validate_state_root: bool) -> None: - # Check slot number - assert block.core.slot == state.slot - - # Check parent block - if block.core.parent_root != Hash(): - assert block.core.parent_root == hash_tree_root(state.most_recent_block_core) - - # Calculate base reward - total_balance = get_total_active_balance(beacon_state) - base_reward = ( - REWARD_COEFFICIENT_BASE * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH - ) - # Check attestations - attester_committee = get_persistent_committee(beacon_state, state.shard, block.core.slot) - pubkeys = [] - attestations = 0 - - for i, validator_index in enumerate(attester_committee): - if block.core.attester_bitfield[i]: - pubkeys.append(beacon_state.validators[validator_index].pubkey) - add_reward(state, beacon_state, validator_index, base_reward) - attestations += 1 - - for i in range(len(attester_committee), TARGET_PERSISTENT_COMMITTEE_SIZE): - assert block.core.attester_bitfield[i] is False or block.core.attester_bitfield[i] == 0 # TODO: FIX Bitvector - - assert bls_verify( - pubkey=bls_aggregate_pubkeys(pubkeys), - message_hash=block.core.parent_root, - signature=block.signatures.attestation_signature, - domain=get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(block.core.slot)) - ) - - # Check proposer - proposer_index = get_shard_block_proposer_index(beacon_state, state.shard, block.core.slot) - assert proposer_index is not None - add_reward(state, beacon_state, proposer_index, attestations * base_reward // PROPOSER_REWARD_QUOTIENT) - assert bls_verify( - pubkey=beacon_state.validators[proposer_index].pubkey, - message_hash=hash_tree_root(block.core), - signature=block.signatures.proposer_signature, - domain=get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.core.slot)), - ) - - # Process and update block data fees - add_fee(state, beacon_state, proposer_index, state.basefee * len(block.core.data) // SHARD_BLOCK_SIZE_LIMIT) - QUOTIENT = SHARD_BLOCK_SIZE_LIMIT * BASEFEE_ADJUSTMENT_FACTOR - if len(block.core.data) > SHARD_BLOCK_SIZE_TARGET: - state.basefee += Gwei(max(1, state.basefee * (len(block.core.data) - SHARD_BLOCK_SIZE_TARGET) // QUOTIENT)) - elif len(block.core.data) < SHARD_BLOCK_SIZE_TARGET: - state.basefee -= Gwei(max(1, state.basefee * (len(block.core.data) - SHARD_BLOCK_SIZE_TARGET) // QUOTIENT)) - state.basefee = Gwei(max( - 1, - min( - EFFECTIVE_BALANCE_INCREMENT // EPOCHS_PER_SHARD_PERIOD // SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH, - state.basefee, - ) - )) - - # Check total bytes - state.total_bytes += len(block.core.data) - assert block.core.total_bytes == state.total_bytes - - # Update in-state block header - state.most_recent_block_core = ShardBlockCore( - slot=block.core.slot, - beacon_chain_root=block.core.beacon_chain_root, - parent_root=block.core.parent_root, - data_root=hash_tree_root(block.core.data), - state_root=Hash(), - total_bytes=block.core.total_bytes, - attester_bitfield=block.core.attester_bitfield, - ) - - # Check state root - if validate_state_root: - assert block.core.state_root == hash_tree_root(state) -``` - ### Beacon attestations Let: -- `pre_state` is the `ShardState` before processing any blocks +- `pre_state` be the `ShardState` before processing any blocks - `shard_blocks_or_state_roots` be the `Union[ShardBlock, Hash]` list such that `shard_blocks[slot]` is the canonical `ShardBlock` for shard `pre_state.shard` at slot `slot` if a block exists, or the post-state-root of processing state up to and including that slot if a block does not exist. - `beacon_state` be the canonical `BeaconState` - `valid_attestations` be the set of valid `Attestation` objects, recursively defined @@ -594,12 +546,9 @@ def is_valid_beacon_attestation(pre_state: ShardState, blocks.append(shard_blocks_or_state_roots[slot]) else: blocks.append(ShardBlock( - core=ExtendedShardBlockCore( - slot=slot, - state_root=shard_blocks_or_state_roots[slot], - total_bytes=pre_state.total_bytes, - ), - signatures=ShardBlockSignatures(), + slot=slot, + state_root=shard_blocks_or_state_roots[slot], + total_bytes=pre_state.total_bytes, )) assert candidate.data.crosslink.data_root == compute_crosslink_data_root(blocks) From d7e628e08fefbd00a88cb204fa554548b96fbd23 Mon Sep 17 00:00:00 2001 From: Justin Date: Wed, 28 Aug 2019 22:57:24 +0100 Subject: [PATCH 02/28] WIP! --- specs/core/1_shard-data-chains.md | 550 +++++++++++++----------------- 1 file changed, 245 insertions(+), 305 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 9335c1803..367091c91 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -19,43 +19,43 @@ - [Signature domain types](#signature-domain-types) - [Containers](#containers) - [`ShardBlockSignatures`](#shardblocksignatures) + - [`ShardBlockData`](#shardblockdata) - [`ShardBlock`](#shardblock) + - [`ShardBlockHeaderData`](#shardblockheaderdata) - [`ShardBlockHeader`](#shardblockheader) - [`ShardState`](#shardstate) - - [`ShardReceiptDelta`](#shardreceiptdelta) + - [`ShardReceipt`](#shardreceipt) + - [`ShardCheckpoint`](#shardcheckpoint) - [Helper functions](#helper-functions) - [Misc](#misc-1) - - [`pad`](#pad) - - [`compute_slot_of_shard_slot`](#compute_slot_of_shard_slot) + - [`compute_padded_data`](#compute_padded_data) - [`compute_epoch_of_shard_slot`](#compute_epoch_of_shard_slot) - - [`compute_period_start_epoch`](#compute_period_start_epoch) + - [`compute_shard_period_start_epoch`](#compute_shard_period_start_epoch) - [`compute_flat_shard_header`](#compute_flat_shard_header) - [`compute_crosslink_data_root`](#compute_crosslink_data_root) - - [State accessors](#state-accessors) + - [Beacon state accessors](#beacon-state-accessors) - [`get_period_committee`](#get_period_committee) - - [`get_persistent_committee`](#get_persistent_committee) + - [`get_shard_committee`](#get_shard_committee) - [`get_shard_proposer_index`](#get_shard_proposer_index) - - [`get_default_shard_state`](#get_default_shard_state) - - [`get_shard_base_reward`](#get_shard_base_reward) - - [State mutators](#state-mutators) - - [`add_fee`](#add_fee) + - [Shard state mutators](#shard-state-mutators) - [`add_reward`](#add_reward) + - [`add_fee`](#add_fee) + - [Genesis](#genesis) + - [`get_genesis_shard_state`](#get_genesis_shard_state) + - [`get_genesis_shard_block`](#get_genesis_shard_block) - [Shard state transition function](#shard-state-transition-function) - [Period processing](#period-processing) - [Block processing](#block-processing) - [Block header](#block-header) - [Attestations](#attestations) - - [Block data fees](#block-data-fees) - - [Object validity](#object-validity) - - [Shard block validation: preliminary](#shard-block-validation-preliminary) - - [Beacon attestations](#beacon-attestations) + - [Block size fee](#block-size-fee) - [Shard fork choice rule](#shard-fork-choice-rule) ## Introduction -This document describes the shard data layer and the shard fork choice rule in Phase 1 of Ethereum 2.0. +This document describes the shard transition function (data layer only) and the shard fork choice rule as part of Phase 1 of Ethereum 2.0. ## Custom types @@ -69,8 +69,8 @@ This document describes the shard data layer and the shard fork choice rule in P | Name | Value | | - | - | -| `SHARD_SLOTS_PER_EPOCH` | `2**7` (= 128) | -| `TARGET_PERSISTENT_COMMITTEE_SIZE` | `2**7` (= 128) | +| `MIN_BLOCK_SIZE_PRICE` | `2**0` (= 1) | +| `MAX_PERIOD_COMMITTEE_SIZE` | `2**7` (= 128) | | `SHARD_HEADER_SIZE` | `2**9` (= 512) | | `SHARD_BLOCK_SIZE_TARGET` | `2**14` (= 16,384) | | `SHARD_BLOCK_SIZE_LIMIT` | `2**16` (= 65,536) | @@ -79,13 +79,13 @@ This document describes the shard data layer and the shard fork choice rule in P | Name | Value | | - | - | -| `PHASE_1_FORK_EPOCH` | **TBD** | +| `SHARD_GENESIS_EPOCH` | **TBD** | ### Time parameters | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `CROSSLINK_LOOKBACK` | `2**0` (= 1) | epochs | 6.4 minutes | +| `SHARD_SLOTS_PER_EPOCH` | `2**7` (= 128) | shard slots | 6.4 minutes | | `EPOCHS_PER_SHARD_PERIOD` | `2**8` (= 256) | epochs | ~27 hours | ### State list lengths @@ -98,8 +98,7 @@ This document describes the shard data layer and the shard fork choice rule in P | Name | Value | | - | - | -| `BASEFEE_ADJUSTMENT_FACTOR` | `2**3` (= 8) | -| `REWARD_COEFFICIENT_BASE` | `2**20` (= 1,048,576) | +| `BLOCK_SIZE_PRICE_QUOTIENT` | `2**3` (= 8) | ### Signature domain types @@ -118,33 +117,45 @@ class ShardBlockSignatures(Container): proposer: BLSSignature ``` +### `ShardBlockData` + +```python +class ShardBlockData(Container): + slot: ShardSlot + beacon_block_root: Hash + parent_root: Hash + state_root: Hash + aggregation_bits: Bitvector[2 * MAX_PERIOD_COMMITTEE_SIZE] + block_size_sum: uint64 + body: List[byte, SHARD_BLOCK_SIZE_LIMIT - SHARD_HEADER_SIZE] +``` + ### `ShardBlock` ```python class ShardBlock(Container): + data: ShardBlockData + signatures: ShardBlockSignatures +``` + +### `ShardBlockHeaderData` + +```python +class ShardBlockHeaderData(Container): slot: ShardSlot - beacon_chain_root: Hash + beacon_block_root: Hash parent_root: Hash state_root: Hash - aggregation_bits: Bitvector[TARGET_PERSISTENT_COMMITTEE_SIZE * 2] - total_bytes: uint64 - body: Bytes[SHARD_BLOCK_SIZE_LIMIT - SHARD_HEADER_SIZE] - padding: Bytes[32] - signatures: ShardBlockSignatures + aggregation_bits: Bitvector[2 * MAX_PERIOD_COMMITTEE_SIZE] + block_size_sum: uint64 + body_root: Hash ``` ### `ShardBlockHeader` ```python class ShardBlockHeader(Container): - slot: ShardSlot - beacon_chain_root: Hash - parent_root: Hash - state_root: Hash - aggregation_bits: Bitvector[TARGET_PERSISTENT_COMMITTEE_SIZE * 2] - total_bytes: uint64 - body_root: Hash - padding: Bytes[32] + data: ShardBlockHeaderData signatures: ShardBlockSignatures ``` @@ -152,51 +163,60 @@ class ShardBlockHeader(Container): ```python class ShardState(Container): - history_accumulator: Vector[Hash, HISTORY_ACCUMULATOR_VECTOR] - earlier_committee_rewards: List[uint64, TARGET_PERSISTENT_COMMITTEE_SIZE] - later_committee_rewards: List[uint64, TARGET_PERSISTENT_COMMITTEE_SIZE] - earlier_committee_fees: List[Gwei, TARGET_PERSISTENT_COMMITTEE_SIZE] - later_committee_fees: List[Gwei, TARGET_PERSISTENT_COMMITTEE_SIZE] - basefee: Gwei - slot: ShardSlot shard: Shard - latest_block_header: ShardBlockHeader + slot: ShardSlot + history_accumulator: Vector[Hash, HISTORY_ACCUMULATOR_VECTOR] + latest_block_header_data: ShardBlockHeader receipt_root: Hash - total_bytes: uint64 + block_size_sum: uint64 + # Rewards and fees + block_size_price: Gwei + older_committee_rewards: List[Gwei, MAX_PERIOD_COMMITTEE_SIZE] + newer_committee_rewards: List[Gwei, MAX_PERIOD_COMMITTEE_SIZE] + older_committee_fees: List[Gwei, MAX_PERIOD_COMMITTEE_SIZE] + newer_committee_fees: List[Gwei, MAX_PERIOD_COMMITTEE_SIZE] ``` -### `ShardReceiptDelta` +### `ShardReceipt` ```python -class ShardReceiptDelta(Container): +class ShardReceipt(Container): index: ValidatorIndex - reward_coefficient: uint64 - block_fee: Gwei + rewards: Gwei + fees: Gwei +``` + +### `ShardCheckpoint` + +```python +class ShardCheckpoint(Container): + slot: ShardSlot + parent_root: Hash ``` ## Helper functions ### Misc -#### `pad` +#### `compute_padded_data` ```python -def pad(x: bytes, length: uint64) -> bytes: - assert len(x) <= length - return x + b'\x00' * (length - len(x)) +def compute_padded_data(data: bytes, length: uint64) -> bytes: + assert len(data) <= length + return data + b'\x00' * (length - len(data)) ``` #### `compute_epoch_of_shard_slot` ```python def compute_epoch_of_shard_slot(slot: ShardSlot) -> Epoch: - return compute_epoch_of_slot(compute_slot_of_shard_slot(slot)) + return compute_epoch_of_slot(slot // SHARD_SLOTS_PER_EPOCH) ``` -#### `compute_period_start_epoch` +#### `compute_shard_period_start_epoch` ```python -def compute_period_start_epoch(epoch: Epoch, lookback: Epoch=0) -> Epoch: +def compute_shard_period_start_epoch(epoch: Epoch, lookback: uint64) -> Epoch: return Epoch(epoch - (epoch % EPOCHS_PER_SHARD_PERIOD) - lookback * EPOCHS_PER_SHARD_PERIOD) ``` @@ -205,19 +225,22 @@ def compute_period_start_epoch(epoch: Epoch, lookback: Epoch=0) -> Epoch: ```python def compute_flat_shard_header(block: ShardBlock) -> Bytes[SHARD_HEADER_SIZE]: """ - Return a flat serialisation of the ``block`` header which preserves hash tree root. + Return a flat serialisation of the ``block`` header, preserving hash tree root. """ + data = block.data return ( - pad(int_to_bytes(block.slot, length=8), 32) + - block.beacon_chain_root + - block.parent_root + - hash_tree_root(block.body) + - block.state_root + - pad(int_to_bytes(block.total_bytes, length=8), 32) + - bytes([sum([block.aggregation_bits[i + j] << j for j in range(8)]) for i in range(0, 256, 8)]) + - block.padding + - pad(block.signatures.attesters, 128) + - pad(block.signatures.proposer, 128) + # Left half of the hash tree + compute_padded_data(int_to_bytes(data.slot, length=8), 32) + + data.beacon_block_root + + data.parent_root + + hash_tree_root(data.body) + + data.state_root + + compute_padded_data(int_to_bytes(data.block_size_sum, length=8), 32) + + bytes([sum([data.aggregation_bits[i + j] << j for j in range(8)]) for i in range(0, 256, 8)]) + + Bytes32() + # Padding + # Right half of the hash tree + compute_padded_data(block.signatures.attesters, 128) + + compute_padded_data(block.signatures.proposer, 128) ) ``` @@ -226,40 +249,32 @@ def compute_flat_shard_header(block: ShardBlock) -> Bytes[SHARD_HEADER_SIZE]: ```python def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Hash: headers = b''.join([compute_flat_shard_header(block) for block in blocks]) - bodies = b''.join([block.body for block in blocks]) - MAX_SIZE = SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK - return hash_tree_root(BytesN[MAX_SIZE](pad(headers + bodies, MAX_SIZE))) + bodies = b''.join([block.data.body for block in blocks]) + MAX_SIZE = MAX_EPOCHS_PER_CROSSLINK * SHARD_SLOTS_PER_EPOCH * SHARD_BLOCK_SIZE_LIMIT + return hash_tree_root(BytesN[MAX_SIZE](compute_padded_data(headers + bodies, MAX_SIZE))) ``` -### State accessors +### Beacon state accessors #### `get_period_committee` ```python -def get_period_committee(state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]: - full_committee = compute_committee( - indices=get_active_validator_indices(state, epoch), - seed=get_seed(state, epoch), - index=shard, - count=SHARD_COUNT, - ) - - return full_committee[:TARGET_PERSISTENT_COMMITTEE_SIZE] +def get_period_committee(state: BeaconState, shard: Shard, epoch: Epoch) -> Sequence[ValidatorIndex]: + active_validator_indices = get_active_validator_indices(state, epoch) + seed = get_seed(state, epoch) + return compute_committee(active_validator_indices, seed, shard, SHARD_COUNT)[:MAX_PERIOD_COMMITTEE_SIZE] ``` -#### `get_persistent_committee` +#### `get_shard_committee` ```python -def get_persistent_committee(state: BeaconState, shard: Shard, epoch: Epoch) -> Sequence[ValidatorIndex]: - earlier_committee = get_period_committee(state, compute_period_start_epoch(epoch, lookback=2), shard) - later_committee = get_period_committee(state, compute_period_start_epoch(epoch, lookback=1), shard) - - # Take not-yet-cycled-out validators from earlier committee and already-cycled-in validators from - # later committee; return a sorted list of the union of the two, deduplicated - return sorted(set( - [i for i in earlier_committee if epoch % EPOCHS_PER_SHARD_PERIOD < i % EPOCHS_PER_SHARD_PERIOD] - + [i for i in later_committee if epoch % EPOCHS_PER_SHARD_PERIOD >= i % EPOCHS_PER_SHARD_PERIOD] - )) +def get_shard_committee(state: BeaconState, shard: Shard, epoch: Epoch) -> Sequence[ValidatorIndex]: + older_committee = get_period_committee(state, shard, compute_shard_period_start_epoch(epoch, 2)) + newer_committee = get_period_committee(state, shard, compute_shard_period_start_epoch(epoch, 1)) + # Every epoch cycle out validators from the older committee and cycle in validators from the newer committee + older_subcommittee = [i for i in older_committee if i % EPOCHS_PER_SHARD_PERIOD > epoch % EPOCHS_PER_SHARD_PERIOD] + newer_subcommittee = [i for i in newer_committee if i % EPOCHS_PER_SHARD_PERIOD <= epoch % EPOCHS_PER_SHARD_PERIOD] + return older_subcommittee + newer_subcommittee ``` #### `get_shard_proposer_index` @@ -267,294 +282,219 @@ def get_persistent_committee(state: BeaconState, shard: Shard, epoch: Epoch) -> ```python def get_shard_proposer_index(state: BeaconState, shard: Shard, slot: ShardSlot) -> ValidatorIndex: epoch = get_current_epoch(state) - persistent_committee = list(get_persistent_committee(state, shard, epoch)) - active_indices = [i for i in persistent_committee if is_active_validator(state.validators[i], epoch)] - assert len(active_indices) > 0 - - MAX_RANDOM_BYTE = 2**8 - 1 - seed = hash(get_seed(state, epoch) + int_to_bytes(shard, length=8) + int_to_bytes(slot, length=8)) - i = 0 - while True: - candidate_index = active_indices[(slot + i) % len(active_indices)] - random_byte = hash(seed + int_to_bytes(i // 32, length=8))[i % 32] - effective_balance = state.validators[candidate_index].effective_balance - if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: - return ValidatorIndex(candidate_index) - i += 1 + active_indices = [i for i in get_shard_committee(state, shard, epoch) if is_active_validator(state.validators[i], epoch)] + seed = hash(get_seed(state, epoch) + int_to_bytes(slot, length=8) + int_to_bytes(shard, length=8)) + compute_proposer_index(state, active_indices, seed) ``` -#### `get_default_shard_state` - -```python -def get_default_shard_state(beacon_state: BeaconState, shard: Shard) -> ShardState: - earlier_committee = get_period_committee( - beacon_state, - Epoch(PHASE_1_FORK_EPOCH - EPOCHS_PER_SHARD_PERIOD * 2), - shard, - ) - later_committee = get_period_committee( - beacon_state, - Epoch(PHASE_1_FORK_EPOCH - EPOCHS_PER_SHARD_PERIOD), - shard, - ) - return ShardState( - basefee=1, - shard=shard, - slot=ShardSlot(PHASE_1_FORK_EPOCH * SHARD_SLOTS_PER_EPOCH), - earlier_committee_rewards=[REWARD_COEFFICIENT_BASE for _ in range(len(earlier_committee))], - later_committee_rewards=[REWARD_COEFFICIENT_BASE for _ in range(len(later_committee))], - earlier_committee_fees=[Gwei(0) for _ in range(len(earlier_committee))], - later_committee_fees=[Gwei(0) for _ in range(len(later_committee))], - ) -``` - -#### `get_shard_base_reward` - -```python -def get_shard_base_reward(beacon_state: BeaconState) -> Gwei: - total_balance_root = integer_squareroot(get_total_active_balance(beacon_state)) - return Gwei(REWARD_COEFFICIENT_BASE * BASE_REWARD_FACTOR // total_balance_root // BASE_REWARDS_PER_EPOCH) -``` - -### State mutators +### Shard state mutators #### `add_reward` ```python -def add_reward(state: ShardState, beacon_state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: +def add_reward(state: BeaconState, shard_state: ShardState, index: ValidatorIndex, delta: Gwei) -> None: epoch = compute_epoch_of_shard_slot(state.slot) - earlier_committee = get_period_committee(beacon_state, compute_period_start_epoch(epoch, lookback=2), state.shard) - later_committee = get_period_committee(beacon_state, compute_period_start_epoch(epoch, lookback=1), state.shard) - if index in earlier_committee: - state.earlier_committee_rewards[earlier_committee.index(index)] += delta - elif index in later_committee: - state.later_committee_rewards[later_committee.index(index)] += delta + older_committee = get_period_committee(state, shard_state.shard, compute_shard_period_start_epoch(epoch, 2)) + newer_committee = get_period_committee(state, shard_state.shard, compute_shard_period_start_epoch(epoch, 1)) + if index in older_committee: + shard_state.older_committee_rewards[older_committee.index(index)] += delta + elif index in newer_committee: + shard_state.newer_committee_rewards[newer_committee.index(index)] += delta ``` #### `add_fee` ```python -def add_fee(state: ShardState, beacon_state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: +def add_fee(state: BeaconState, shard_state: ShardState, index: ValidatorIndex, delta: Gwei) -> None: epoch = compute_epoch_of_shard_slot(state.slot) - earlier_committee = get_period_committee(beacon_state, compute_period_start_epoch(epoch, lookback=2), state.shard) - later_committee = get_period_committee(beacon_state, compute_period_start_epoch(epoch, lookback=1), state.shard) - if index in earlier_committee: - state.earlier_committee_fees[earlier_committee.index(index)] += delta - elif index in later_committee: - state.later_committee_fees[later_committee.index(index)] += delta + older_committee = get_period_committee(state, shard_state.shard, compute_shard_period_start_epoch(epoch, 2)) + newer_committee = get_period_committee(state, shard_state.shard, compute_shard_period_start_epoch(epoch, 1)) + if index in older_committee: + shard_state.older_committee_fees[older_committee.index(index)] += delta + elif index in newer_committee: + shard_state.newer_committee_fees[newer_committee.index(index)] += delta +``` + +## Genesis + +### `get_genesis_shard_state` + +```python +def get_genesis_shard_state(state: BeaconState, shard: Shard) -> ShardState: + older_committee = get_period_committee(state, shard, compute_shard_period_start_epoch(SHARD_GENESIS_EPOCH, 2)) + newer_committee = get_period_committee(state, shard, compute_shard_period_start_epoch(SHARD_GENESIS_EPOCH, 1)) + return ShardState( + shard=shard, + slot=ShardSlot(SHARD_GENESIS_EPOCH * SHARD_SLOTS_PER_EPOCH), + block_size_price=MIN_BLOCK_SIZE_PRICE, + older_committee_rewards=[Gwei(0) for _ in range(len(older_committee))], + newer_committee_rewards=[Gwei(0) for _ in range(len(newer_committee))], + older_committee_fees=[Gwei(0) for _ in range(len(older_committee))], + newer_committee_fees=[Gwei(0) for _ in range(len(newer_committee))], + ) +``` + +### `get_genesis_shard_block` + +```python +def get_genesis_shard_block(state: BeaconState, shard: Shard) -> ShardBlock: + genesis_state = get_genesis_shard_state(state, shard) + return ShardBlock(data=ShardBlockData( + shard=shard, + slot=ShardSlot(SHARD_GENESIS_EPOCH * SHARD_SLOTS_PER_EPOCH), + state_root=hash_tree_root(genesis_state), + )) ``` ## Shard state transition function -The post-state corresponding to a pre-state `state`, a beacon state `beacon_state`, and a block `block` is defined as `shard_state_transition(state, beacon_state, block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. - ```python -def shard_state_transition(state: ShardState, - beacon_state: BeaconState, +def shard_state_transition(state: BeaconState, + shard_state: ShardState, block: ShardBlock, validate_state_root: bool=False) -> ShardState: # Process slots (including those with no blocks) since block - process_shard_slots(state, beacon_state, block.slot) + process_shard_slots(state, shard_state, block.data.slot) # Process block - process_shard_block(state, beacon_state, block) + process_shard_block(state, shard_state, block) # Validate state root (`validate_state_root == True` in production) if validate_state_root: - assert block.state_root == hash_tree_root(state) + assert block.data.state_root == hash_tree_root(shard_state) # Return post-state - return state + return shard_state ``` ```python -def process_shard_slots(state: ShardState, beacon_state: BeaconState, slot: ShardSlot) -> None: - assert state.slot <= slot - while state.slot < slot: - process_shard_slot(state) +def process_shard_slots(state: BeaconState, shard_state: ShardState, slot: ShardSlot) -> None: + assert shard_state.slot <= slot + while shard_state.slot < slot: + process_shard_slot(state, shard_state) # Process period on the start slot of the next period - if (state.slot + 1) % (SHARD_SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD) == 0: - process_shard_period(state) - state.slot += ShardSlot(1) + if (shard_state.slot + 1) % (SHARD_SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD) == 0: + process_shard_period(state, shard_state) + shard_state.slot += ShardSlot(1) ``` ```python -def process_shard_slot(state: ShardState, beacon_state: BeaconState, slot: ShardSlot) -> None: +def process_shard_slot(state: BeaconState, shard_state: ShardState) -> None: # Cache state root - if state.latest_block_header.state_root == Hash(): - state.latest_block_header.state_root = hash_tree_root(state) - # Save state roots in history accumulator + previous_state_root = hash_tree_root(state) + if state.latest_block_header_data.state_root == Bytes32(): + state.latest_block_header_data.state_root = previous_state_root + # Cache state root in history accumulator depth = 0 - state_root = hash_tree_root(state) while state.slot % 2**depth == 0 and depth < HISTORY_ACCUMULATOR_VECTOR: - state.history_accumulator[depth] = state_root + state.history_accumulator[depth] = previous_state_root depth += 1 ``` ### Period processing ```python -def process_shard_period(state: ShardState, beacon_state: BeaconState) -> None: +def process_shard_period(shard_state: ShardState, state: BeaconState) -> None: epoch = compute_epoch_of_shard_slot(state.slot) - earlier_committee = get_period_committee( - beacon_state, - compute_period_start_epoch(epoch, lookback=2), - state.shard, - ) - later_committee = get_period_committee( - beacon_state, - compute_period_start_epoch(epoch, lookback=1), - state.shard, - ) - - state.receipt_root = hash_tree_root(List[ShardReceiptDelta, TARGET_PERSISTENT_COMMITTEE_SIZE]([ - ShardReceiptDelta(validator_index, state.earlier_committee_rewards[i], state.earlier_committee_fees[i]) - for i, validator_index in enumerate(earlier_committee) + older_committee = get_period_committee(state, state.shard, compute_shard_period_start_epoch(epoch, 2)) + newer_committee = get_period_committee(state, state.shard, compute_shard_period_start_epoch(epoch, 1)) + # Compute receipt root for older committee + state.receipt_root = hash_tree_root(List[ShardReceipt, MAX_PERIOD_COMMITTEE_SIZE]([ + ShardReceipt(validator_index, state.older_committee_rewards[i], state.older_committee_fees[i]) + for i, validator_index in enumerate(older_committee) ])) - state.earlier_committee_rewards = state.later_committee_rewards - state.earlier_committee_fees = state.later_committee_fees - state.later_committee_rewards = [REWARD_COEFFICIENT_BASE for _ in range(len(later_committee))] - state.later_committee_fees = [Gwei(0) for _ in range(len(later_committee))] + # Rotate rewards and fees + state.older_committee_rewards = state.newer_committee_rewards + state.newer_committee_rewards = [Gwei(0) for _ in range(len(newer_committee))] + state.older_committee_fees = state.newer_committee_fees + state.newer_committee_fees = [Gwei(0) for _ in range(len(newer_committee))] ``` ### Block processing ```python -def process_shard_block(state: ShardState, beacon_state: BeaconState, block: ShardBlock) -> None: - process_shard_block_header(state, beacon_state, block) - process_shard_attestations(state, beacon_state, block - process_shard_block_data_fees(state, beacon_state, block) +def process_shard_block(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: + process_shard_block_header(state, shard_state, block) + process_shard_attestations(state, shard_state, block) + process_shard_block_size_fee(state, shard_state, block) ``` #### Block header ```python -def process_shard_block_header(state: ShardState, beacon_state: BeaconState, block: ShardBlock) -> None: +def process_shard_block_header(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: # Verify that the slots match - assert block.slot == state.slot + data = block.data + assert data.slot == state.slot + # Verify that the beacon chain root matches + parent_epoch = compute_epoch_of_shard_slot(state.latest_block_header_data.slot) + assert data.beacon_block_root == get_block_root(state, parent_epoch) # Verify that the parent matches - if block.parent_root != Hash(): - assert block.parent_root == signing_root(state.latest_block_header) + assert data.parent_root == hash_tree_root(state.latest_block_header_data) # Save current block as the new latest block - state.latest_block_header = ShardBlockHeader( - slot=block.slot, - beacon_chain_root=block.beacon_chain_root, - parent_root=block.parent_root, + state.latest_block_header_data = ShardBlockHeaderData( + slot=data.slot, + beacon_block_root=data.beacon_block_root, + parent_root=data.parent_root, # `state_root` is zeroed and overwritten in the next `process_shard_slot` call - aggregation_bits=block.aggregation_bits, - total_bytes=block.total_bytes, - body_root=hash_tree_root(block.body), - # `signatures` is zeroed + aggregation_bits=data.aggregation_bits, + block_size_sum=data.block_size_sum, + body_root=hash_tree_root(data.body), ) # Verify proposer signature - proposer_index = get_shard_proposer_index(beacon_state, state.shard, block.slot) - pubkey = beacon_state.validators[proposer_index].pubkey - domain = get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.slot)) - assert bls_verify(pubkey, signing_root(block), block.signatures.proposer, domain) - # Verify total bytes count - state.total_bytes += len(block.body) - assert block.total_bytes == state.total_bytes + proposer_index = get_shard_proposer_index(state, state.shard, data.slot) + pubkey = state.validators[proposer_index].pubkey + domain = get_domain(state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(data.slot)) + assert bls_verify(pubkey, hash_tree_root(block.data), block.signatures.proposer, domain) + # Verify total body bytes count + state.block_size_sum += SHARD_HEADER_SIZE + len(data.body) + assert data.block_size_sum == state.block_size_sum ``` #### Attestations ```python -def process_shard_attestations(state: ShardState, beacon_state: BeaconState, block: ShardBlock) -> None: - persistent_committee = get_persistent_committee(beacon_state, state.shard, block.slot) +def process_shard_attestations(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: + data = block.data pubkeys = [] attestation_count = 0 - base_reward = get_shard_base_reward(beacon_state) - for i, validator_index in enumerate(persistent_committee): - if block.aggregation_bits[i]: - pubkeys.append(beacon_state.validators[validator_index].pubkey) - add_reward(state, beacon_state, validator_index, base_reward) + shard_committee = get_shard_committee(state, state.shard, data.slot) + for i, validator_index in enumerate(shard_committee): + if data.aggregation_bits[i]: + pubkeys.append(state.validators[validator_index].pubkey) + add_reward(state, shard_state, validator_index, get_base_reward(state, validator_index)) attestation_count += 1 - for i in range(len(persistent_committee), TARGET_PERSISTENT_COMMITTEE_SIZE): - assert block.aggregation_bits[i] is False or block.aggregation_bits[i] == 0 # TODO: Fix Bitvector - # Verify aggregate signature - domain = get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(block.slot)) - assert bls_verify(bls_aggregate_pubkeys(pubkeys), block.parent_root, block.signatures.attesters, domain) - # Proposer micro-rewards - add_reward(state, beacon_state, proposer_index, attestation_count * get_shard_base_reward(beacon_state) // PROPOSER_REWARD_QUOTIENT) + # Verify there are no extraneous bits set beyond the shard committee + for i in range(len(shard_committee), 2 * MAX_PERIOD_COMMITTEE_SIZE): + assert data.aggregation_bits[i] == 0b0 + # Verify attester aggregate signature + domain = get_domain(state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(data.slot)) + message = hash_tree_root(ShardCheckpoint(shard_state.slot, data.parent_root)) + assert bls_verify(bls_aggregate_pubkeys(pubkeys), message, block.signatures.attesters, domain) + # Proposer micro-reward + proposer_index = get_shard_proposer_index(state, state.shard, data.slot) + reward = attestation_count * get_base_reward(state, proposer_index) // PROPOSER_REWARD_QUOTIENT + add_reward(state, shard_state, proposer_index, reward) ``` -#### Block data fees +#### Block size fee ```python -def process_shard_block_data_fees(state: ShardState, beacon_state: BeaconState, block: ShardBlock) -> None: - base_reward = get_shard_base_reward(beacon_state) - add_fee(state, beacon_state, proposer_index, state.basefee * len(block.body) // SHARD_BLOCK_SIZE_LIMIT) - QUOTIENT = SHARD_BLOCK_SIZE_LIMIT * BASEFEE_ADJUSTMENT_FACTOR - if len(block.body) > SHARD_BLOCK_SIZE_TARGET: - state.basefee += Gwei(max(1, state.basefee * (len(block.body) - SHARD_BLOCK_SIZE_TARGET) // QUOTIENT)) - elif len(block.body) < SHARD_BLOCK_SIZE_TARGET: - state.basefee -= Gwei(max(1, state.basefee * (len(block.body) - SHARD_BLOCK_SIZE_TARGET) // QUOTIENT)) - state.basefee = Gwei(max(1, min( EFFECTIVE_BALANCE_INCREMENT // EPOCHS_PER_SHARD_PERIOD // SHARD_SLOTS_PER_EPOCH, - state.basefee, - ))) -``` - -## Object validity - -### Shard block validation: preliminary - -Accept a shard block `block` only if all of the following are correct: - -* Either `block.parent_root == Hash()` or a block `parent` such that `signing_root(parent) == block.parent_root` has already been accepted. -* `block.beacon_chain_root == get_block_root(head_beacon_state, compute_epoch_of_shard_slot(parent.slot))` where `head_beacon_state` is the current beacon chain head state. Alternatively phrased, a beacon chain block `beacon_ref` such that `signing_root(beacon_ref) == block.beacon_chain_root` has already been accepted and is part of the canonical chain, and no block with slot `beacon_ref.slot < slot <= compute_start_slot_of_epoch(compute_epoch_of_shard_slot(parent.slot))` is part of the canonical chain. -* Let `beacon_state` be the state where `beacon_ref.state_root == hash_tree_root(beacon_state)`. Let `prev_state` be the post-state of the `parent` if the `parent` exists, otherwise let it be `get_default_shard_state(beacon_state, shard)` (defined below). `block.state_root` must equal the `hash_tree_root` of the state after applying `shard_state_transition(prev_state, beacon_state, block)`. - -Note that these acceptance conditions depend on the canonical beacon chain; when the canonical beacon chain reorganizes, the eligibility of shard blocks should be re-evaluated. - -### Beacon attestations - -Let: - -- `pre_state` be the `ShardState` before processing any blocks -- `shard_blocks_or_state_roots` be the `Union[ShardBlock, Hash]` list such that `shard_blocks[slot]` is the canonical `ShardBlock` for shard `pre_state.shard` at slot `slot` if a block exists, or the post-state-root of processing state up to and including that slot if a block does not exist. -- `beacon_state` be the canonical `BeaconState` -- `valid_attestations` be the set of valid `Attestation` objects, recursively defined -- `candidate` be a candidate `Attestation` which is valid under Phase 0 rules, and for which validity is to be determined under Phase 1 rules by running `is_valid_beacon_attestation` - -```python -def is_valid_beacon_attestation(pre_state: ShardState, - shard_blocks_or_state_roots: Sequence[Union[ShardBlock, Hash]], - beacon_state: BeaconState, - valid_attestations: Set[Attestation], - candidate: Attestation) -> bool: - # Check if attestation is already determined valid - for attestation in valid_attestations: - if candidate == attestation: - return True - - # Check previous attestation - if candidate.data.previous_crosslink.epoch <= PHASE_1_FORK_EPOCH: - assert candidate.data.previous_crosslink.data_root == Hash() +def process_shard_block_size_fee(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: + # Charge proposer block size fee + proposer_index = get_shard_proposer_index(state, state.shard, block.data.slot) + block_size = SHARD_HEADER_SIZE + len(block.data.body) + add_fee(state, shard_state, proposer_index, state.block_size_price * block_size // SHARD_BLOCK_SIZE_LIMIT) + # Calculate new block size price + if block_size > SHARD_BLOCK_SIZE_TARGET: + size_delta = block_size - SHARD_BLOCK_SIZE_TARGET + price_delta = Gwei(state.block_size_price * size_delta // SHARD_BLOCK_SIZE_LIMIT // BLOCK_SIZE_PRICE_QUOTIENT) + # The maximum gas price caps the amount burnt on gas fees within a period to 32 ETH + MAX_BLOCK_SIZE_PRICE = MAX_EFFECTIVE_BALANCE // EPOCHS_PER_SHARD_PERIOD // SHARD_SLOTS_PER_EPOCH + state.block_size_price = min(MAX_BLOCK_SIZE_PRICE, state.block_size_price + price_delta) else: - previous_attestation = next( - (attestation for attestation in valid_attestations - if attestation.data.crosslink.data_root == candidate.data.previous_crosslink.data_root), - None, - ) - assert previous_attestation is not None - assert candidate.data.previous_attestation.epoch < compute_epoch_of_slot(candidate.data.slot) - - # Check crosslink data root - start_epoch = beacon_state.crosslinks[pre_state.shard].epoch - end_epoch = min(compute_epoch_of_slot(candidate.data.slot) - CROSSLINK_LOOKBACK, - start_epoch + MAX_EPOCHS_PER_CROSSLINK) - blocks = [] - for slot in range(start_epoch * SLOTS_PER_EPOCH, end_epoch * SLOTS_PER_EPOCH): - if isinstance(shard_blocks_or_state_roots[slot], ShardBlock): - blocks.append(shard_blocks_or_state_roots[slot]) - else: - blocks.append(ShardBlock( - slot=slot, - state_root=shard_blocks_or_state_roots[slot], - total_bytes=pre_state.total_bytes, - )) - assert candidate.data.crosslink.data_root == compute_crosslink_data_root(blocks) - - return True + size_delta = SHARD_BLOCK_SIZE_TARGET - block_size + price_delta = Gwei(state.block_size_price * size_delta // SHARD_BLOCK_SIZE_LIMIT // BLOCK_SIZE_PRICE_QUOTIENT) + state.block_size_price = max(MIN_BLOCK_SIZE_PRICE, state.block_size_price - price_delta) ``` ## Shard fork choice rule -The fork choice rule for any shard is LMD GHOST using the shard attestations of the persistent committee and the beacon chain attestations of the crosslink committee currently assigned to that shard, but instead of being rooted in the genesis it is rooted in the block referenced in the most recent accepted crosslink (i.e. `state.crosslinks[shard].shard_block_root`). Only blocks whose `beacon_chain_root` is the block in the main beacon chain at the specified `slot` should be considered. (If the beacon chain skips a slot, then the block at that slot is considered to be the block in the beacon chain at the highest slot lower than that slot.) +The fork choice rule for any shard is LMD GHOST using the shard attestations of the shard committee and the beacon chain attestations of the crosslink committee currently assigned to that shard, but instead of being rooted in the genesis it is rooted in the block referenced in the most recent accepted crosslink (i.e. `state.crosslinks[shard].shard_block_root`). Only blocks whose `beacon_block_root` is the block in the main beacon chain at the specified `slot` should be considered. (If the beacon chain skips a slot, then the block at that slot is considered to be the block in the beacon chain at the highest slot lower than that slot.) From 66c3c391d4714b03adce0ec82e7cc00c326f19eb Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 29 Aug 2019 16:36:13 +0100 Subject: [PATCH 03/28] Update 1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 79 ++++--------------------------- 1 file changed, 9 insertions(+), 70 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 367091c91..bd4b6d53c 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -24,15 +24,11 @@ - [`ShardBlockHeaderData`](#shardblockheaderdata) - [`ShardBlockHeader`](#shardblockheader) - [`ShardState`](#shardstate) - - [`ShardReceipt`](#shardreceipt) - [`ShardCheckpoint`](#shardcheckpoint) - [Helper functions](#helper-functions) - [Misc](#misc-1) - - [`compute_padded_data`](#compute_padded_data) - [`compute_epoch_of_shard_slot`](#compute_epoch_of_shard_slot) - [`compute_shard_period_start_epoch`](#compute_shard_period_start_epoch) - - [`compute_flat_shard_header`](#compute_flat_shard_header) - - [`compute_crosslink_data_root`](#compute_crosslink_data_root) - [Beacon state accessors](#beacon-state-accessors) - [`get_period_committee`](#get_period_committee) - [`get_shard_committee`](#get_shard_committee) @@ -167,23 +163,13 @@ class ShardState(Container): slot: ShardSlot history_accumulator: Vector[Hash, HISTORY_ACCUMULATOR_VECTOR] latest_block_header_data: ShardBlockHeader - receipt_root: Hash block_size_sum: uint64 - # Rewards and fees + # Fees and rewards block_size_price: Gwei - older_committee_rewards: List[Gwei, MAX_PERIOD_COMMITTEE_SIZE] - newer_committee_rewards: List[Gwei, MAX_PERIOD_COMMITTEE_SIZE] older_committee_fees: List[Gwei, MAX_PERIOD_COMMITTEE_SIZE] newer_committee_fees: List[Gwei, MAX_PERIOD_COMMITTEE_SIZE] -``` - -### `ShardReceipt` - -```python -class ShardReceipt(Container): - index: ValidatorIndex - rewards: Gwei - fees: Gwei + older_committee_rewards: List[Gwei, MAX_PERIOD_COMMITTEE_SIZE] + newer_committee_rewards: List[Gwei, MAX_PERIOD_COMMITTEE_SIZE] ``` ### `ShardCheckpoint` @@ -198,14 +184,6 @@ class ShardCheckpoint(Container): ### Misc -#### `compute_padded_data` - -```python -def compute_padded_data(data: bytes, length: uint64) -> bytes: - assert len(data) <= length - return data + b'\x00' * (length - len(data)) -``` - #### `compute_epoch_of_shard_slot` ```python @@ -220,40 +198,6 @@ def compute_shard_period_start_epoch(epoch: Epoch, lookback: uint64) -> Epoch: return Epoch(epoch - (epoch % EPOCHS_PER_SHARD_PERIOD) - lookback * EPOCHS_PER_SHARD_PERIOD) ``` -#### `compute_flat_shard_header` - -```python -def compute_flat_shard_header(block: ShardBlock) -> Bytes[SHARD_HEADER_SIZE]: - """ - Return a flat serialisation of the ``block`` header, preserving hash tree root. - """ - data = block.data - return ( - # Left half of the hash tree - compute_padded_data(int_to_bytes(data.slot, length=8), 32) + - data.beacon_block_root + - data.parent_root + - hash_tree_root(data.body) + - data.state_root + - compute_padded_data(int_to_bytes(data.block_size_sum, length=8), 32) + - bytes([sum([data.aggregation_bits[i + j] << j for j in range(8)]) for i in range(0, 256, 8)]) + - Bytes32() + # Padding - # Right half of the hash tree - compute_padded_data(block.signatures.attesters, 128) + - compute_padded_data(block.signatures.proposer, 128) - ) -``` - -#### `compute_crosslink_data_root` - -```python -def compute_crosslink_data_root(blocks: Sequence[ShardBlock]) -> Hash: - headers = b''.join([compute_flat_shard_header(block) for block in blocks]) - bodies = b''.join([block.data.body for block in blocks]) - MAX_SIZE = MAX_EPOCHS_PER_CROSSLINK * SHARD_SLOTS_PER_EPOCH * SHARD_BLOCK_SIZE_LIMIT - return hash_tree_root(BytesN[MAX_SIZE](compute_padded_data(headers + bodies, MAX_SIZE))) -``` - ### Beacon state accessors #### `get_period_committee` @@ -338,11 +282,10 @@ def get_genesis_shard_state(state: BeaconState, shard: Shard) -> ShardState: ```python def get_genesis_shard_block(state: BeaconState, shard: Shard) -> ShardBlock: - genesis_state = get_genesis_shard_state(state, shard) return ShardBlock(data=ShardBlockData( shard=shard, slot=ShardSlot(SHARD_GENESIS_EPOCH * SHARD_SLOTS_PER_EPOCH), - state_root=hash_tree_root(genesis_state), + state_root=hash_tree_root(get_genesis_shard_state(state, shard)), )) ``` @@ -392,15 +335,9 @@ def process_shard_slot(state: BeaconState, shard_state: ShardState) -> None: ```python def process_shard_period(shard_state: ShardState, state: BeaconState) -> None: - epoch = compute_epoch_of_shard_slot(state.slot) - older_committee = get_period_committee(state, state.shard, compute_shard_period_start_epoch(epoch, 2)) - newer_committee = get_period_committee(state, state.shard, compute_shard_period_start_epoch(epoch, 1)) - # Compute receipt root for older committee - state.receipt_root = hash_tree_root(List[ShardReceipt, MAX_PERIOD_COMMITTEE_SIZE]([ - ShardReceipt(validator_index, state.older_committee_rewards[i], state.older_committee_fees[i]) - for i, validator_index in enumerate(older_committee) - ])) # Rotate rewards and fees + epoch = compute_epoch_of_shard_slot(state.slot) + newer_committee = get_period_committee(state, state.shard, compute_shard_period_start_epoch(epoch, 1)) state.older_committee_rewards = state.newer_committee_rewards state.newer_committee_rewards = [Gwei(0) for _ in range(len(newer_committee))] state.older_committee_fees = state.newer_committee_fees @@ -443,7 +380,9 @@ def process_shard_block_header(state: BeaconState, shard_state: ShardState, bloc pubkey = state.validators[proposer_index].pubkey domain = get_domain(state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(data.slot)) assert bls_verify(pubkey, hash_tree_root(block.data), block.signatures.proposer, domain) - # Verify total body bytes count + # Verify body size is a multiple of the header size + assert len(data.body) % SHARD_HEADER_SIZE == 0 + # Verify the sum of the block sizes since genesis state.block_size_sum += SHARD_HEADER_SIZE + len(data.body) assert data.block_size_sum == state.block_size_sum ``` From b9390f09676e3457c419c97e4d3b16b4c6ec3f29 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 2 Sep 2019 14:00:14 +0800 Subject: [PATCH 04/28] import `byte` --- scripts/build_spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 83f9a2145..5b1fa6fcf 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -57,7 +57,7 @@ from eth2spec.utils.ssz.ssz_typing import ( BasicValue, Elements, BaseBytes, BaseList, SSZType, Container, List, Vector, Bytes, BytesN, Bitlist, Bitvector, Bits, Bytes1, Bytes4, Bytes8, Bytes32, Bytes48, Bytes96, - uint64, bit, boolean, + uint64, bit, boolean, byte, ) from eth2spec.utils.bls import ( bls_aggregate_pubkeys, From 834edc2dbc6ac31ed658b1b83af24342bc1b9a66 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 3 Sep 2019 13:44:50 +0100 Subject: [PATCH 05/28] Removed shard from genesis shard block --- specs/core/1_shard-data-chains.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index bd4b6d53c..63bfe45e4 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -283,7 +283,6 @@ def get_genesis_shard_state(state: BeaconState, shard: Shard) -> ShardState: ```python def get_genesis_shard_block(state: BeaconState, shard: Shard) -> ShardBlock: return ShardBlock(data=ShardBlockData( - shard=shard, slot=ShardSlot(SHARD_GENESIS_EPOCH * SHARD_SLOTS_PER_EPOCH), state_root=hash_tree_root(get_genesis_shard_state(state, shard)), )) From 3ef24870d6f30039d2a8dfefca5b8ae23320e5d0 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 3 Sep 2019 18:55:46 +0100 Subject: [PATCH 06/28] Changes as per today's discussion 1) Make `ShardBlock` and `ShardState` flat containers (as opposed to plain containers) 2) Make Gwei deltas `int64` (as opposed `uint64`) 3) Make `older_committee_deltas` a `Vector` (as opposed to `List`) 4) Apply size fee on block body only (as opposed to block header and body) 5) Enshrine minimum "extra" block body fee for proposers (reusing `PROPOSER_REWARD_QUOTIENT`) 6) Fix bugs reported by @terencechain and @hwwhww :+1: --- specs/core/1_shard-data-chains.md | 221 ++++++++++++------------------ 1 file changed, 89 insertions(+), 132 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 63bfe45e4..e052e2bcb 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -18,10 +18,7 @@ - [Rewards and penalties](#rewards-and-penalties) - [Signature domain types](#signature-domain-types) - [Containers](#containers) - - [`ShardBlockSignatures`](#shardblocksignatures) - - [`ShardBlockData`](#shardblockdata) - [`ShardBlock`](#shardblock) - - [`ShardBlockHeaderData`](#shardblockheaderdata) - [`ShardBlockHeader`](#shardblockheader) - [`ShardState`](#shardstate) - [`ShardCheckpoint`](#shardcheckpoint) @@ -34,8 +31,7 @@ - [`get_shard_committee`](#get_shard_committee) - [`get_shard_proposer_index`](#get_shard_proposer_index) - [Shard state mutators](#shard-state-mutators) - - [`add_reward`](#add_reward) - - [`add_fee`](#add_fee) + - [`process_delta`](#process_delta) - [Genesis](#genesis) - [`get_genesis_shard_state`](#get_genesis_shard_state) - [`get_genesis_shard_block`](#get_genesis_shard_block) @@ -44,7 +40,7 @@ - [Block processing](#block-processing) - [Block header](#block-header) - [Attestations](#attestations) - - [Block size fee](#block-size-fee) + - [Block body fee](#block-body-fee) - [Shard fork choice rule](#shard-fork-choice-rule) @@ -58,6 +54,7 @@ This document describes the shard transition function (data layer only) and the | Name | SSZ equivalent | Description | | - | - | - | | `ShardSlot` | `uint64` | a shard slot number | +| `GweiDelta` | `int64` | a signed Gwei delta | ## Configuration @@ -94,7 +91,7 @@ This document describes the shard transition function (data layer only) and the | Name | Value | | - | - | -| `BLOCK_SIZE_PRICE_QUOTIENT` | `2**3` (= 8) | +| `SHARD_BLOCK_SIZE_PRICE_QUOTIENT` | `2**3` (= 8) | ### Signature domain types @@ -105,71 +102,51 @@ This document describes the shard transition function (data layer only) and the ## Containers -### `ShardBlockSignatures` - -```python -class ShardBlockSignatures(Container): - attesters: BLSSignature - proposer: BLSSignature -``` - -### `ShardBlockData` - -```python -class ShardBlockData(Container): - slot: ShardSlot - beacon_block_root: Hash - parent_root: Hash - state_root: Hash - aggregation_bits: Bitvector[2 * MAX_PERIOD_COMMITTEE_SIZE] - block_size_sum: uint64 - body: List[byte, SHARD_BLOCK_SIZE_LIMIT - SHARD_HEADER_SIZE] -``` - ### `ShardBlock` ```python -class ShardBlock(Container): - data: ShardBlockData - signatures: ShardBlockSignatures -``` - -### `ShardBlockHeaderData` - -```python -class ShardBlockHeaderData(Container): +class ShardBlock(FlatContainer): + shard: Shard slot: ShardSlot beacon_block_root: Hash parent_root: Hash state_root: Hash - aggregation_bits: Bitvector[2 * MAX_PERIOD_COMMITTEE_SIZE] block_size_sum: uint64 - body_root: Hash + body: List[byte, SHARD_BLOCK_SIZE_LIMIT - SHARD_HEADER_SIZE] + attestation_bits: Bitvector[2 * MAX_PERIOD_COMMITTEE_SIZE] + attestation_signature: BLSSignature + signature: BLSSignature ``` ### `ShardBlockHeader` ```python -class ShardBlockHeader(Container): - data: ShardBlockHeaderData - signatures: ShardBlockSignatures +class ShardBlockHeader(FlatContainer): + shard: Shard + slot: ShardSlot + beacon_block_root: Hash + parent_root: Hash + state_root: Hash + block_size_sum: uint64 + body_root: List[byte, SHARD_BLOCK_SIZE_LIMIT - SHARD_HEADER_SIZE] + attestation_bits: Bitvector[2 * MAX_PERIOD_COMMITTEE_SIZE] + attestation_signature: BLSSignature + signature: BLSSignature ``` ### `ShardState` ```python -class ShardState(Container): +class ShardState(FlatContainer): shard: Shard slot: ShardSlot history_accumulator: Vector[Hash, HISTORY_ACCUMULATOR_VECTOR] - latest_block_header_data: ShardBlockHeader + latest_block_header: ShardBlockHeader block_size_sum: uint64 # Fees and rewards - block_size_price: Gwei - older_committee_fees: List[Gwei, MAX_PERIOD_COMMITTEE_SIZE] - newer_committee_fees: List[Gwei, MAX_PERIOD_COMMITTEE_SIZE] - older_committee_rewards: List[Gwei, MAX_PERIOD_COMMITTEE_SIZE] - newer_committee_rewards: List[Gwei, MAX_PERIOD_COMMITTEE_SIZE] + block_body_price: Gwei + older_committee_deltas: Vector[GweiDelta, MAX_PERIOD_COMMITTEE_SIZE] + newer_committee_deltas: Vector[GweiDelta, MAX_PERIOD_COMMITTEE_SIZE] ``` ### `ShardCheckpoint` @@ -188,7 +165,7 @@ class ShardCheckpoint(Container): ```python def compute_epoch_of_shard_slot(slot: ShardSlot) -> Epoch: - return compute_epoch_of_slot(slot // SHARD_SLOTS_PER_EPOCH) + return Epoch(slot // SHARD_SLOTS_PER_EPOCH) ``` #### `compute_shard_period_start_epoch` @@ -226,37 +203,25 @@ def get_shard_committee(state: BeaconState, shard: Shard, epoch: Epoch) -> Seque ```python def get_shard_proposer_index(state: BeaconState, shard: Shard, slot: ShardSlot) -> ValidatorIndex: epoch = get_current_epoch(state) - active_indices = [i for i in get_shard_committee(state, shard, epoch) if is_active_validator(state.validators[i], epoch)] + shard_committee = get_shard_committee(state, shard, epoch) + active_indices = [i for i in shard_committee if is_active_validator(state.validators[i], epoch)] seed = hash(get_seed(state, epoch) + int_to_bytes(slot, length=8) + int_to_bytes(shard, length=8)) compute_proposer_index(state, active_indices, seed) ``` ### Shard state mutators -#### `add_reward` +#### `process_delta` ```python -def add_reward(state: BeaconState, shard_state: ShardState, index: ValidatorIndex, delta: Gwei) -> None: +def process_delta(state: BeaconState, shard_state: ShardState, index: ValidatorIndex, delta: GweiDelta) -> None: epoch = compute_epoch_of_shard_slot(state.slot) older_committee = get_period_committee(state, shard_state.shard, compute_shard_period_start_epoch(epoch, 2)) newer_committee = get_period_committee(state, shard_state.shard, compute_shard_period_start_epoch(epoch, 1)) if index in older_committee: - shard_state.older_committee_rewards[older_committee.index(index)] += delta + shard_state.older_committee_deltas[older_committee.index(index)] += delta elif index in newer_committee: - shard_state.newer_committee_rewards[newer_committee.index(index)] += delta -``` - -#### `add_fee` - -```python -def add_fee(state: BeaconState, shard_state: ShardState, index: ValidatorIndex, delta: Gwei) -> None: - epoch = compute_epoch_of_shard_slot(state.slot) - older_committee = get_period_committee(state, shard_state.shard, compute_shard_period_start_epoch(epoch, 2)) - newer_committee = get_period_committee(state, shard_state.shard, compute_shard_period_start_epoch(epoch, 1)) - if index in older_committee: - shard_state.older_committee_fees[older_committee.index(index)] += delta - elif index in newer_committee: - shard_state.newer_committee_fees[newer_committee.index(index)] += delta + shard_state.newer_committee_deltas[newer_committee.index(index)] += delta ``` ## Genesis @@ -265,16 +230,10 @@ def add_fee(state: BeaconState, shard_state: ShardState, index: ValidatorIndex, ```python def get_genesis_shard_state(state: BeaconState, shard: Shard) -> ShardState: - older_committee = get_period_committee(state, shard, compute_shard_period_start_epoch(SHARD_GENESIS_EPOCH, 2)) - newer_committee = get_period_committee(state, shard, compute_shard_period_start_epoch(SHARD_GENESIS_EPOCH, 1)) return ShardState( shard=shard, slot=ShardSlot(SHARD_GENESIS_EPOCH * SHARD_SLOTS_PER_EPOCH), - block_size_price=MIN_BLOCK_SIZE_PRICE, - older_committee_rewards=[Gwei(0) for _ in range(len(older_committee))], - newer_committee_rewards=[Gwei(0) for _ in range(len(newer_committee))], - older_committee_fees=[Gwei(0) for _ in range(len(older_committee))], - newer_committee_fees=[Gwei(0) for _ in range(len(newer_committee))], + block_body_price=MIN_BLOCK_SIZE_PRICE, ) ``` @@ -282,10 +241,11 @@ def get_genesis_shard_state(state: BeaconState, shard: Shard) -> ShardState: ```python def get_genesis_shard_block(state: BeaconState, shard: Shard) -> ShardBlock: - return ShardBlock(data=ShardBlockData( + return ShardBlock( + shard=shard, slot=ShardSlot(SHARD_GENESIS_EPOCH * SHARD_SLOTS_PER_EPOCH), state_root=hash_tree_root(get_genesis_shard_state(state, shard)), - )) + ) ``` ## Shard state transition function @@ -296,12 +256,12 @@ def shard_state_transition(state: BeaconState, block: ShardBlock, validate_state_root: bool=False) -> ShardState: # Process slots (including those with no blocks) since block - process_shard_slots(state, shard_state, block.data.slot) + process_shard_slots(state, shard_state, block.slot) # Process block process_shard_block(state, shard_state, block) # Validate state root (`validate_state_root == True` in production) if validate_state_root: - assert block.data.state_root == hash_tree_root(shard_state) + assert block.state_root == hash_tree_root(shard_state) # Return post-state return shard_state ``` @@ -321,8 +281,8 @@ def process_shard_slots(state: BeaconState, shard_state: ShardState, slot: Shard def process_shard_slot(state: BeaconState, shard_state: ShardState) -> None: # Cache state root previous_state_root = hash_tree_root(state) - if state.latest_block_header_data.state_root == Bytes32(): - state.latest_block_header_data.state_root = previous_state_root + if state.latest_block_header.state_root == Bytes32(): + state.latest_block_header.state_root = previous_state_root # Cache state root in history accumulator depth = 0 while state.slot % 2**depth == 0 and depth < HISTORY_ACCUMULATOR_VECTOR: @@ -333,14 +293,10 @@ def process_shard_slot(state: BeaconState, shard_state: ShardState) -> None: ### Period processing ```python -def process_shard_period(shard_state: ShardState, state: BeaconState) -> None: +def process_shard_period(state: BeaconState, shard_state: ShardState) -> None: # Rotate rewards and fees - epoch = compute_epoch_of_shard_slot(state.slot) - newer_committee = get_period_committee(state, state.shard, compute_shard_period_start_epoch(epoch, 1)) - state.older_committee_rewards = state.newer_committee_rewards - state.newer_committee_rewards = [Gwei(0) for _ in range(len(newer_committee))] - state.older_committee_fees = state.newer_committee_fees - state.newer_committee_fees = [Gwei(0) for _ in range(len(newer_committee))] + state.older_committee_deltas = state.newer_committee_deltas + state.newer_committee_deltas = [GweiDelta(0) for _ in range(MAX_PERIOD_COMMITTEE_SIZE)] ``` ### Block processing @@ -349,7 +305,7 @@ def process_shard_period(shard_state: ShardState, state: BeaconState) -> None: def process_shard_block(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: process_shard_block_header(state, shard_state, block) process_shard_attestations(state, shard_state, block) - process_shard_block_size_fee(state, shard_state, block) + process_shard_block_body_fee(state, shard_state, block) ``` #### Block header @@ -357,80 +313,81 @@ def process_shard_block(state: BeaconState, shard_state: ShardState, block: Shar ```python def process_shard_block_header(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: # Verify that the slots match - data = block.data - assert data.slot == state.slot + assert block.slot == state.slot # Verify that the beacon chain root matches - parent_epoch = compute_epoch_of_shard_slot(state.latest_block_header_data.slot) - assert data.beacon_block_root == get_block_root(state, parent_epoch) + parent_epoch = compute_epoch_of_shard_slot(state.latest_block_header.slot) + assert block.beacon_block_root == get_block_root(state, parent_epoch) # Verify that the parent matches - assert data.parent_root == hash_tree_root(state.latest_block_header_data) + assert block.parent_root == hash_tree_root(state.latest_block_header) # Save current block as the new latest block - state.latest_block_header_data = ShardBlockHeaderData( - slot=data.slot, - beacon_block_root=data.beacon_block_root, - parent_root=data.parent_root, + state.latest_block_header = ShardBlockHeader( + shard=block.shard, + slot=block.slot, + beacon_block_root=block.beacon_block_root, + parent_root=block.parent_root, # `state_root` is zeroed and overwritten in the next `process_shard_slot` call - aggregation_bits=data.aggregation_bits, - block_size_sum=data.block_size_sum, - body_root=hash_tree_root(data.body), + block_size_sum=block.block_size_sum, + body_root=hash_tree_root(block.body), + attestation_bits=block.attestation_bits, + attestation_signature=block.attestation_signature, + # `signature` is zeroed ) # Verify proposer signature - proposer_index = get_shard_proposer_index(state, state.shard, data.slot) + proposer_index = get_shard_proposer_index(state, state.shard, block.slot) pubkey = state.validators[proposer_index].pubkey - domain = get_domain(state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(data.slot)) - assert bls_verify(pubkey, hash_tree_root(block.data), block.signatures.proposer, domain) + domain = get_domain(state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.slot)) + assert bls_verify(pubkey, hash_tree_root(block.block), block.proposer, domain) # Verify body size is a multiple of the header size - assert len(data.body) % SHARD_HEADER_SIZE == 0 + assert len(block.body) % SHARD_HEADER_SIZE == 0 # Verify the sum of the block sizes since genesis - state.block_size_sum += SHARD_HEADER_SIZE + len(data.body) - assert data.block_size_sum == state.block_size_sum + state.block_size_sum += SHARD_HEADER_SIZE + len(block.body) + assert block.block_size_sum == state.block_size_sum ``` #### Attestations ```python def process_shard_attestations(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: - data = block.data pubkeys = [] attestation_count = 0 - shard_committee = get_shard_committee(state, state.shard, data.slot) + shard_committee = get_shard_committee(state, state.shard, block.slot) for i, validator_index in enumerate(shard_committee): - if data.aggregation_bits[i]: + if block.attestation_bits[i]: pubkeys.append(state.validators[validator_index].pubkey) - add_reward(state, shard_state, validator_index, get_base_reward(state, validator_index)) + process_delta(state, shard_state, validator_index, get_base_reward(state, validator_index)) attestation_count += 1 # Verify there are no extraneous bits set beyond the shard committee for i in range(len(shard_committee), 2 * MAX_PERIOD_COMMITTEE_SIZE): - assert data.aggregation_bits[i] == 0b0 + assert block.attestation_bits[i] == 0b0 # Verify attester aggregate signature - domain = get_domain(state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(data.slot)) - message = hash_tree_root(ShardCheckpoint(shard_state.slot, data.parent_root)) - assert bls_verify(bls_aggregate_pubkeys(pubkeys), message, block.signatures.attesters, domain) + domain = get_domain(state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(block.slot)) + message = hash_tree_root(ShardCheckpoint(shard_state.slot, block.parent_root)) + assert bls_verify(bls_aggregate_pubkeys(pubkeys), message, block.attestation_signature, domain) # Proposer micro-reward - proposer_index = get_shard_proposer_index(state, state.shard, data.slot) + proposer_index = get_shard_proposer_index(state, state.shard, block.slot) reward = attestation_count * get_base_reward(state, proposer_index) // PROPOSER_REWARD_QUOTIENT - add_reward(state, shard_state, proposer_index, reward) + process_delta(state, shard_state, proposer_index, reward) ``` -#### Block size fee +#### Block body fee ```python -def process_shard_block_size_fee(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: - # Charge proposer block size fee - proposer_index = get_shard_proposer_index(state, state.shard, block.data.slot) - block_size = SHARD_HEADER_SIZE + len(block.data.body) - add_fee(state, shard_state, proposer_index, state.block_size_price * block_size // SHARD_BLOCK_SIZE_LIMIT) - # Calculate new block size price - if block_size > SHARD_BLOCK_SIZE_TARGET: - size_delta = block_size - SHARD_BLOCK_SIZE_TARGET - price_delta = Gwei(state.block_size_price * size_delta // SHARD_BLOCK_SIZE_LIMIT // BLOCK_SIZE_PRICE_QUOTIENT) - # The maximum gas price caps the amount burnt on gas fees within a period to 32 ETH +def process_shard_block_body_fee(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: + # Apply proposer block body fee + proposer_index = get_shard_proposer_index(state, state.shard, block.slot) + block_body_fee = state.block_body_price * len(block.body) // SHARD_BLOCK_SIZE_LIMIT + process_delta(state, shard_state, proposer_index, -block_body_fee) # Burn + process_delta(state, shard_state, proposer_index, block_body_fee // PROPOSER_REWARD_QUOTIENT) # Reward + # Calculate new block body price + block_size = SHARD_HEADER_SIZE + len(block.body) + QUOTIENT = SHARD_BLOCK_SIZE_LIMIT * SHARD_BLOCK_SIZE_PRICE_QUOTIENT + price_delta = GweiDelta(state.block_body_price * (block_size - SHARD_BLOCK_SIZE_TARGET) // QUOTIENT) + if price_delta > 0: + # The maximum gas price caps the amount burnt on gas fees within a period MAX_BLOCK_SIZE_PRICE = MAX_EFFECTIVE_BALANCE // EPOCHS_PER_SHARD_PERIOD // SHARD_SLOTS_PER_EPOCH - state.block_size_price = min(MAX_BLOCK_SIZE_PRICE, state.block_size_price + price_delta) + state.block_body_price = Gwei(min(MAX_BLOCK_SIZE_PRICE, state.block_body_price + price_delta)) else: - size_delta = SHARD_BLOCK_SIZE_TARGET - block_size - price_delta = Gwei(state.block_size_price * size_delta // SHARD_BLOCK_SIZE_LIMIT // BLOCK_SIZE_PRICE_QUOTIENT) - state.block_size_price = max(MIN_BLOCK_SIZE_PRICE, state.block_size_price - price_delta) + state.block_body_price = Gwei(max(MIN_BLOCK_SIZE_PRICE, state.block_body_price + price_delta)) ``` ## Shard fork choice rule From c4297ae330231afbe9fdc883118ee7bbc44fcfce Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 3 Sep 2019 19:17:19 +0100 Subject: [PATCH 07/28] Update 1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index e052e2bcb..5601da936 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -62,11 +62,11 @@ This document describes the shard transition function (data layer only) and the | Name | Value | | - | - | -| `MIN_BLOCK_SIZE_PRICE` | `2**0` (= 1) | +| `MIN_BLOCK_BODY_PRICE` | `2**0` (= 1) | | `MAX_PERIOD_COMMITTEE_SIZE` | `2**7` (= 128) | | `SHARD_HEADER_SIZE` | `2**9` (= 512) | | `SHARD_BLOCK_SIZE_TARGET` | `2**14` (= 16,384) | -| `SHARD_BLOCK_SIZE_LIMIT` | `2**16` (= 65,536) | +| `MAX_SHARD_BLOCK_SIZE` | `2**16` (= 65,536) | ### Initial values @@ -91,7 +91,7 @@ This document describes the shard transition function (data layer only) and the | Name | Value | | - | - | -| `SHARD_BLOCK_SIZE_PRICE_QUOTIENT` | `2**3` (= 8) | +| `BLOCK_BODY_PRICE_QUOTIENT` | `2**3` (= 8) | ### Signature domain types @@ -112,7 +112,7 @@ class ShardBlock(FlatContainer): parent_root: Hash state_root: Hash block_size_sum: uint64 - body: List[byte, SHARD_BLOCK_SIZE_LIMIT - SHARD_HEADER_SIZE] + body: List[byte, MAX_SHARD_BLOCK_SIZE - SHARD_HEADER_SIZE] attestation_bits: Bitvector[2 * MAX_PERIOD_COMMITTEE_SIZE] attestation_signature: BLSSignature signature: BLSSignature @@ -128,7 +128,7 @@ class ShardBlockHeader(FlatContainer): parent_root: Hash state_root: Hash block_size_sum: uint64 - body_root: List[byte, SHARD_BLOCK_SIZE_LIMIT - SHARD_HEADER_SIZE] + body_root: List[byte, MAX_SHARD_BLOCK_SIZE - SHARD_HEADER_SIZE] attestation_bits: Bitvector[2 * MAX_PERIOD_COMMITTEE_SIZE] attestation_signature: BLSSignature signature: BLSSignature @@ -233,7 +233,7 @@ def get_genesis_shard_state(state: BeaconState, shard: Shard) -> ShardState: return ShardState( shard=shard, slot=ShardSlot(SHARD_GENESIS_EPOCH * SHARD_SLOTS_PER_EPOCH), - block_body_price=MIN_BLOCK_SIZE_PRICE, + block_body_price=MIN_BLOCK_BODY_PRICE, ) ``` @@ -375,19 +375,19 @@ def process_shard_attestations(state: BeaconState, shard_state: ShardState, bloc def process_shard_block_body_fee(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: # Apply proposer block body fee proposer_index = get_shard_proposer_index(state, state.shard, block.slot) - block_body_fee = state.block_body_price * len(block.body) // SHARD_BLOCK_SIZE_LIMIT + block_body_fee = state.block_body_price * len(block.body) // MAX_SHARD_BLOCK_SIZE process_delta(state, shard_state, proposer_index, -block_body_fee) # Burn process_delta(state, shard_state, proposer_index, block_body_fee // PROPOSER_REWARD_QUOTIENT) # Reward # Calculate new block body price block_size = SHARD_HEADER_SIZE + len(block.body) - QUOTIENT = SHARD_BLOCK_SIZE_LIMIT * SHARD_BLOCK_SIZE_PRICE_QUOTIENT + QUOTIENT = MAX_SHARD_BLOCK_SIZE * BLOCK_BODY_PRICE_QUOTIENT price_delta = GweiDelta(state.block_body_price * (block_size - SHARD_BLOCK_SIZE_TARGET) // QUOTIENT) if price_delta > 0: - # The maximum gas price caps the amount burnt on gas fees within a period - MAX_BLOCK_SIZE_PRICE = MAX_EFFECTIVE_BALANCE // EPOCHS_PER_SHARD_PERIOD // SHARD_SLOTS_PER_EPOCH - state.block_body_price = Gwei(min(MAX_BLOCK_SIZE_PRICE, state.block_body_price + price_delta)) + # The maximum block body price caps the amount burnt on fees within a period + MAX_BLOCK_BODY_PRICE = MAX_EFFECTIVE_BALANCE // EPOCHS_PER_SHARD_PERIOD // SHARD_SLOTS_PER_EPOCH + state.block_body_price = Gwei(min(MAX_BLOCK_BODY_PRICE, state.block_body_price + price_delta)) else: - state.block_body_price = Gwei(max(MIN_BLOCK_SIZE_PRICE, state.block_body_price + price_delta)) + state.block_body_price = Gwei(max(MIN_BLOCK_BODY_PRICE, state.block_body_price + price_delta)) ``` ## Shard fork choice rule From a0bbc940ceb1f014f6d02143c4f9e4631e0ab760 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 3 Sep 2019 19:27:30 +0100 Subject: [PATCH 08/28] Update 1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 46 ++++++++++++++++--------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 5601da936..cc1e9b0a2 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -40,7 +40,7 @@ - [Block processing](#block-processing) - [Block header](#block-header) - [Attestations](#attestations) - - [Block body fee](#block-body-fee) + - [Block body](#block-body) - [Shard fork choice rule](#shard-fork-choice-rule) @@ -113,8 +113,8 @@ class ShardBlock(FlatContainer): state_root: Hash block_size_sum: uint64 body: List[byte, MAX_SHARD_BLOCK_SIZE - SHARD_HEADER_SIZE] - attestation_bits: Bitvector[2 * MAX_PERIOD_COMMITTEE_SIZE] - attestation_signature: BLSSignature + aggregation_bits: Bitvector[2 * MAX_PERIOD_COMMITTEE_SIZE] + attestations: BLSSignature signature: BLSSignature ``` @@ -128,9 +128,9 @@ class ShardBlockHeader(FlatContainer): parent_root: Hash state_root: Hash block_size_sum: uint64 - body_root: List[byte, MAX_SHARD_BLOCK_SIZE - SHARD_HEADER_SIZE] - attestation_bits: Bitvector[2 * MAX_PERIOD_COMMITTEE_SIZE] - attestation_signature: BLSSignature + body_root: Hash + aggregation_bits: Bitvector[2 * MAX_PERIOD_COMMITTEE_SIZE] + attestations: BLSSignature signature: BLSSignature ``` @@ -305,19 +305,21 @@ def process_shard_period(state: BeaconState, shard_state: ShardState) -> None: def process_shard_block(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: process_shard_block_header(state, shard_state, block) process_shard_attestations(state, shard_state, block) - process_shard_block_body_fee(state, shard_state, block) + process_shard_block_body(state, shard_state, block) ``` #### Block header ```python def process_shard_block_header(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: - # Verify that the slots match + # Verify the shard number + assert block.shard == state.shard + # Verify the slot number assert block.slot == state.slot - # Verify that the beacon chain root matches + # Verify the beacon chain root parent_epoch = compute_epoch_of_shard_slot(state.latest_block_header.slot) assert block.beacon_block_root == get_block_root(state, parent_epoch) - # Verify that the parent matches + # Verify the parent root assert block.parent_root == hash_tree_root(state.latest_block_header) # Save current block as the new latest block state.latest_block_header = ShardBlockHeader( @@ -328,20 +330,18 @@ def process_shard_block_header(state: BeaconState, shard_state: ShardState, bloc # `state_root` is zeroed and overwritten in the next `process_shard_slot` call block_size_sum=block.block_size_sum, body_root=hash_tree_root(block.body), - attestation_bits=block.attestation_bits, - attestation_signature=block.attestation_signature, + aggregation_bits=block.aggregation_bits, + attestations=block.attestations, # `signature` is zeroed ) + # Verify the sum of the block sizes since genesis + state.block_size_sum += SHARD_HEADER_SIZE + len(block.body) + assert block.block_size_sum == state.block_size_sum # Verify proposer signature proposer_index = get_shard_proposer_index(state, state.shard, block.slot) pubkey = state.validators[proposer_index].pubkey domain = get_domain(state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.slot)) assert bls_verify(pubkey, hash_tree_root(block.block), block.proposer, domain) - # Verify body size is a multiple of the header size - assert len(block.body) % SHARD_HEADER_SIZE == 0 - # Verify the sum of the block sizes since genesis - state.block_size_sum += SHARD_HEADER_SIZE + len(block.body) - assert block.block_size_sum == state.block_size_sum ``` #### Attestations @@ -352,27 +352,29 @@ def process_shard_attestations(state: BeaconState, shard_state: ShardState, bloc attestation_count = 0 shard_committee = get_shard_committee(state, state.shard, block.slot) for i, validator_index in enumerate(shard_committee): - if block.attestation_bits[i]: + if block.aggregation_bits[i]: pubkeys.append(state.validators[validator_index].pubkey) process_delta(state, shard_state, validator_index, get_base_reward(state, validator_index)) attestation_count += 1 # Verify there are no extraneous bits set beyond the shard committee for i in range(len(shard_committee), 2 * MAX_PERIOD_COMMITTEE_SIZE): - assert block.attestation_bits[i] == 0b0 + assert block.aggregation_bits[i] == 0b0 # Verify attester aggregate signature domain = get_domain(state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(block.slot)) message = hash_tree_root(ShardCheckpoint(shard_state.slot, block.parent_root)) - assert bls_verify(bls_aggregate_pubkeys(pubkeys), message, block.attestation_signature, domain) + assert bls_verify(bls_aggregate_pubkeys(pubkeys), message, block.attestations, domain) # Proposer micro-reward proposer_index = get_shard_proposer_index(state, state.shard, block.slot) reward = attestation_count * get_base_reward(state, proposer_index) // PROPOSER_REWARD_QUOTIENT process_delta(state, shard_state, proposer_index, reward) ``` -#### Block body fee +#### Block body ```python -def process_shard_block_body_fee(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: +def process_shard_block_body(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: + # Verify block body size is a multiple of the header size + assert len(block.body) % SHARD_HEADER_SIZE == 0 # Apply proposer block body fee proposer_index = get_shard_proposer_index(state, state.shard, block.slot) block_body_fee = state.block_body_price * len(block.body) // MAX_SHARD_BLOCK_SIZE From 225c740107698fe2b89ec7fb0a6a4d9daeb04284 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 3 Sep 2019 21:56:12 +0100 Subject: [PATCH 09/28] Apply suggestions from code review Co-Authored-By: terence tsao --- specs/core/1_shard-data-chains.md | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index cc1e9b0a2..7db452718 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -229,7 +229,7 @@ def process_delta(state: BeaconState, shard_state: ShardState, index: ValidatorI ### `get_genesis_shard_state` ```python -def get_genesis_shard_state(state: BeaconState, shard: Shard) -> ShardState: +def get_genesis_shard_state(shard: Shard) -> ShardState: return ShardState( shard=shard, slot=ShardSlot(SHARD_GENESIS_EPOCH * SHARD_SLOTS_PER_EPOCH), @@ -295,8 +295,8 @@ def process_shard_slot(state: BeaconState, shard_state: ShardState) -> None: ```python def process_shard_period(state: BeaconState, shard_state: ShardState) -> None: # Rotate rewards and fees - state.older_committee_deltas = state.newer_committee_deltas - state.newer_committee_deltas = [GweiDelta(0) for _ in range(MAX_PERIOD_COMMITTEE_SIZE)] + shard_state.older_committee_deltas = shard_state.newer_committee_deltas + shard_state.newer_committee_deltas = [GweiDelta(0) for _ in range(MAX_PERIOD_COMMITTEE_SIZE)] ``` ### Block processing @@ -313,16 +313,16 @@ def process_shard_block(state: BeaconState, shard_state: ShardState, block: Shar ```python def process_shard_block_header(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: # Verify the shard number - assert block.shard == state.shard + assert block.shard == shard_state.shard # Verify the slot number - assert block.slot == state.slot + assert block.slot == shard_state.slot # Verify the beacon chain root - parent_epoch = compute_epoch_of_shard_slot(state.latest_block_header.slot) + parent_epoch = compute_epoch_of_shard_slot(shard_state.latest_block_header.slot) assert block.beacon_block_root == get_block_root(state, parent_epoch) # Verify the parent root - assert block.parent_root == hash_tree_root(state.latest_block_header) + assert block.parent_root == hash_tree_root(shard_state.latest_block_header) # Save current block as the new latest block - state.latest_block_header = ShardBlockHeader( + shard_state.latest_block_header = ShardBlockHeader( shard=block.shard, slot=block.slot, beacon_block_root=block.beacon_block_root, @@ -335,10 +335,10 @@ def process_shard_block_header(state: BeaconState, shard_state: ShardState, bloc # `signature` is zeroed ) # Verify the sum of the block sizes since genesis - state.block_size_sum += SHARD_HEADER_SIZE + len(block.body) + shard_state.block_size_sum += SHARD_HEADER_SIZE + len(block.body) assert block.block_size_sum == state.block_size_sum # Verify proposer signature - proposer_index = get_shard_proposer_index(state, state.shard, block.slot) + proposer_index = get_shard_proposer_index(state, shard_state.shard, block.slot) pubkey = state.validators[proposer_index].pubkey domain = get_domain(state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.slot)) assert bls_verify(pubkey, hash_tree_root(block.block), block.proposer, domain) @@ -350,7 +350,7 @@ def process_shard_block_header(state: BeaconState, shard_state: ShardState, bloc def process_shard_attestations(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: pubkeys = [] attestation_count = 0 - shard_committee = get_shard_committee(state, state.shard, block.slot) + shard_committee = get_shard_committee(state, shard_state.shard, block.slot) for i, validator_index in enumerate(shard_committee): if block.aggregation_bits[i]: pubkeys.append(state.validators[validator_index].pubkey) @@ -364,7 +364,7 @@ def process_shard_attestations(state: BeaconState, shard_state: ShardState, bloc message = hash_tree_root(ShardCheckpoint(shard_state.slot, block.parent_root)) assert bls_verify(bls_aggregate_pubkeys(pubkeys), message, block.attestations, domain) # Proposer micro-reward - proposer_index = get_shard_proposer_index(state, state.shard, block.slot) + proposer_index = get_shard_proposer_index(state, shard_state.shard, block.slot) reward = attestation_count * get_base_reward(state, proposer_index) // PROPOSER_REWARD_QUOTIENT process_delta(state, shard_state, proposer_index, reward) ``` @@ -376,20 +376,20 @@ def process_shard_block_body(state: BeaconState, shard_state: ShardState, block: # Verify block body size is a multiple of the header size assert len(block.body) % SHARD_HEADER_SIZE == 0 # Apply proposer block body fee - proposer_index = get_shard_proposer_index(state, state.shard, block.slot) + proposer_index = get_shard_proposer_index(state, shard_state.shard, block.slot) block_body_fee = state.block_body_price * len(block.body) // MAX_SHARD_BLOCK_SIZE process_delta(state, shard_state, proposer_index, -block_body_fee) # Burn process_delta(state, shard_state, proposer_index, block_body_fee // PROPOSER_REWARD_QUOTIENT) # Reward # Calculate new block body price block_size = SHARD_HEADER_SIZE + len(block.body) QUOTIENT = MAX_SHARD_BLOCK_SIZE * BLOCK_BODY_PRICE_QUOTIENT - price_delta = GweiDelta(state.block_body_price * (block_size - SHARD_BLOCK_SIZE_TARGET) // QUOTIENT) + price_delta = GweiDelta(shard_state.block_body_price * (block_size - SHARD_BLOCK_SIZE_TARGET) // QUOTIENT) if price_delta > 0: # The maximum block body price caps the amount burnt on fees within a period MAX_BLOCK_BODY_PRICE = MAX_EFFECTIVE_BALANCE // EPOCHS_PER_SHARD_PERIOD // SHARD_SLOTS_PER_EPOCH - state.block_body_price = Gwei(min(MAX_BLOCK_BODY_PRICE, state.block_body_price + price_delta)) + shard_state.block_body_price = Gwei(min(MAX_BLOCK_BODY_PRICE, shard_state.block_body_price + price_delta)) else: - state.block_body_price = Gwei(max(MIN_BLOCK_BODY_PRICE, state.block_body_price + price_delta)) + shard_state.block_body_price = Gwei(max(MIN_BLOCK_BODY_PRICE, shard_state.block_body_price + price_delta)) ``` ## Shard fork choice rule From 40dc0622793c904021cc9de70fb5ad41b92c1459 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 3 Sep 2019 22:04:59 +0100 Subject: [PATCH 10/28] Update 1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 110 +++++++++++++++--------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 7db452718..94d941f38 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -180,18 +180,18 @@ def compute_shard_period_start_epoch(epoch: Epoch, lookback: uint64) -> Epoch: #### `get_period_committee` ```python -def get_period_committee(state: BeaconState, shard: Shard, epoch: Epoch) -> Sequence[ValidatorIndex]: - active_validator_indices = get_active_validator_indices(state, epoch) - seed = get_seed(state, epoch) +def get_period_committee(beacon_state: BeaconState, shard: Shard, epoch: Epoch) -> Sequence[ValidatorIndex]: + active_validator_indices = get_active_validator_indices(beacon_state, epoch) + seed = get_seed(beacon_state, epoch) return compute_committee(active_validator_indices, seed, shard, SHARD_COUNT)[:MAX_PERIOD_COMMITTEE_SIZE] ``` #### `get_shard_committee` ```python -def get_shard_committee(state: BeaconState, shard: Shard, epoch: Epoch) -> Sequence[ValidatorIndex]: - older_committee = get_period_committee(state, shard, compute_shard_period_start_epoch(epoch, 2)) - newer_committee = get_period_committee(state, shard, compute_shard_period_start_epoch(epoch, 1)) +def get_shard_committee(beacon_state: BeaconState, shard: Shard, epoch: Epoch) -> Sequence[ValidatorIndex]: + older_committee = get_period_committee(beacon_state, shard, compute_shard_period_start_epoch(epoch, 2)) + newer_committee = get_period_committee(beacon_state, shard, compute_shard_period_start_epoch(epoch, 1)) # Every epoch cycle out validators from the older committee and cycle in validators from the newer committee older_subcommittee = [i for i in older_committee if i % EPOCHS_PER_SHARD_PERIOD > epoch % EPOCHS_PER_SHARD_PERIOD] newer_subcommittee = [i for i in newer_committee if i % EPOCHS_PER_SHARD_PERIOD <= epoch % EPOCHS_PER_SHARD_PERIOD] @@ -201,12 +201,12 @@ def get_shard_committee(state: BeaconState, shard: Shard, epoch: Epoch) -> Seque #### `get_shard_proposer_index` ```python -def get_shard_proposer_index(state: BeaconState, shard: Shard, slot: ShardSlot) -> ValidatorIndex: - epoch = get_current_epoch(state) - shard_committee = get_shard_committee(state, shard, epoch) - active_indices = [i for i in shard_committee if is_active_validator(state.validators[i], epoch)] - seed = hash(get_seed(state, epoch) + int_to_bytes(slot, length=8) + int_to_bytes(shard, length=8)) - compute_proposer_index(state, active_indices, seed) +def get_shard_proposer_index(beacon_state: BeaconState, shard: Shard, slot: ShardSlot) -> ValidatorIndex: + epoch = get_current_epoch(beacon_state) + shard_committee = get_shard_committee(beacon_state, shard, epoch) + active_indices = [i for i in shard_committee if is_active_validator(beacon_state.validators[i], epoch)] + seed = hash(get_seed(beacon_state, epoch) + int_to_bytes(slot, length=8) + int_to_bytes(shard, length=8)) + compute_proposer_index(beacon_state, active_indices, seed) ``` ### Shard state mutators @@ -214,10 +214,10 @@ def get_shard_proposer_index(state: BeaconState, shard: Shard, slot: ShardSlot) #### `process_delta` ```python -def process_delta(state: BeaconState, shard_state: ShardState, index: ValidatorIndex, delta: GweiDelta) -> None: - epoch = compute_epoch_of_shard_slot(state.slot) - older_committee = get_period_committee(state, shard_state.shard, compute_shard_period_start_epoch(epoch, 2)) - newer_committee = get_period_committee(state, shard_state.shard, compute_shard_period_start_epoch(epoch, 1)) +def process_delta(beacon_state: BeaconState, shard_state: ShardState, index: ValidatorIndex, delta: GweiDelta) -> None: + epoch = compute_epoch_of_shard_slot(beacon_state.slot) + older_committee = get_period_committee(beacon_state, shard_state.shard, compute_shard_period_start_epoch(epoch, 2)) + newer_committee = get_period_committee(beacon_state, shard_state.shard, compute_shard_period_start_epoch(epoch, 1)) if index in older_committee: shard_state.older_committee_deltas[older_committee.index(index)] += delta elif index in newer_committee: @@ -240,25 +240,25 @@ def get_genesis_shard_state(shard: Shard) -> ShardState: ### `get_genesis_shard_block` ```python -def get_genesis_shard_block(state: BeaconState, shard: Shard) -> ShardBlock: +def get_genesis_shard_block(beacon_state: BeaconState, shard: Shard) -> ShardBlock: return ShardBlock( shard=shard, slot=ShardSlot(SHARD_GENESIS_EPOCH * SHARD_SLOTS_PER_EPOCH), - state_root=hash_tree_root(get_genesis_shard_state(state, shard)), + state_root=hash_tree_root(get_genesis_shard_state(beacon_state, shard)), ) ``` ## Shard state transition function ```python -def shard_state_transition(state: BeaconState, +def shard_state_transition(beacon_state: BeaconState, shard_state: ShardState, block: ShardBlock, validate_state_root: bool=False) -> ShardState: # Process slots (including those with no blocks) since block - process_shard_slots(state, shard_state, block.slot) + process_shard_slots(beacon_state, shard_state, block.slot) # Process block - process_shard_block(state, shard_state, block) + process_shard_block(beacon_state, shard_state, block) # Validate state root (`validate_state_root == True` in production) if validate_state_root: assert block.state_root == hash_tree_root(shard_state) @@ -267,33 +267,33 @@ def shard_state_transition(state: BeaconState, ``` ```python -def process_shard_slots(state: BeaconState, shard_state: ShardState, slot: ShardSlot) -> None: +def process_shard_slots(beacon_state: BeaconState, shard_state: ShardState, slot: ShardSlot) -> None: assert shard_state.slot <= slot while shard_state.slot < slot: - process_shard_slot(state, shard_state) + process_shard_slot(shard_state) # Process period on the start slot of the next period if (shard_state.slot + 1) % (SHARD_SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD) == 0: - process_shard_period(state, shard_state) + process_shard_period(shard_state) shard_state.slot += ShardSlot(1) ``` ```python -def process_shard_slot(state: BeaconState, shard_state: ShardState) -> None: +def process_shard_slot(shard_state: ShardState) -> None: # Cache state root - previous_state_root = hash_tree_root(state) - if state.latest_block_header.state_root == Bytes32(): - state.latest_block_header.state_root = previous_state_root + previous_state_root = hash_tree_root(shard_state) + if shard_state.latest_block_header.state_root == Bytes32(): + shard_state.latest_block_header.state_root = previous_state_root # Cache state root in history accumulator depth = 0 - while state.slot % 2**depth == 0 and depth < HISTORY_ACCUMULATOR_VECTOR: - state.history_accumulator[depth] = previous_state_root + while shard_state.slot % 2**depth == 0 and depth < HISTORY_ACCUMULATOR_VECTOR: + shard_state.history_accumulator[depth] = previous_state_root depth += 1 ``` ### Period processing ```python -def process_shard_period(state: BeaconState, shard_state: ShardState) -> None: +def process_shard_period(shard_state: ShardState) -> None: # Rotate rewards and fees shard_state.older_committee_deltas = shard_state.newer_committee_deltas shard_state.newer_committee_deltas = [GweiDelta(0) for _ in range(MAX_PERIOD_COMMITTEE_SIZE)] @@ -302,23 +302,23 @@ def process_shard_period(state: BeaconState, shard_state: ShardState) -> None: ### Block processing ```python -def process_shard_block(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: - process_shard_block_header(state, shard_state, block) - process_shard_attestations(state, shard_state, block) - process_shard_block_body(state, shard_state, block) +def process_shard_block(beacon_state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: + process_shard_block_header(beacon_state, shard_state, block) + process_shard_attestations(beacon_state, shard_state, block) + process_shard_block_body(beacon_state, shard_state, block) ``` #### Block header ```python -def process_shard_block_header(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: +def process_shard_block_header(beacon_state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: # Verify the shard number assert block.shard == shard_state.shard # Verify the slot number assert block.slot == shard_state.slot # Verify the beacon chain root parent_epoch = compute_epoch_of_shard_slot(shard_state.latest_block_header.slot) - assert block.beacon_block_root == get_block_root(state, parent_epoch) + assert block.beacon_block_root == get_block_root(beacon_state, parent_epoch) # Verify the parent root assert block.parent_root == hash_tree_root(shard_state.latest_block_header) # Save current block as the new latest block @@ -336,50 +336,50 @@ def process_shard_block_header(state: BeaconState, shard_state: ShardState, bloc ) # Verify the sum of the block sizes since genesis shard_state.block_size_sum += SHARD_HEADER_SIZE + len(block.body) - assert block.block_size_sum == state.block_size_sum + assert block.block_size_sum == shard_state.block_size_sum # Verify proposer signature - proposer_index = get_shard_proposer_index(state, shard_state.shard, block.slot) - pubkey = state.validators[proposer_index].pubkey - domain = get_domain(state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.slot)) + proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) + pubkey = beacon_state.validators[proposer_index].pubkey + domain = get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.slot)) assert bls_verify(pubkey, hash_tree_root(block.block), block.proposer, domain) ``` #### Attestations ```python -def process_shard_attestations(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: +def process_shard_attestations(beacon_state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: pubkeys = [] attestation_count = 0 - shard_committee = get_shard_committee(state, shard_state.shard, block.slot) + shard_committee = get_shard_committee(beacon_state, shard_state.shard, block.slot) for i, validator_index in enumerate(shard_committee): if block.aggregation_bits[i]: - pubkeys.append(state.validators[validator_index].pubkey) - process_delta(state, shard_state, validator_index, get_base_reward(state, validator_index)) + pubkeys.append(beacon_state.validators[validator_index].pubkey) + process_delta(beacon_state, shard_state, validator_index, get_base_reward(beacon_state, validator_index)) attestation_count += 1 # Verify there are no extraneous bits set beyond the shard committee for i in range(len(shard_committee), 2 * MAX_PERIOD_COMMITTEE_SIZE): assert block.aggregation_bits[i] == 0b0 # Verify attester aggregate signature - domain = get_domain(state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(block.slot)) + domain = get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(block.slot)) message = hash_tree_root(ShardCheckpoint(shard_state.slot, block.parent_root)) assert bls_verify(bls_aggregate_pubkeys(pubkeys), message, block.attestations, domain) # Proposer micro-reward - proposer_index = get_shard_proposer_index(state, shard_state.shard, block.slot) - reward = attestation_count * get_base_reward(state, proposer_index) // PROPOSER_REWARD_QUOTIENT - process_delta(state, shard_state, proposer_index, reward) + proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) + reward = attestation_count * get_base_reward(beacon_state, proposer_index) // PROPOSER_REWARD_QUOTIENT + process_delta(beacon_state, shard_state, proposer_index, reward) ``` #### Block body ```python -def process_shard_block_body(state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: +def process_shard_block_body(beacon_state: BeaconState, shard_state: ShardState, block: ShardBlock) -> None: # Verify block body size is a multiple of the header size assert len(block.body) % SHARD_HEADER_SIZE == 0 # Apply proposer block body fee - proposer_index = get_shard_proposer_index(state, shard_state.shard, block.slot) - block_body_fee = state.block_body_price * len(block.body) // MAX_SHARD_BLOCK_SIZE - process_delta(state, shard_state, proposer_index, -block_body_fee) # Burn - process_delta(state, shard_state, proposer_index, block_body_fee // PROPOSER_REWARD_QUOTIENT) # Reward + proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) + block_body_fee = shard_state.block_body_price * len(block.body) // MAX_SHARD_BLOCK_SIZE + process_delta(beacon_state, shard_state, proposer_index, -block_body_fee) # Burn + process_delta(beacon_state, shard_state, proposer_index, block_body_fee // PROPOSER_REWARD_QUOTIENT) # Reward # Calculate new block body price block_size = SHARD_HEADER_SIZE + len(block.body) QUOTIENT = MAX_SHARD_BLOCK_SIZE * BLOCK_BODY_PRICE_QUOTIENT @@ -394,4 +394,4 @@ def process_shard_block_body(state: BeaconState, shard_state: ShardState, block: ## Shard fork choice rule -The fork choice rule for any shard is LMD GHOST using the shard attestations of the shard committee and the beacon chain attestations of the crosslink committee currently assigned to that shard, but instead of being rooted in the genesis it is rooted in the block referenced in the most recent accepted crosslink (i.e. `state.crosslinks[shard].shard_block_root`). Only blocks whose `beacon_block_root` is the block in the main beacon chain at the specified `slot` should be considered. (If the beacon chain skips a slot, then the block at that slot is considered to be the block in the beacon chain at the highest slot lower than that slot.) +The fork choice rule for any shard is LMD GHOST using the shard attestations of the shard committee and the beacon chain attestations of the crosslink committee currently assigned to that shard, but instead of being rooted in the genesis it is rooted in the block referenced in the most recent accepted crosslink (i.e. `beacon_state.crosslinks[shard].shard_block_root`). Only blocks whose `beacon_block_root` is the block in the main beacon chain at the specified `slot` should be considered. (If the beacon chain skips a slot, then the block at that slot is considered to be the block in the beacon chain at the highest slot lower than that slot.) From 2eda4c5dbc051696ba11e10b08b0717910834b54 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 3 Sep 2019 22:09:47 +0100 Subject: [PATCH 11/28] Update 1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 94d941f38..61c5c43aa 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -256,7 +256,7 @@ def shard_state_transition(beacon_state: BeaconState, block: ShardBlock, validate_state_root: bool=False) -> ShardState: # Process slots (including those with no blocks) since block - process_shard_slots(beacon_state, shard_state, block.slot) + process_shard_slots(shard_state, block.slot) # Process block process_shard_block(beacon_state, shard_state, block) # Validate state root (`validate_state_root == True` in production) @@ -267,7 +267,7 @@ def shard_state_transition(beacon_state: BeaconState, ``` ```python -def process_shard_slots(beacon_state: BeaconState, shard_state: ShardState, slot: ShardSlot) -> None: +def process_shard_slots(shard_state: ShardState, slot: ShardSlot) -> None: assert shard_state.slot <= slot while shard_state.slot < slot: process_shard_slot(shard_state) From 91e73c1f5794082c05665bb29a12cf941a92f909 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 3 Sep 2019 22:15:52 +0100 Subject: [PATCH 12/28] Deduplicate indices --- specs/core/1_shard-data-chains.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 61c5c43aa..28585d02b 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -195,7 +195,8 @@ def get_shard_committee(beacon_state: BeaconState, shard: Shard, epoch: Epoch) - # Every epoch cycle out validators from the older committee and cycle in validators from the newer committee older_subcommittee = [i for i in older_committee if i % EPOCHS_PER_SHARD_PERIOD > epoch % EPOCHS_PER_SHARD_PERIOD] newer_subcommittee = [i for i in newer_committee if i % EPOCHS_PER_SHARD_PERIOD <= epoch % EPOCHS_PER_SHARD_PERIOD] - return older_subcommittee + newer_subcommittee + # Deduplicate and sort indices + return sorted(set(older_subcommittee + newer_subcommittee)) ``` #### `get_shard_proposer_index` From 2b60c9cf90f989a73973bea8db00b88f532c28af Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 3 Sep 2019 22:45:13 +0100 Subject: [PATCH 13/28] Update 1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 28585d02b..61c5c43aa 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -195,8 +195,7 @@ def get_shard_committee(beacon_state: BeaconState, shard: Shard, epoch: Epoch) - # Every epoch cycle out validators from the older committee and cycle in validators from the newer committee older_subcommittee = [i for i in older_committee if i % EPOCHS_PER_SHARD_PERIOD > epoch % EPOCHS_PER_SHARD_PERIOD] newer_subcommittee = [i for i in newer_committee if i % EPOCHS_PER_SHARD_PERIOD <= epoch % EPOCHS_PER_SHARD_PERIOD] - # Deduplicate and sort indices - return sorted(set(older_subcommittee + newer_subcommittee)) + return older_subcommittee + newer_subcommittee ``` #### `get_shard_proposer_index` From c01d036ed1293afa16edc97bcc02aa58cb571a80 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 3 Sep 2019 22:49:33 +0100 Subject: [PATCH 14/28] Update 1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 61c5c43aa..72228c633 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -240,11 +240,11 @@ def get_genesis_shard_state(shard: Shard) -> ShardState: ### `get_genesis_shard_block` ```python -def get_genesis_shard_block(beacon_state: BeaconState, shard: Shard) -> ShardBlock: +def get_genesis_shard_block(shard: Shard) -> ShardBlock: return ShardBlock( shard=shard, slot=ShardSlot(SHARD_GENESIS_EPOCH * SHARD_SLOTS_PER_EPOCH), - state_root=hash_tree_root(get_genesis_shard_state(beacon_state, shard)), + state_root=hash_tree_root(get_genesis_shard_state(shard)), ) ``` From 2a4957c6ccee89769c9413427486a1393c7e3c12 Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 5 Sep 2019 20:07:25 +0100 Subject: [PATCH 15/28] Update 1_shard-data-chains.md --- specs/core/1_shard-data-chains.md | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 72228c633..c985294aa 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -83,9 +83,9 @@ This document describes the shard transition function (data layer only) and the ### State list lengths -| Name | Value | Unit | -| - | - | :-: | -| `HISTORY_ACCUMULATOR_VECTOR` | `2**6` (= 64) | state tree maximum depth | +| Name | Value | +| - | - | +| `HISTORY_ACCUMULATOR_DEPTH` | `2**6` (= 64) | ### Rewards and penalties @@ -111,8 +111,8 @@ class ShardBlock(FlatContainer): beacon_block_root: Hash parent_root: Hash state_root: Hash - block_size_sum: uint64 body: List[byte, MAX_SHARD_BLOCK_SIZE - SHARD_HEADER_SIZE] + block_size_sum: uint64 aggregation_bits: Bitvector[2 * MAX_PERIOD_COMMITTEE_SIZE] attestations: BLSSignature signature: BLSSignature @@ -127,8 +127,8 @@ class ShardBlockHeader(FlatContainer): beacon_block_root: Hash parent_root: Hash state_root: Hash - block_size_sum: uint64 body_root: Hash + block_size_sum: uint64 aggregation_bits: Bitvector[2 * MAX_PERIOD_COMMITTEE_SIZE] attestations: BLSSignature signature: BLSSignature @@ -140,7 +140,7 @@ class ShardBlockHeader(FlatContainer): class ShardState(FlatContainer): shard: Shard slot: ShardSlot - history_accumulator: Vector[Hash, HISTORY_ACCUMULATOR_VECTOR] + history_accumulator: Vector[Hash, HISTORY_ACCUMULATOR_DEPTH] latest_block_header: ShardBlockHeader block_size_sum: uint64 # Fees and rewards @@ -271,7 +271,7 @@ def process_shard_slots(shard_state: ShardState, slot: ShardSlot) -> None: assert shard_state.slot <= slot while shard_state.slot < slot: process_shard_slot(shard_state) - # Process period on the start slot of the next period + # Process shard period on the start slot of the next shard period if (shard_state.slot + 1) % (SHARD_SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD) == 0: process_shard_period(shard_state) shard_state.slot += ShardSlot(1) @@ -285,7 +285,7 @@ def process_shard_slot(shard_state: ShardState) -> None: shard_state.latest_block_header.state_root = previous_state_root # Cache state root in history accumulator depth = 0 - while shard_state.slot % 2**depth == 0 and depth < HISTORY_ACCUMULATOR_VECTOR: + while shard_state.slot % 2**depth == 0 and depth < HISTORY_ACCUMULATOR_DEPTH: shard_state.history_accumulator[depth] = previous_state_root depth += 1 ``` @@ -294,7 +294,7 @@ def process_shard_slot(shard_state: ShardState) -> None: ```python def process_shard_period(shard_state: ShardState) -> None: - # Rotate rewards and fees + # Rotate committee deltas shard_state.older_committee_deltas = shard_state.newer_committee_deltas shard_state.newer_committee_deltas = [GweiDelta(0) for _ in range(MAX_PERIOD_COMMITTEE_SIZE)] ``` @@ -328,20 +328,19 @@ def process_shard_block_header(beacon_state: BeaconState, shard_state: ShardStat beacon_block_root=block.beacon_block_root, parent_root=block.parent_root, # `state_root` is zeroed and overwritten in the next `process_shard_slot` call - block_size_sum=block.block_size_sum, body_root=hash_tree_root(block.body), + block_size_sum=block.block_size_sum, aggregation_bits=block.aggregation_bits, attestations=block.attestations, # `signature` is zeroed ) # Verify the sum of the block sizes since genesis - shard_state.block_size_sum += SHARD_HEADER_SIZE + len(block.body) + shard_state.block_size_sum += SHARD_HEADER_SIZE + len(block.body) assert block.block_size_sum == shard_state.block_size_sum # Verify proposer signature proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) - pubkey = beacon_state.validators[proposer_index].pubkey domain = get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.slot)) - assert bls_verify(pubkey, hash_tree_root(block.block), block.proposer, domain) + assert bls_verify(beacon_state.validators[proposer_index].pubkey, signing_root(block), block.signature, domain) ``` #### Attestations @@ -376,8 +375,8 @@ def process_shard_block_body(beacon_state: BeaconState, shard_state: ShardState, # Verify block body size is a multiple of the header size assert len(block.body) % SHARD_HEADER_SIZE == 0 # Apply proposer block body fee - proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) block_body_fee = shard_state.block_body_price * len(block.body) // MAX_SHARD_BLOCK_SIZE + proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) process_delta(beacon_state, shard_state, proposer_index, -block_body_fee) # Burn process_delta(beacon_state, shard_state, proposer_index, block_body_fee // PROPOSER_REWARD_QUOTIENT) # Reward # Calculate new block body price @@ -385,7 +384,7 @@ def process_shard_block_body(beacon_state: BeaconState, shard_state: ShardState, QUOTIENT = MAX_SHARD_BLOCK_SIZE * BLOCK_BODY_PRICE_QUOTIENT price_delta = GweiDelta(shard_state.block_body_price * (block_size - SHARD_BLOCK_SIZE_TARGET) // QUOTIENT) if price_delta > 0: - # The maximum block body price caps the amount burnt on fees within a period + # The maximum block body price caps the amount burnt on fees within a shard period MAX_BLOCK_BODY_PRICE = MAX_EFFECTIVE_BALANCE // EPOCHS_PER_SHARD_PERIOD // SHARD_SLOTS_PER_EPOCH shard_state.block_body_price = Gwei(min(MAX_BLOCK_BODY_PRICE, shard_state.block_body_price + price_delta)) else: From fd759a2cca9ba993ff0ce590845dc9479f33fa81 Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Wed, 25 Sep 2019 21:51:35 +0100 Subject: [PATCH 16/28] Remove flat containers and revert back to uint64 --- specs/core/1_shard-data-chains.md | 42 ++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index fceac061c..4596d250b 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -54,7 +54,6 @@ This document describes the shard transition function (data layer only) and the | Name | SSZ equivalent | Description | | - | - | - | | `ShardSlot` | `uint64` | a shard slot number | -| `GweiDelta` | `int64` | a signed Gwei delta | ## Configuration @@ -105,7 +104,7 @@ This document describes the shard transition function (data layer only) and the ### `ShardBlock` ```python -class ShardBlock(FlatContainer): +class ShardBlock(Container): shard: Shard slot: ShardSlot beacon_block_root: Hash @@ -121,7 +120,7 @@ class ShardBlock(FlatContainer): ### `ShardBlockHeader` ```python -class ShardBlockHeader(FlatContainer): +class ShardBlockHeader(Container): shard: Shard slot: ShardSlot beacon_block_root: Hash @@ -137,7 +136,7 @@ class ShardBlockHeader(FlatContainer): ### `ShardState` ```python -class ShardState(FlatContainer): +class ShardState(Container): shard: Shard slot: ShardSlot history_accumulator: Vector[Hash, HISTORY_ACCUMULATOR_DEPTH] @@ -145,8 +144,10 @@ class ShardState(FlatContainer): block_size_sum: uint64 # Fees and rewards block_body_price: Gwei - older_committee_deltas: Vector[GweiDelta, MAX_PERIOD_COMMITTEE_SIZE] - newer_committee_deltas: Vector[GweiDelta, MAX_PERIOD_COMMITTEE_SIZE] + older_committee_positive_deltas: Vector[Gwei, MAX_PERIOD_COMMITTEE_SIZE] + older_committee_negative_deltas: Vector[Gwei, MAX_PERIOD_COMMITTEE_SIZE] + newer_committee_positive_deltas: Vector[Gwei, MAX_PERIOD_COMMITTEE_SIZE] + newer_committee_negative_deltas: Vector[Gwei, MAX_PERIOD_COMMITTEE_SIZE] ``` ### `ShardCheckpoint` @@ -218,14 +219,24 @@ def get_shard_proposer_index(beacon_state: BeaconState, shard: Shard, slot: Shar #### `process_delta` ```python -def process_delta(beacon_state: BeaconState, shard_state: ShardState, index: ValidatorIndex, delta: GweiDelta) -> None: +def process_delta(beacon_state: BeaconState, + shard_state: ShardState, + index: ValidatorIndex, + delta: Gwei, + positive: bool=True) -> None: epoch = compute_epoch_of_shard_slot(beacon_state.slot) older_committee = get_period_committee(beacon_state, shard_state.shard, compute_shard_period_start_epoch(epoch, 2)) newer_committee = get_period_committee(beacon_state, shard_state.shard, compute_shard_period_start_epoch(epoch, 1)) if index in older_committee: - shard_state.older_committee_deltas[older_committee.index(index)] += delta + if positive: + shard_state.older_committee_positive_deltas[older_committee.index(index)] += delta + else: + shard_state.older_committee_negative_deltas[older_committee.index(index)] += delta elif index in newer_committee: - shard_state.newer_committee_deltas[newer_committee.index(index)] += delta + if positive: + shard_state.newer_committee_positive_deltas[newer_committee.index(index)] += delta + else: + shard_state.newer_committee_negative_deltas[newer_committee.index(index)] += delta ``` ## Genesis @@ -299,8 +310,10 @@ def process_shard_slot(shard_state: ShardState) -> None: ```python def process_shard_period(shard_state: ShardState) -> None: # Rotate committee deltas - shard_state.older_committee_deltas = shard_state.newer_committee_deltas - shard_state.newer_committee_deltas = [GweiDelta(0) for _ in range(MAX_PERIOD_COMMITTEE_SIZE)] + shard_state.older_committee_positive_deltas = shard_state.newer_committee_positive_deltas + shard_state.older_committee_negative_deltas = shard_state.newer_committee_negative_deltas + shard_state.newer_committee_positive_deltas = [Gwei(0) for _ in range(MAX_PERIOD_COMMITTEE_SIZE)] + shard_state.newer_committee_negative_deltas = [Gwei(0) for _ in range(MAX_PERIOD_COMMITTEE_SIZE)] ``` ### Block processing @@ -381,17 +394,18 @@ def process_shard_block_body(beacon_state: BeaconState, shard_state: ShardState, # Apply proposer block body fee block_body_fee = shard_state.block_body_price * len(block.body) // MAX_SHARD_BLOCK_SIZE proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) - process_delta(beacon_state, shard_state, proposer_index, -block_body_fee) # Burn + process_delta(beacon_state, shard_state, proposer_index, block_body_fee, positive=False) # Burn process_delta(beacon_state, shard_state, proposer_index, block_body_fee // PROPOSER_REWARD_QUOTIENT) # Reward # Calculate new block body price block_size = SHARD_HEADER_SIZE + len(block.body) QUOTIENT = MAX_SHARD_BLOCK_SIZE * BLOCK_BODY_PRICE_QUOTIENT - price_delta = GweiDelta(shard_state.block_body_price * (block_size - SHARD_BLOCK_SIZE_TARGET) // QUOTIENT) - if price_delta > 0: + if block_size > SHARD_BLOCK_SIZE_TARGET: + price_delta = Gwei(shard_state.block_body_price * (block_size - SHARD_BLOCK_SIZE_TARGET) // QUOTIENT) # The maximum block body price caps the amount burnt on fees within a shard period MAX_BLOCK_BODY_PRICE = MAX_EFFECTIVE_BALANCE // EPOCHS_PER_SHARD_PERIOD // SHARD_SLOTS_PER_EPOCH shard_state.block_body_price = Gwei(min(MAX_BLOCK_BODY_PRICE, shard_state.block_body_price + price_delta)) else: + price_delta = Gwei(shard_state.block_body_price * (SHARD_BLOCK_SIZE_TARGET - block_size) // QUOTIENT) shard_state.block_body_price = Gwei(max(MIN_BLOCK_BODY_PRICE, shard_state.block_body_price + price_delta)) ``` From d4f48117d3463cddb18f0f65d66798933d679beb Mon Sep 17 00:00:00 2001 From: Justin Drake Date: Thu, 26 Sep 2019 19:32:50 +0100 Subject: [PATCH 17/28] Rename ShardCheckpoint, double header size, verify proposer not slashed --- specs/core/1_shard-data-chains.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 4596d250b..595909745 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -21,7 +21,7 @@ - [`ShardBlock`](#shardblock) - [`ShardBlockHeader`](#shardblockheader) - [`ShardState`](#shardstate) - - [`ShardCheckpoint`](#shardcheckpoint) + - [`ShardAttestationData`](#ShardAttestationData) - [Helper functions](#helper-functions) - [Misc](#misc-1) - [`compute_epoch_of_shard_slot`](#compute_epoch_of_shard_slot) @@ -63,7 +63,7 @@ This document describes the shard transition function (data layer only) and the | - | - | | `MIN_BLOCK_BODY_PRICE` | `2**0` (= 1) | | `MAX_PERIOD_COMMITTEE_SIZE` | `2**7` (= 128) | -| `SHARD_HEADER_SIZE` | `2**9` (= 512) | +| `SHARD_HEADER_SIZE` | `2**10` (= 1024) | | `SHARD_BLOCK_SIZE_TARGET` | `2**14` (= 16,384) | | `MAX_SHARD_BLOCK_SIZE` | `2**16` (= 65,536) | @@ -150,10 +150,10 @@ class ShardState(Container): newer_committee_negative_deltas: Vector[Gwei, MAX_PERIOD_COMMITTEE_SIZE] ``` -### `ShardCheckpoint` +### `ShardAttestationData` ```python -class ShardCheckpoint(Container): +class ShardAttestationData(Container): slot: ShardSlot parent_root: Hash ``` @@ -354,10 +354,12 @@ def process_shard_block_header(beacon_state: BeaconState, shard_state: ShardStat # Verify the sum of the block sizes since genesis shard_state.block_size_sum += SHARD_HEADER_SIZE + len(block.body) assert block.block_size_sum == shard_state.block_size_sum + # Verify proposer is not slashed + proposer = beacon_state.validators[get_shard_proposer_index(beacon_state, shard_state.shard, block.slot)] + assert not proposer.slashed # Verify proposer signature - proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) domain = get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.slot)) - assert bls_verify(beacon_state.validators[proposer_index].pubkey, signing_root(block), block.signature, domain) + assert bls_verify(proposer.pubkey, signing_root(block), block.signature, domain) ``` #### Attestations @@ -377,7 +379,7 @@ def process_shard_attestations(beacon_state: BeaconState, shard_state: ShardStat assert block.aggregation_bits[i] == 0b0 # Verify attester aggregate signature domain = get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(block.slot)) - message = hash_tree_root(ShardCheckpoint(shard_state.slot, block.parent_root)) + message = hash_tree_root(ShardAttestationData(shard_state.slot, block.parent_root)) assert bls_verify(bls_aggregate_pubkeys(pubkeys), message, block.attestations, domain) # Proposer micro-reward proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) From 9ce7a8e5919dbafba7d7b9436759dd931647b114 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 27 Sep 2019 09:41:12 +0900 Subject: [PATCH 18/28] working through lint and format on shard chains --- specs/core/1_beacon-chain-misc.md | 29 ++++++++++++++++++++--------- specs/core/1_shard-data-chains.md | 10 +++++++--- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 5bb0f6da0..0b5ed5234 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -34,6 +34,7 @@ | `MAX_SHARD_RECEIPT_PROOFS` | `2**0` (= 1) | - | - | | `PERIOD_COMMITTEE_ROOT_LENGTH` | `2**8` (= 256) | periods | ~9 months | | `MINOR_REWARD_QUOTIENT` | `2**8` (=256) | - | - | +| `REWARD_COEFFICIENT_BASE` | **TBD** | - | - | ## Containers @@ -45,6 +46,16 @@ class CompactCommittee(Container): compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] ``` +#### `ShardReceiptDelta` + +```python +class ShardReceiptDelta(Container): + index: ValidatorIndex + reward_coefficient: uint64 + block_fee: Gwei +``` + + #### `ShardReceiptProof` ```python @@ -112,16 +123,17 @@ def verify_merkle_proof(leaf: Hash, proof: Sequence[Hash], index: GeneralizedInd ```python def compute_historical_state_generalized_index(earlier: ShardSlot, later: ShardSlot) -> GeneralizedIndex: """ - Computes the generalized index of the state root of slot `frm` based on the state root of slot `to`. - Relies on the `history_acc` in the `ShardState`, where `history_acc[i]` maintains the most recent 2**i'th - slot state. Works by tracing a `log(later-earlier)` step path from `later` to `earlier` through intermediate - blocks at the next available multiples of descending powers of two. + Computes the generalized index of the state root of slot `earlier` based on the state root of slot `later`. + Relies on the `history_accumulator` in the `ShardState`, where `history_accumulator[i]` maintains the most + recent 2**i'th slot state. Works by tracing a `log(later-earlier)` step path from `later` to `earlier` + through intermediate blocks at the next available multiples of descending powers of two. """ o = GeneralizedIndex(1) - for i in range(HISTORY_ACCUMULATOR_VECTOR - 1, -1, -1): + for i in range(HISTORY_ACCUMULATOR_DEPTH - 1, -1, -1): if (later - 1) & 2**i > (earlier - 1) & 2**i: later = later - ((later - 1) % 2**i) - 1 - o = concat_generalized_indices(o, GeneralizedIndex(get_generalized_index(ShardState, ['history_acc', i]))) + gindex = GeneralizedIndex(get_generalized_index(ShardState, ['history_accumulator', i])) + o = concat_generalized_indices(o, gindex) return o ``` @@ -133,7 +145,7 @@ def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: Gets the generalized index for the root of the index'th header in a crosslink. """ MAX_CROSSLINK_SIZE = ( - SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK + MAX_SHARD_BLOCK_SIZE * SHARD_SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK ) assert MAX_CROSSLINK_SIZE == get_previous_power_of_two(MAX_CROSSLINK_SIZE) return GeneralizedIndex(MAX_CROSSLINK_SIZE // SHARD_HEADER_SIZE + index) @@ -146,10 +158,9 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP """ Processes a ShardReceipt object. """ - SHARD_SLOTS_PER_EPOCH = SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH receipt_slot = ( state.next_shard_receipt_period[receipt_proof.shard] * - SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD + SHARD_SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD ) first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SHARD_SLOTS_PER_EPOCH gindex = concat_generalized_indices( diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 4596d250b..4e26c7c72 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -356,6 +356,8 @@ def process_shard_block_header(beacon_state: BeaconState, shard_state: ShardStat assert block.block_size_sum == shard_state.block_size_sum # Verify proposer signature proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) + assert proposer_index is not None + domain = get_domain(beacon_state, DOMAIN_SHARD_PROPOSER, compute_epoch_of_shard_slot(block.slot)) assert bls_verify(beacon_state.validators[proposer_index].pubkey, signing_root(block), block.signature, domain) ``` @@ -381,8 +383,9 @@ def process_shard_attestations(beacon_state: BeaconState, shard_state: ShardStat assert bls_verify(bls_aggregate_pubkeys(pubkeys), message, block.attestations, domain) # Proposer micro-reward proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) + assert proposer_index is not None reward = attestation_count * get_base_reward(beacon_state, proposer_index) // PROPOSER_REWARD_QUOTIENT - process_delta(beacon_state, shard_state, proposer_index, reward) + process_delta(beacon_state, shard_state, proposer_index, Gwei(reward)) ``` #### Block body @@ -394,8 +397,9 @@ def process_shard_block_body(beacon_state: BeaconState, shard_state: ShardState, # Apply proposer block body fee block_body_fee = shard_state.block_body_price * len(block.body) // MAX_SHARD_BLOCK_SIZE proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) - process_delta(beacon_state, shard_state, proposer_index, block_body_fee, positive=False) # Burn - process_delta(beacon_state, shard_state, proposer_index, block_body_fee // PROPOSER_REWARD_QUOTIENT) # Reward + assert proposer_index is not None + process_delta(beacon_state, shard_state, proposer_index, Gwei(block_body_fee), positive=False) # Burn + process_delta(beacon_state, shard_state, proposer_index, Gwei(block_body_fee // PROPOSER_REWARD_QUOTIENT)) # Reward # Calculate new block body price block_size = SHARD_HEADER_SIZE + len(block.body) QUOTIENT = MAX_SHARD_BLOCK_SIZE * BLOCK_BODY_PRICE_QUOTIENT From 86ed3937dc6f5e52ae7b869e4696a132ec054171 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 27 Sep 2019 10:34:19 +0900 Subject: [PATCH 19/28] fix a couple of minor shard chain bugs --- specs/core/1_beacon-chain-misc.md | 22 ++++++++++++---------- specs/core/1_shard-data-chains.md | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 0b5ed5234..6dd6e19c3 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -226,16 +226,18 @@ def update_period_committee(state: BeaconState) -> None: """ Updates period committee roots at boundary blocks. """ - if (get_current_epoch(state) + 1) % EPOCHS_PER_SHARD_PERIOD == 0: - period = (get_current_epoch(state) + 1) // EPOCHS_PER_SHARD_PERIOD - committees = Vector[CompactCommittee, SHARD_COUNT]([ - committee_to_compact_committee( - state, - get_period_committee(state, Epoch(get_current_epoch(state) + 1), Shard(shard)), - ) - for shard in range(SHARD_COUNT) - ]) - state.period_committee_roots[period % PERIOD_COMMITTEE_ROOT_LENGTH] = hash_tree_root(committees) + if (get_current_epoch(state) + 1) % EPOCHS_PER_SHARD_PERIOD != 0: + return + + period = (get_current_epoch(state) + 1) // EPOCHS_PER_SHARD_PERIOD + committees = Vector[CompactCommittee, SHARD_COUNT]([ + committee_to_compact_committee( + state, + get_period_committee(state, Shard(shard), Epoch(get_current_epoch(state) + 1)), + ) + for shard in range(SHARD_COUNT) + ]) + state.period_committee_roots[period % PERIOD_COMMITTEE_ROOT_LENGTH] = hash_tree_root(committees) ``` ### Shard receipt processing diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index ebb1b6b68..96b2de41a 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -224,7 +224,7 @@ def process_delta(beacon_state: BeaconState, index: ValidatorIndex, delta: Gwei, positive: bool=True) -> None: - epoch = compute_epoch_of_shard_slot(beacon_state.slot) + epoch = compute_epoch_of_shard_slot(shard_state.slot) older_committee = get_period_committee(beacon_state, shard_state.shard, compute_shard_period_start_epoch(epoch, 2)) newer_committee = get_period_committee(beacon_state, shard_state.shard, compute_shard_period_start_epoch(epoch, 1)) if index in older_committee: From b892d46f26da482af948c7da53586299d335c4f6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 27 Sep 2019 13:02:16 +0900 Subject: [PATCH 20/28] working through shard chain tests --- scripts/function_puller.py | 2 +- specs/core/1_shard-data-chains.md | 21 ++++++- .../test/helpers/phase1/attestations.py | 13 ++-- .../test/helpers/phase1/shard_block.py | 61 +++++++++---------- .../shard_data_chain/test_shard_block.py | 34 +++++------ 5 files changed, 71 insertions(+), 60 deletions(-) diff --git a/scripts/function_puller.py b/scripts/function_puller.py index 26671bafc..b30e5b75c 100644 --- a/scripts/function_puller.py +++ b/scripts/function_puller.py @@ -81,7 +81,7 @@ def get_spec(file_name: str) -> SpecObject: if c not in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789': is_constant_def = False if is_constant_def: - constants[row[0]] = row[1].replace('**TBD**', '0x1234567890123456789012345678901234567890') + constants[row[0]] = row[1].replace('**TBD**', '2**32') elif row[1].startswith('uint') or row[1].startswith('Bytes'): custom_types[row[0]] = row[1] return functions, custom_types, constants, ssz_objects, inserts diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 96b2de41a..866140831 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -248,6 +248,11 @@ def get_genesis_shard_state(shard: Shard) -> ShardState: return ShardState( shard=shard, slot=ShardSlot(SHARD_GENESIS_EPOCH * SHARD_SLOTS_PER_EPOCH), + latest_block_header=ShardBlockHeader( + shard=shard, + slot=ShardSlot(SHARD_GENESIS_EPOCH * SHARD_SLOTS_PER_EPOCH), + body_root=hash_tree_root(List[byte, MAX_SHARD_BLOCK_SIZE - SHARD_HEADER_SIZE]()), + ), block_body_price=MIN_BLOCK_BODY_PRICE, ) ``` @@ -335,9 +340,14 @@ def process_shard_block_header(beacon_state: BeaconState, shard_state: ShardStat assert block.slot == shard_state.slot # Verify the beacon chain root parent_epoch = compute_epoch_of_shard_slot(shard_state.latest_block_header.slot) - assert block.beacon_block_root == get_block_root(beacon_state, parent_epoch) + # --super dirty. need to think-- # + if parent_epoch * SLOTS_PER_EPOCH == beacon_state.slot: + beacon_block_root = signing_root(beacon_state.latest_block_header) + else: + beacon_block_root = get_block_root(beacon_state, parent_epoch) + assert block.beacon_block_root == beacon_block_root # Verify the parent root - assert block.parent_root == hash_tree_root(shard_state.latest_block_header) + assert block.parent_root == signing_root(shard_state.latest_block_header) # Save current block as the new latest block shard_state.latest_block_header = ShardBlockHeader( shard=block.shard, @@ -376,12 +386,17 @@ def process_shard_attestations(beacon_state: BeaconState, shard_state: ShardStat pubkeys.append(beacon_state.validators[validator_index].pubkey) process_delta(beacon_state, shard_state, validator_index, get_base_reward(beacon_state, validator_index)) attestation_count += 1 + # Exit early if no participants + if not any(pubkeys): + assert block.attestations == BLSSignature() + return + # Verify there are no extraneous bits set beyond the shard committee for i in range(len(shard_committee), 2 * MAX_PERIOD_COMMITTEE_SIZE): assert block.aggregation_bits[i] == 0b0 # Verify attester aggregate signature domain = get_domain(beacon_state, DOMAIN_SHARD_ATTESTER, compute_epoch_of_shard_slot(block.slot)) - message = hash_tree_root(ShardAttestationData(shard_state.slot, block.parent_root)) + message = hash_tree_root(ShardAttestationData(slot=shard_state.slot, parent_root=block.parent_root)) assert bls_verify(bls_aggregate_pubkeys(pubkeys), message, block.attestations, domain) # Proposer micro-reward proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py index 750ab5048..4f0a9fb0a 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/attestations.py @@ -5,17 +5,20 @@ from eth2spec.utils.bls import ( ) -def sign_shard_attestation(spec, shard_state, beacon_state, block, participants): +def sign_shard_attestation(spec, beacon_state, shard_state, block, participants): signatures = [] - message_hash = block.core.parent_root - block_epoch = spec.compute_epoch_of_shard_slot(block.core.slot) + message_hash = spec.ShardAttestationData( + slot=block.slot, + parent_root=block.parent_root, + ).hash_tree_root() + block_epoch = spec.compute_epoch_of_shard_slot(block.slot) for validator_index in participants: privkey = privkeys[validator_index] signatures.append( get_attestation_signature( spec, - shard_state, beacon_state, + shard_state, message_hash, block_epoch, privkey, @@ -25,7 +28,7 @@ def sign_shard_attestation(spec, shard_state, beacon_state, block, participants) return bls_aggregate_signatures(signatures) -def get_attestation_signature(spec, shard_state, beacon_state, message_hash, block_epoch, privkey): +def get_attestation_signature(spec, beacon_state, shard_state, message_hash, block_epoch, privkey): return bls_sign( message_hash=message_hash, privkey=privkey, diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py index b9c388a3f..3ceb3e0aa 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py @@ -1,3 +1,5 @@ +from copy import deepcopy + from eth2spec.test.helpers.keys import privkeys from eth2spec.utils.bls import ( bls_sign, @@ -13,66 +15,63 @@ from .attestations import ( @only_with_bls() -def sign_shard_block(spec, state, block, shard, proposer_index=None): +def sign_shard_block(spec, beacon_state, block, shard, proposer_index=None): if proposer_index is None: - proposer_index = spec.get_shard_block_proposer_index(state, shard, block.core.slot) + proposer_index = spec.get_shard_proposer_index(beacon_state, shard, block.slot) privkey = privkeys[proposer_index] - block.signatures.proposer_signature = bls_sign( + block.signature = bls_sign( message_hash=signing_root(block), privkey=privkey, domain=spec.get_domain( - state, + beacon_state, spec.DOMAIN_SHARD_PROPOSER, - spec.compute_epoch_of_shard_slot(block.core.slot), + spec.compute_epoch_of_shard_slot(block.slot), ) ) def build_empty_shard_block(spec, - shard_state, beacon_state, + shard_state, slot, - parent_root, signed=False, full_attestation=False): if slot is None: slot = shard_state.slot + parent_epoch = spec.compute_epoch_of_shard_slot(shard_state.latest_block_header.slot) + if parent_epoch * spec.SLOTS_PER_EPOCH == beacon_state.slot: + beacon_block_root = spec.signing_root(beacon_state.latest_block_header) + else: + beacon_block_root = spec.get_block_root(beacon_state, parent_epoch) + + previous_block_header = deepcopy(shard_state.latest_block_header) + if previous_block_header.state_root == spec.Hash(): + previous_block_header.state_root = shard_state.hash_tree_root() + parent_root = signing_root(previous_block_header) + block = spec.ShardBlock( - core=spec.ExtendedShardBlockCore( - slot=slot, - beacon_chain_root=beacon_state.block_roots[beacon_state.slot % spec.SLOTS_PER_HISTORICAL_ROOT], - parent_root=parent_root, - ), - signatures=spec.ShardBlockSignatures( - attestation_signature=b'\x00' * 96, - proposer_signature=b'\x25' * 96, - ) + shard=shard_state.shard, + slot=slot, + beacon_block_root=beacon_block_root, + parent_root=parent_root, + block_size_sum=shard_state.block_size_sum + spec.SHARD_HEADER_SIZE, ) - # attestation if full_attestation: - attester_committee = spec.get_persistent_committee(beacon_state, shard_state.shard, block.core.slot) - block.core.attester_bitfield = list( - (True,) * len(attester_committee) + - (False,) * (spec.TARGET_PERSISTENT_COMMITTEE_SIZE * 2 - len(attester_committee)) + shard_committee = spec.get_shard_committee(beacon_state, shard_state.shard, block.slot) + block.aggregation_bits = list( + (True,) * len(shard_committee) + + (False,) * (spec.MAX_PERIOD_COMMITTEE_SIZE * 2 - len(shard_committee)) ) - block.signatures.attestation_signature = sign_shard_attestation( + block.attestations = sign_shard_attestation( spec, - shard_state, beacon_state, - block, - participants=attester_committee, - ) - else: - block.signatures.attestation_signature = sign_shard_attestation( - spec, shard_state, - beacon_state, block, - participants=(), + participants=shard_committee, ) if signed: diff --git a/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py b/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py index 2bb0232f0..b0d8ad5e6 100644 --- a/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py +++ b/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py @@ -9,22 +9,19 @@ from eth2spec.test.context import ( @with_all_phases_except(['phase0']) -@always_bls @spec_state_test +@always_bls def test_process_empty_shard_block(spec, state): beacon_state = state - - shard_slot = spec.PHASE_1_FORK_SLOT - beacon_state.slot = spec.Slot(spec.PHASE_1_FORK_EPOCH * spec.SLOTS_PER_EPOCH) - shard_state = spec.get_default_shard_state(beacon_state, shard=spec.Shard(0)) - shard_state.slot = shard_slot + beacon_state.slot = spec.Slot(spec.SHARD_GENESIS_EPOCH * spec.SLOTS_PER_EPOCH) + shard_state = spec.get_genesis_shard_state(spec.Shard(0)) + shard_state.slot = spec.ShardSlot(spec.SHARD_GENESIS_EPOCH * spec.SHARD_SLOTS_PER_EPOCH) block = build_empty_shard_block( spec, - shard_state, beacon_state, - slot=shard_slot + 1, - parent_root=spec.Hash(), + shard_state, + slot=shard_state.slot + 1, signed=True, full_attestation=False, ) @@ -33,28 +30,25 @@ def test_process_empty_shard_block(spec, state): yield 'beacon_state', beacon_state yield 'block', block - spec.shard_state_transition(shard_state, beacon_state, block) + spec.shard_state_transition(beacon_state, shard_state, block) yield 'post', shard_state @with_all_phases_except(['phase0']) -@always_bls @spec_state_test +@always_bls def test_process_full_attestation_shard_block(spec, state): beacon_state = state - - shard_slot = spec.PHASE_1_FORK_SLOT - beacon_state.slot = spec.Slot(spec.PHASE_1_FORK_EPOCH * spec.SLOTS_PER_EPOCH) - shard_state = spec.get_default_shard_state(beacon_state, shard=spec.Shard(0)) - shard_state.slot = shard_slot + beacon_state.slot = spec.Slot(spec.SHARD_GENESIS_EPOCH * spec.SLOTS_PER_EPOCH) + shard_state = spec.get_genesis_shard_state(spec.Shard(0)) + shard_state.slot = spec.SHARD_GENESIS_EPOCH * spec.SHARD_SLOTS_PER_EPOCH block = build_empty_shard_block( spec, - shard_state, beacon_state, - slot=shard_slot + 1, - parent_root=spec.Hash(), + shard_state, + slot=shard_state.slot + 1, signed=True, full_attestation=True, ) @@ -63,6 +57,6 @@ def test_process_full_attestation_shard_block(spec, state): yield 'beacon_state', beacon_state yield 'block', block - spec.shard_state_transition(shard_state, beacon_state, block) + spec.shard_state_transition(beacon_state, shard_state, block) yield 'post', shard_state From 3bc05dfff7cf0833a381d54b679f898a7d9753bf Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 27 Sep 2019 13:20:23 +0900 Subject: [PATCH 21/28] remove outdated beacon attestation test --- .../test_beacon_attestation.py | 48 ------------------- .../shard_data_chain/test_shard_block.py | 6 +-- 2 files changed, 2 insertions(+), 52 deletions(-) delete mode 100644 test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_beacon_attestation.py diff --git a/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_beacon_attestation.py b/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_beacon_attestation.py deleted file mode 100644 index aface905b..000000000 --- a/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_beacon_attestation.py +++ /dev/null @@ -1,48 +0,0 @@ -from eth2spec.test.context import ( - with_all_phases_except, - spec_state_test, - always_bls, -) -from eth2spec.test.helpers.phase1.shard_block import ( - build_empty_shard_block, -) -from eth2spec.test.helpers.attestations import get_valid_attestation - - -@with_all_phases_except(['phase0']) -@always_bls -@spec_state_test -def test_process_empty_shard_block(spec, state): - beacon_state = state - - shard_slot = spec.PHASE_1_FORK_SLOT - beacon_state.slot = spec.Slot(spec.PHASE_1_FORK_EPOCH * spec.SLOTS_PER_EPOCH) - shard_state = spec.get_default_shard_state(beacon_state, shard=spec.Shard(0)) - shard_state.slot = shard_slot - - block = build_empty_shard_block( - spec, - shard_state, - beacon_state, - slot=shard_slot + 1, - parent_root=spec.Hash(), - signed=True, - full_attestation=True, - ) - - yield 'pre', shard_state - yield 'beacon_state', beacon_state - yield 'block', block - - beacon_attestation = get_valid_attestation(spec, beacon_state, signed=True) - yield 'beacon_attestation', beacon_attestation - - is_valid_beacon_attestation = spec.is_valid_beacon_attestation( - pre_state=shard_state, - shard_blocks_or_state_roots=(block,), - beacon_state=beacon_state, - valid_attestations=set([beacon_attestation]), - candidate=beacon_attestation, - ) - assert is_valid_beacon_attestation - yield 'is_valid_beacon_attestation', is_valid_beacon_attestation diff --git a/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py b/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py index b0d8ad5e6..6622c1940 100644 --- a/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py +++ b/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py @@ -11,8 +11,7 @@ from eth2spec.test.context import ( @with_all_phases_except(['phase0']) @spec_state_test @always_bls -def test_process_empty_shard_block(spec, state): - beacon_state = state +def test_process_empty_shard_block(spec, beacon_state): beacon_state.slot = spec.Slot(spec.SHARD_GENESIS_EPOCH * spec.SLOTS_PER_EPOCH) shard_state = spec.get_genesis_shard_state(spec.Shard(0)) shard_state.slot = spec.ShardSlot(spec.SHARD_GENESIS_EPOCH * spec.SHARD_SLOTS_PER_EPOCH) @@ -38,8 +37,7 @@ def test_process_empty_shard_block(spec, state): @with_all_phases_except(['phase0']) @spec_state_test @always_bls -def test_process_full_attestation_shard_block(spec, state): - beacon_state = state +def test_process_full_attestation_shard_block(spec, beacon_state): beacon_state.slot = spec.Slot(spec.SHARD_GENESIS_EPOCH * spec.SLOTS_PER_EPOCH) shard_state = spec.get_genesis_shard_state(spec.Shard(0)) shard_state.slot = spec.SHARD_GENESIS_EPOCH * spec.SHARD_SLOTS_PER_EPOCH From 77faa026a03834b33a01852175f8919f0089ab72 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 27 Sep 2019 13:30:31 +0900 Subject: [PATCH 22/28] minor fix to tests --- .../test/phase_1/shard_data_chain/test_shard_block.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py b/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py index 6622c1940..b0d8ad5e6 100644 --- a/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py +++ b/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py @@ -11,7 +11,8 @@ from eth2spec.test.context import ( @with_all_phases_except(['phase0']) @spec_state_test @always_bls -def test_process_empty_shard_block(spec, beacon_state): +def test_process_empty_shard_block(spec, state): + beacon_state = state beacon_state.slot = spec.Slot(spec.SHARD_GENESIS_EPOCH * spec.SLOTS_PER_EPOCH) shard_state = spec.get_genesis_shard_state(spec.Shard(0)) shard_state.slot = spec.ShardSlot(spec.SHARD_GENESIS_EPOCH * spec.SHARD_SLOTS_PER_EPOCH) @@ -37,7 +38,8 @@ def test_process_empty_shard_block(spec, beacon_state): @with_all_phases_except(['phase0']) @spec_state_test @always_bls -def test_process_full_attestation_shard_block(spec, beacon_state): +def test_process_full_attestation_shard_block(spec, state): + beacon_state = state beacon_state.slot = spec.Slot(spec.SHARD_GENESIS_EPOCH * spec.SLOTS_PER_EPOCH) shard_state = spec.get_genesis_shard_state(spec.Shard(0)) shard_state.slot = spec.SHARD_GENESIS_EPOCH * spec.SHARD_SLOTS_PER_EPOCH From b259d3518b28622b2627a4b23932d790ff8d652c Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 30 Sep 2019 11:34:28 +0900 Subject: [PATCH 23/28] move assert to get_beacon_proposer_index --- scripts/build_spec.py | 2 +- specs/core/1_shard-data-chains.md | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/scripts/build_spec.py b/scripts/build_spec.py index ffdbda505..f35332e64 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -37,7 +37,7 @@ from eth2spec.utils.bls import ( from eth2spec.utils.hash_function import hash ''' PHASE1_IMPORTS = '''from typing import ( - Any, Dict, Optional, Set, Sequence, MutableSequence, NewType, Tuple, Union, + Any, Dict, Set, Sequence, MutableSequence, NewType, Tuple, Union, ) from math import ( log2, diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 96b2de41a..9293a50da 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -202,12 +202,11 @@ def get_shard_committee(beacon_state: BeaconState, shard: Shard, epoch: Epoch) - #### `get_shard_proposer_index` ```python -def get_shard_proposer_index(beacon_state: BeaconState, shard: Shard, slot: ShardSlot) -> Optional[ValidatorIndex]: +def get_shard_proposer_index(beacon_state: BeaconState, shard: Shard, slot: ShardSlot) -> ValidatorIndex: epoch = get_current_epoch(beacon_state) shard_committee = get_shard_committee(beacon_state, shard, epoch) active_indices = [i for i in shard_committee if is_active_validator(beacon_state.validators[i], epoch)] - if not any(active_indices): - return None + assert any(active_indices) epoch_seed = get_seed(beacon_state, epoch, DOMAIN_SHARD_PROPOSER) seed = hash(epoch_seed + int_to_bytes(slot, length=8) + int_to_bytes(shard, length=8)) @@ -356,7 +355,6 @@ def process_shard_block_header(beacon_state: BeaconState, shard_state: ShardStat assert block.block_size_sum == shard_state.block_size_sum # Verify proposer is not slashed proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) - assert proposer_index is not None proposer = beacon_state.validators[proposer_index] assert not proposer.slashed # Verify proposer signature @@ -385,7 +383,6 @@ def process_shard_attestations(beacon_state: BeaconState, shard_state: ShardStat assert bls_verify(bls_aggregate_pubkeys(pubkeys), message, block.attestations, domain) # Proposer micro-reward proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) - assert proposer_index is not None reward = attestation_count * get_base_reward(beacon_state, proposer_index) // PROPOSER_REWARD_QUOTIENT process_delta(beacon_state, shard_state, proposer_index, Gwei(reward)) ``` @@ -399,7 +396,6 @@ def process_shard_block_body(beacon_state: BeaconState, shard_state: ShardState, # Apply proposer block body fee block_body_fee = shard_state.block_body_price * len(block.body) // MAX_SHARD_BLOCK_SIZE proposer_index = get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) - assert proposer_index is not None process_delta(beacon_state, shard_state, proposer_index, Gwei(block_body_fee), positive=False) # Burn process_delta(beacon_state, shard_state, proposer_index, Gwei(block_body_fee // PROPOSER_REWARD_QUOTIENT)) # Reward # Calculate new block body price From 49a291909936cec4653db407f8e53ca09631f136 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 30 Sep 2019 12:58:05 +0900 Subject: [PATCH 24/28] add more shard block sanity tests --- .../test/helpers/phase1/shard_block.py | 6 +- .../test/helpers/phase1/shard_state.py | 18 ++ .../test/phase_1/sanity/test_shard_blocks.py | 170 ++++++++++++++++++ .../shard_data_chain/test_shard_block.py | 62 ------- 4 files changed, 191 insertions(+), 65 deletions(-) create mode 100644 test_libs/pyspec/eth2spec/test/helpers/phase1/shard_state.py create mode 100644 test_libs/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py delete mode 100644 test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py index 3ceb3e0aa..ea5783655 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py @@ -15,9 +15,9 @@ from .attestations import ( @only_with_bls() -def sign_shard_block(spec, beacon_state, block, shard, proposer_index=None): +def sign_shard_block(spec, beacon_state, shard_state, block, proposer_index=None): if proposer_index is None: - proposer_index = spec.get_shard_proposer_index(beacon_state, shard, block.slot) + proposer_index = spec.get_shard_proposer_index(beacon_state, shard_state.shard, block.slot) privkey = privkeys[proposer_index] @@ -75,6 +75,6 @@ def build_empty_shard_block(spec, ) if signed: - sign_shard_block(spec, beacon_state, block, shard_state.shard) + sign_shard_block(spec, beacon_state, shard_state, block) return block diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_state.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_state.py new file mode 100644 index 000000000..24240b5fa --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_state.py @@ -0,0 +1,18 @@ +from eth2spec.test.helpers.phase1.shard_block import sign_shard_block + + +def configure_shard_state(spec, beacon_state, shard=0): + beacon_state.slot = spec.Slot(spec.SHARD_GENESIS_EPOCH * spec.SLOTS_PER_EPOCH) + shard_state = spec.get_genesis_shard_state(spec.Shard(shard)) + shard_state.slot = spec.ShardSlot(spec.SHARD_GENESIS_EPOCH * spec.SHARD_SLOTS_PER_EPOCH) + return beacon_state, shard_state + + +def shard_state_transition_and_sign_block(spec, beacon_state, shard_state, block): + """ + Shard state transition via the provided ``block`` + then package the block with the state root and signature. + """ + spec.shard_state_transition(beacon_state, shard_state, block) + block.state_root = shard_state.hash_tree_root() + sign_shard_block(spec, beacon_state, shard_state, block) diff --git a/test_libs/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py b/test_libs/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py new file mode 100644 index 000000000..2669ffb53 --- /dev/null +++ b/test_libs/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py @@ -0,0 +1,170 @@ +from copy import deepcopy + +from eth2spec.test.helpers.phase1.shard_block import ( + build_empty_shard_block, + sign_shard_block, +) +from eth2spec.test.helpers.phase1.shard_state import ( + configure_shard_state, + shard_state_transition_and_sign_block, +) +from eth2spec.test.context import ( + always_bls, + expect_assertion_error, + spec_state_test, + with_all_phases_except, +) + + +@with_all_phases_except(['phase0']) +@spec_state_test +@always_bls +def test_process_empty_shard_block(spec, state): + beacon_state, shard_state = configure_shard_state(spec, state) + + block = build_empty_shard_block( + spec, + beacon_state, + shard_state, + slot=shard_state.slot + 1, + signed=True, + full_attestation=False, + ) + + yield 'pre', shard_state + yield 'beacon_state', beacon_state + + shard_state_transition_and_sign_block(spec, beacon_state, shard_state, block) + + yield 'blocks', [block] + yield 'post', shard_state + + +@with_all_phases_except(['phase0']) +@spec_state_test +@always_bls +def test_process_full_attestation_shard_block(spec, state): + beacon_state, shard_state = configure_shard_state(spec, state) + + block = build_empty_shard_block( + spec, + beacon_state, + shard_state, + slot=shard_state.slot + 1, + signed=True, + full_attestation=True, + ) + + yield 'pre', shard_state + yield 'beacon_state', beacon_state + + shard_state_transition_and_sign_block(spec, beacon_state, shard_state, block) + + yield 'blocks', [block] + yield 'post', shard_state + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_prev_slot_block_transition(spec, state): + beacon_state, shard_state = configure_shard_state(spec, state) + + # Go to clean slot + spec.process_shard_slots(shard_state, shard_state.slot + 1) + # Make a block for it + block = build_empty_shard_block(spec, beacon_state, shard_state, slot=shard_state.slot, signed=True) + # Transition to next slot, above block will not be invalid on top of new state. + spec.process_shard_slots(shard_state, shard_state.slot + 1) + + yield 'pre', shard_state + yield 'beacon_state', beacon_state + expect_assertion_error( + lambda: spec.shard_state_transition(beacon_state, shard_state, block) + ) + yield 'blocks', [block] + yield 'post', None + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_same_slot_block_transition(spec, state): + beacon_state, shard_state = configure_shard_state(spec, state) + + # Same slot on top of pre-state, but move out of slot 0 first. + spec.process_shard_slots(shard_state, shard_state.slot + 1) + block = build_empty_shard_block(spec, beacon_state, shard_state, slot=shard_state.slot, signed=True) + + yield 'pre', shard_state + yield 'beacon_state', beacon_state + + shard_state_transition_and_sign_block(spec, beacon_state, shard_state, block) + + yield 'blocks', [block] + yield 'post', shard_state + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_invalid_state_root(spec, state): + beacon_state, shard_state = configure_shard_state(spec, state) + + spec.process_shard_slots(shard_state, shard_state.slot + 1) + block = build_empty_shard_block(spec, beacon_state, shard_state, slot=shard_state.slot) + block.state_root = b'\x36' * 32 + sign_shard_block(spec, beacon_state, shard_state, block) + + yield 'pre', shard_state + yield 'beacon_state', beacon_state + expect_assertion_error( + lambda: spec.shard_state_transition(beacon_state, shard_state, block, validate_state_root=True) + ) + yield 'blocks', [block] + yield 'post', None + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_skipped_slots(spec, state): + beacon_state, shard_state = configure_shard_state(spec, state) + + block = build_empty_shard_block(spec, beacon_state, shard_state, slot=shard_state.slot + 3, signed=True) + + yield 'pre', shard_state + yield 'beacon_state', beacon_state + + shard_state_transition_and_sign_block(spec, beacon_state, shard_state, block) + + yield 'blocks', [block] + yield 'post', shard_state + + assert shard_state.slot == block.slot + latest_block_header = deepcopy(shard_state.latest_block_header) + latest_block_header.state_root = shard_state.hash_tree_root() + assert latest_block_header.signing_root() == block.signing_root() + + +@with_all_phases_except(['phase0']) +@spec_state_test +def test_empty_shard_period_transition(spec, state): + beacon_state, shard_state = configure_shard_state(spec, state) + + # modify some of the deltas to ensure the period transition works properly + stub_delta = 10 + shard_state.newer_committee_positive_deltas[0] = stub_delta + shard_state.newer_committee_negative_deltas[0] = stub_delta + + slot = shard_state.slot + spec.SHARD_SLOTS_PER_EPOCH * spec.EPOCHS_PER_SHARD_PERIOD + block = build_empty_shard_block(spec, beacon_state, shard_state, slot=slot, signed=True) + + yield 'pre', shard_state + yield 'beacon_state', beacon_state + + shard_state_transition_and_sign_block(spec, beacon_state, shard_state, block) + + yield 'blocks', [block] + yield 'post', shard_state + + shard_state.older_committee_positive_deltas[0] == stub_delta + shard_state.older_committee_negative_deltas[0] == stub_delta + shard_state.newer_committee_positive_deltas[0] == 0 + shard_state.newer_committee_negative_deltas[0] == 0 diff --git a/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py b/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py deleted file mode 100644 index b0d8ad5e6..000000000 --- a/test_libs/pyspec/eth2spec/test/phase_1/shard_data_chain/test_shard_block.py +++ /dev/null @@ -1,62 +0,0 @@ -from eth2spec.test.helpers.phase1.shard_block import ( - build_empty_shard_block, -) -from eth2spec.test.context import ( - with_all_phases_except, - spec_state_test, - always_bls, -) - - -@with_all_phases_except(['phase0']) -@spec_state_test -@always_bls -def test_process_empty_shard_block(spec, state): - beacon_state = state - beacon_state.slot = spec.Slot(spec.SHARD_GENESIS_EPOCH * spec.SLOTS_PER_EPOCH) - shard_state = spec.get_genesis_shard_state(spec.Shard(0)) - shard_state.slot = spec.ShardSlot(spec.SHARD_GENESIS_EPOCH * spec.SHARD_SLOTS_PER_EPOCH) - - block = build_empty_shard_block( - spec, - beacon_state, - shard_state, - slot=shard_state.slot + 1, - signed=True, - full_attestation=False, - ) - - yield 'pre', shard_state - yield 'beacon_state', beacon_state - yield 'block', block - - spec.shard_state_transition(beacon_state, shard_state, block) - - yield 'post', shard_state - - -@with_all_phases_except(['phase0']) -@spec_state_test -@always_bls -def test_process_full_attestation_shard_block(spec, state): - beacon_state = state - beacon_state.slot = spec.Slot(spec.SHARD_GENESIS_EPOCH * spec.SLOTS_PER_EPOCH) - shard_state = spec.get_genesis_shard_state(spec.Shard(0)) - shard_state.slot = spec.SHARD_GENESIS_EPOCH * spec.SHARD_SLOTS_PER_EPOCH - - block = build_empty_shard_block( - spec, - beacon_state, - shard_state, - slot=shard_state.slot + 1, - signed=True, - full_attestation=True, - ) - - yield 'pre', shard_state - yield 'beacon_state', beacon_state - yield 'block', block - - spec.shard_state_transition(beacon_state, shard_state, block) - - yield 'post', shard_state From a5fb9408aee6f6bdfc9b9ff57c3989d8b25dd080 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 30 Sep 2019 13:11:23 +0900 Subject: [PATCH 25/28] make beacon state for shard sate transition from current shard epoch --- specs/core/1_shard-data-chains.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 3fe6f6a65..2045c7a7e 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -338,12 +338,12 @@ def process_shard_block_header(beacon_state: BeaconState, shard_state: ShardStat # Verify the slot number assert block.slot == shard_state.slot # Verify the beacon chain root - parent_epoch = compute_epoch_of_shard_slot(shard_state.latest_block_header.slot) - # --super dirty. need to think-- # - if parent_epoch * SLOTS_PER_EPOCH == beacon_state.slot: + epoch = compute_epoch_of_shard_slot(shard_state.slot) + assert epoch == compute_epoch_of_slot(beacon_state.slot) + if epoch * SLOTS_PER_EPOCH == beacon_state.slot: beacon_block_root = signing_root(beacon_state.latest_block_header) else: - beacon_block_root = get_block_root(beacon_state, parent_epoch) + beacon_block_root = get_block_root(beacon_state, epoch) assert block.beacon_block_root == beacon_block_root # Verify the parent root assert block.parent_root == signing_root(shard_state.latest_block_header) From 2af39ad4694a1ad7eb91856e3820d2d50c21953f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 30 Sep 2019 13:58:48 +0900 Subject: [PATCH 26/28] fix shard period sanity test --- .../pyspec/eth2spec/test/helpers/phase1/shard_block.py | 6 +++--- .../eth2spec/test/phase_1/sanity/test_shard_blocks.py | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py index ea5783655..a339a0e70 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py @@ -41,11 +41,11 @@ def build_empty_shard_block(spec, if slot is None: slot = shard_state.slot - parent_epoch = spec.compute_epoch_of_shard_slot(shard_state.latest_block_header.slot) - if parent_epoch * spec.SLOTS_PER_EPOCH == beacon_state.slot: + epoch = spec.compute_epoch_of_shard_slot(slot) + if epoch * spec.SLOTS_PER_EPOCH == beacon_state.slot: beacon_block_root = spec.signing_root(beacon_state.latest_block_header) else: - beacon_block_root = spec.get_block_root(beacon_state, parent_epoch) + beacon_block_root = spec.get_block_root(beacon_state, epoch) previous_block_header = deepcopy(shard_state.latest_block_header) if previous_block_header.state_root == spec.Hash(): diff --git a/test_libs/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py b/test_libs/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py index 2669ffb53..51575f2d5 100644 --- a/test_libs/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py +++ b/test_libs/pyspec/eth2spec/test/phase_1/sanity/test_shard_blocks.py @@ -154,6 +154,13 @@ def test_empty_shard_period_transition(spec, state): shard_state.newer_committee_negative_deltas[0] = stub_delta slot = shard_state.slot + spec.SHARD_SLOTS_PER_EPOCH * spec.EPOCHS_PER_SHARD_PERIOD + beacon_state.slot = spec.compute_epoch_of_shard_slot(slot) * spec.SLOTS_PER_EPOCH - 4 + spec.process_slots(beacon_state, spec.compute_epoch_of_shard_slot(slot) * spec.SLOTS_PER_EPOCH) + + # all validators get slashed for not revealing keys + # undo this to allow for a block proposal + for index in range(len(beacon_state.validators)): + beacon_state.validators[index].slashed = False block = build_empty_shard_block(spec, beacon_state, shard_state, slot=slot, signed=True) yield 'pre', shard_state From dc4869349c4d0a14a07476863959add418c22ab2 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 5 Oct 2019 06:56:54 +0900 Subject: [PATCH 27/28] remove explicitly handling empty attestation --- specs/core/1_shard-data-chains.md | 5 ----- .../eth2spec/test/helpers/phase1/shard_block.py | 17 ++++++++++------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 2045c7a7e..a31f16880 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -384,11 +384,6 @@ def process_shard_attestations(beacon_state: BeaconState, shard_state: ShardStat pubkeys.append(beacon_state.validators[validator_index].pubkey) process_delta(beacon_state, shard_state, validator_index, get_base_reward(beacon_state, validator_index)) attestation_count += 1 - # Exit early if no participants - if not any(pubkeys): - assert block.attestations == BLSSignature() - return - # Verify there are no extraneous bits set beyond the shard committee for i in range(len(shard_committee), 2 * MAX_PERIOD_COMMITTEE_SIZE): assert block.aggregation_bits[i] == 0b0 diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py index a339a0e70..834adc93f 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py @@ -66,13 +66,16 @@ def build_empty_shard_block(spec, (True,) * len(shard_committee) + (False,) * (spec.MAX_PERIOD_COMMITTEE_SIZE * 2 - len(shard_committee)) ) - block.attestations = sign_shard_attestation( - spec, - beacon_state, - shard_state, - block, - participants=shard_committee, - ) + else: + shard_committee = [] + + block.attestations = sign_shard_attestation( + spec, + beacon_state, + shard_state, + block, + participants=shard_committee, + ) if signed: sign_shard_block(spec, beacon_state, shard_state, block) From 3c3ae9ac02ed896b59b03c95f60778c6f4f22f8b Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 5 Oct 2019 07:22:20 +0900 Subject: [PATCH 28/28] enforce beacon state at epoch boundary slot --- specs/core/1_shard-data-chains.md | 16 ++++++++++------ .../eth2spec/test/helpers/phase1/shard_block.py | 11 +++++------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index a31f16880..69962b6fe 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -339,12 +339,16 @@ def process_shard_block_header(beacon_state: BeaconState, shard_state: ShardStat assert block.slot == shard_state.slot # Verify the beacon chain root epoch = compute_epoch_of_shard_slot(shard_state.slot) - assert epoch == compute_epoch_of_slot(beacon_state.slot) - if epoch * SLOTS_PER_EPOCH == beacon_state.slot: - beacon_block_root = signing_root(beacon_state.latest_block_header) - else: - beacon_block_root = get_block_root(beacon_state, epoch) - assert block.beacon_block_root == beacon_block_root + assert epoch * SLOTS_PER_EPOCH == beacon_state.slot + beacon_block_header = BeaconBlockHeader( + slot=beacon_state.latest_block_header.slot, + parent_root=beacon_state.latest_block_header.parent_root, + state_root=beacon_state.latest_block_header.state_root, + body_root=beacon_state.latest_block_header.body_root, + ) + if beacon_block_header.state_root == Bytes32(): + beacon_block_header.state_root = hash_tree_root(beacon_state) + assert block.beacon_block_root == signing_root(beacon_block_header) # Verify the parent root assert block.parent_root == signing_root(shard_state.latest_block_header) # Save current block as the new latest block diff --git a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py index 834adc93f..7955c613e 100644 --- a/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py +++ b/test_libs/pyspec/eth2spec/test/helpers/phase1/shard_block.py @@ -41,14 +41,13 @@ def build_empty_shard_block(spec, if slot is None: slot = shard_state.slot - epoch = spec.compute_epoch_of_shard_slot(slot) - if epoch * spec.SLOTS_PER_EPOCH == beacon_state.slot: - beacon_block_root = spec.signing_root(beacon_state.latest_block_header) - else: - beacon_block_root = spec.get_block_root(beacon_state, epoch) + previous_beacon_header = deepcopy(beacon_state.latest_block_header) + if previous_beacon_header.state_root == spec.Bytes32(): + previous_beacon_header.state_root = beacon_state.hash_tree_root() + beacon_block_root = spec.signing_root(previous_beacon_header) previous_block_header = deepcopy(shard_state.latest_block_header) - if previous_block_header.state_root == spec.Hash(): + if previous_block_header.state_root == spec.Bytes32(): previous_block_header.state_root = shard_state.hash_tree_root() parent_root = signing_root(previous_block_header)