diff --git a/presets/mainnet/sharding.yaml b/presets/mainnet/sharding.yaml index 9a81c8cdc..2b78855fc 100644 --- a/presets/mainnet/sharding.yaml +++ b/presets/mainnet/sharding.yaml @@ -15,6 +15,8 @@ MAX_SHARD_PROPOSER_SLASHINGS: 16 # Shard block configs # --------------------------------------------------------------- MAX_SHARD_HEADERS_PER_SHARD: 4 +# 2**8 (= 256) +SHARD_STATE_MEMORY_SLOTS: 256 # 2**11 (= 2,048) MAX_SAMPLES_PER_BLOCK: 2048 # 2**10 (= 1,1024) diff --git a/presets/minimal/sharding.yaml b/presets/minimal/sharding.yaml index 7dedbc926..10f79c96e 100644 --- a/presets/minimal/sharding.yaml +++ b/presets/minimal/sharding.yaml @@ -15,6 +15,8 @@ MAX_SHARD_PROPOSER_SLASHINGS: 4 # Shard block configs # --------------------------------------------------------------- MAX_SHARD_HEADERS_PER_SHARD: 4 +# 2**8 (= 256) +SHARD_STATE_MEMORY_SLOTS: 256 # 2**11 (= 2,048) MAX_SAMPLES_PER_BLOCK: 2048 # 2**10 (= 1,1024) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index a15a002e4..98feba22b 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -13,6 +13,7 @@ - [Constants](#constants) - [Misc](#misc) - [Domain types](#domain-types) + - [Shard Work Status](#shard-work-status) - [Preset](#preset) - [Misc](#misc-1) - [Shard block samples](#shard-block-samples) @@ -32,6 +33,7 @@ - [`ShardBlobReference`](#shardblobreference) - [`SignedShardBlobReference`](#signedshardblobreference) - [`ShardProposerSlashing`](#shardproposerslashing) + - [`ShardWork`](#shardwork) - [Helper functions](#helper-functions) - [Misc](#misc-2) - [`next_power_of_two`](#next_power_of_two) @@ -49,14 +51,14 @@ - [`compute_committee_index_from_shard`](#compute_committee_index_from_shard) - [Block processing](#block-processing) - [Operations](#operations) - - [New Attestation processing](#new-attestation-processing) - - [Updated `process_attestation`](#updated-process_attestation) - - [`update_pending_votes`](#update_pending_votes) - - [`process_shard_header`](#process_shard_header) - - [Shard Proposer slashings](#shard-proposer-slashings) + - [Extended Attestation processing](#extended-attestation-processing) + - [`process_shard_header`](#process_shard_header) + - [`process_shard_proposer_slashing`](#process_shard_proposer_slashing) - [Epoch transition](#epoch-transition) - - [Pending headers](#pending-headers) - - [Shard epoch increment](#shard-epoch-increment) + - [`process_pending_shard_confirmations`](#process_pending_shard_confirmations) + - [`charge_confirmed_shard_fees`](#charge_confirmed_shard_fees) + - [`reset_pending_shard_work`](#reset_pending_shard_work) + - [`process_shard_epoch_increment`](#process_shard_epoch_increment) @@ -99,6 +101,14 @@ The following values are (non-configurable) constants used throughout the specif | `DOMAIN_SHARD_PROPOSER` | `DomainType('0x80000000')` | | `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | +### Shard Work Status + +| Name | Value | Notes | +| - | - | - | +| `SHARD_WORK_UNCONFIRMED` | `0` | Unconfirmed, nullified after confirmation time elapses | +| `SHARD_WORK_CONFIRMED` | `1` | Confirmed, reduced to just the commitment | +| `SHARD_WORK_PENDING` | `2` | Pending, a list of competing headers | + ## Preset ### Misc @@ -109,6 +119,7 @@ The following values are (non-configurable) constants used throughout the specif | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `uint64(2**3)` (= 8) | Gasprice may decrease/increase by at most exp(1 / this value) *per epoch* | | `MAX_SHARD_PROPOSER_SLASHINGS` | `2**4` (= 16) | Maximum amount of shard proposer slashing operations per block | | `MAX_SHARD_HEADERS_PER_SHARD` | `4` | | +| `SHARD_STATE_MEMORY_SLOTS` | `uint64(2**8)` (= 256) | Number of slots for which shard commitments and confirmation status is directly available in the state | ### Shard block samples @@ -169,13 +180,12 @@ class BeaconBlockBody(merge.BeaconBlockBody): # [extends The Merge block body] ```python class BeaconState(merge.BeaconState): # [extends The Merge state] - # [Updated fields] + # [Updated fields] (Warning: this changes with Altair, Sharding will rebase to use participation-flags) previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] # [New fields] - previous_epoch_pending_shard_headers: List[PendingShardHeader, MAX_SHARDS * MAX_SHARD_HEADERS_PER_SHARD * SLOTS_PER_EPOCH] - current_epoch_pending_shard_headers: List[PendingShardHeader, MAX_SHARDS * MAX_SHARD_HEADERS_PER_SHARD * SLOTS_PER_EPOCH] - grandparent_epoch_confirmed_commitments: Vector[Vector[DataCommitment, SLOTS_PER_EPOCH], MAX_SHARDS] + # A ring buffer of the latest slots, with information per active shard. + shard_buffer: Vector[List[ShardWork, MAX_SHARDS], SHARD_STATE_MEMORY_SLOTS] shard_gasprice: uint64 current_epoch_start_shard: Shard ``` @@ -234,17 +244,16 @@ class SignedShardBlobHeader(Container): ```python class PendingShardHeader(Container): - # Slot and shard that this header is intended for - slot: Slot - shard: Shard # KZG10 commitment to the data commitment: DataCommitment # hash_tree_root of the ShardHeader (stored so that attestations can be checked against it) root: Root # Who voted for the header votes: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] - # Has this header been confirmed? - confirmed: boolean + # Sum of effective balances of votes + weight: Gwei + # When the header was last updated, as reference for weight accuracy + update_slot: Slot ``` ### `ShardBlobReference` @@ -276,6 +285,18 @@ class ShardProposerSlashing(Container): signed_reference_2: SignedShardBlobReference ``` +### `ShardWork` + +```python +class ShardWork(Container): + # Upon confirmation the data is reduced to just the header. + status: Union[ # See Shard Work Status enum + None, # SHARD_WORK_UNCONFIRMED + DataCommitment, # SHARD_WORK_CONFIRMED + List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD] # SHARD_WORK_PENDING + ] +``` + ## Helper functions ### Misc @@ -449,6 +470,7 @@ def get_start_shard(state: BeaconState, slot: Slot) -> Shard: ```python def compute_shard_from_committee_index(state: BeaconState, slot: Slot, index: CommitteeIndex) -> Shard: active_shards = get_active_shard_count(state, compute_epoch_at_slot(slot)) + assert index < active_shards return Shard((index + get_start_shard(state, slot)) % active_shards) ``` @@ -456,8 +478,11 @@ def compute_shard_from_committee_index(state: BeaconState, slot: Slot, index: Co ```python def compute_committee_index_from_shard(state: BeaconState, slot: Slot, shard: Shard) -> CommitteeIndex: - active_shards = get_active_shard_count(state, compute_epoch_at_slot(slot)) - return CommitteeIndex((active_shards + shard - get_start_shard(state, slot)) % active_shards) + epoch = compute_epoch_at_slot(slot) + active_shards = get_active_shard_count(state, epoch) + index = CommitteeIndex((active_shards + shard - get_start_shard(state, slot)) % active_shards) + assert index < get_committee_count_per_slot(state, epoch) + return index ``` @@ -498,67 +523,74 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: for_ops(body.voluntary_exits, process_voluntary_exit) ``` -### New Attestation processing - -#### Updated `process_attestation` +##### Extended Attestation processing ```python def process_attestation(state: BeaconState, attestation: Attestation) -> None: phase0.process_attestation(state, attestation) - update_pending_votes(state, attestation) + update_pending_shard_work(state, attestation) ``` -#### `update_pending_votes` - ```python -def update_pending_votes(state: BeaconState, attestation: Attestation) -> None: - # Find and update the PendingShardHeader object, invalid block if pending header not in state - if compute_epoch_at_slot(attestation.data.slot) == get_current_epoch(state): - pending_headers = state.current_epoch_pending_shard_headers - else: - pending_headers = state.previous_epoch_pending_shard_headers - +def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> None: attestation_shard = compute_shard_from_committee_index( state, attestation.data.slot, attestation.data.index, ) - pending_header = None - for header in pending_headers: - if ( - header.root == attestation.data.shard_header_root - and header.slot == attestation.data.slot - and header.shard == attestation_shard - ): - pending_header = header - assert pending_header is not None + buffer_index = attestation.data.slot % SHARD_STATE_MEMORY_SLOTS + committee_work = state.shard_buffer[buffer_index][attestation_shard] - for i in range(len(pending_header.votes)): - pending_header.votes[i] = pending_header.votes[i] or attestation.aggregation_bits[i] - - # Check if the PendingShardHeader is eligible for expedited confirmation - # Requirement 1: nothing else confirmed - all_candidates = [ - c for c in pending_headers if - (c.slot, c.shard) == (pending_header.slot, pending_header.shard) - ] - if True in [c.confirmed for c in all_candidates]: + # Skip attestation vote accounting if the header is not pending + if committee_work.status.selector != SHARD_WORK_PENDING: + # TODO In Altair: set participation bit flag, if attestation matches winning header. return - # Requirement 2: >= 2/3 of balance attesting - participants = get_attesting_indices(state, attestation.data, pending_header.votes) - participants_balance = get_total_balance(state, participants) + current_headers: Sequence[PendingShardHeader] = committee_work.status.value + + # Find the corresponding header, abort if it cannot be found + header_index = [header.root for header in current_headers].index(attestation.data.shard_header_root) + + pending_header: PendingShardHeader = current_headers[header_index] full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) - full_committee_balance = get_total_balance(state, set(full_committee)) - if participants_balance * 3 >= full_committee_balance * 2: - pending_header.confirmed = True + + # The weight may be outdated if it is not the initial weight, and from a previous epoch + if pending_header.weight != 0 and compute_epoch_at_slot(pending_header.update_slot) < get_current_epoch(state): + pending_header.weight = sum(state.validators[index].effective_balance for index, bit + in zip(full_committee, pending_header.votes) if bit) + + pending_header.update_slot = state.slot + + full_committee_balance = Gwei(0) + # Update votes bitfield in the state, update weights + for i, bit in enumerate(attestation.aggregation_bits): + weight = state.validators[full_committee[i]].effective_balance + full_committee_balance += weight + if bit: + if not pending_header.votes[i]: + pending_header.weight += weight + pending_header.votes[i] = True + + # Check if the PendingShardHeader is eligible for expedited confirmation, requiring 2/3 of balance attesting + if pending_header.weight * 3 >= full_committee_balance * 2: + # TODO In Altair: set participation bit flag for voters of this early winning header + if pending_header.commitment == DataCommitment(): + # The committee voted to not confirm anything + state.shard_buffer[buffer_index][attestation_shard].change( + selector=SHARD_WORK_UNCONFIRMED, + value=None, + ) + else: + state.shard_buffer[buffer_index][attestation_shard].change( + selector=SHARD_WORK_CONFIRMED, + value=pending_header.commitment, + ) ``` -#### `process_shard_header` +##### `process_shard_header` ```python -def process_shard_header(state: BeaconState, - signed_header: SignedShardBlobHeader) -> None: +def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeader) -> None: header = signed_header.message # Verify the header is not 0, and not from the future. assert Slot(0) < header.slot <= state.slot @@ -570,6 +602,16 @@ def process_shard_header(state: BeaconState, # Verify that the block root matches, # to ensure the header will only be included in this specific Beacon Chain sub-tree. assert header.body_summary.beacon_block_root == get_block_root_at_slot(state, header.slot - 1) + + # Check that this data is still pending + committee_work = state.shard_buffer[header.slot % SHARD_STATE_MEMORY_SLOTS][header.shard] + assert committee_work.status.selector == SHARD_WORK_PENDING + + # Check that this header is not yet in the pending list + current_headers: Sequence[PendingShardHeader] = committee_work.status.value + header_root = hash_tree_root(header) + assert header_root not in [pending_header.root for pending_header in current_headers] + # Verify proposer assert header.proposer_index == get_shard_proposer_index(state, header.slot, header.shard) # Verify signature @@ -585,27 +627,20 @@ def process_shard_header(state: BeaconState, == bls.Pairing(body_summary.commitment.point, G2_SETUP[-body_summary.commitment.length]) ) - # Get the correct pending header list - if header_epoch == get_current_epoch(state): - pending_headers = state.current_epoch_pending_shard_headers - else: - pending_headers = state.previous_epoch_pending_shard_headers - - header_root = hash_tree_root(header) - # Check that this header is not yet in the pending list - assert header_root not in [pending_header.root for pending_header in pending_headers] - - # Include it in the pending list + # Initialize the pending header index = compute_committee_index_from_shard(state, header.slot, header.shard) committee_length = len(get_beacon_committee(state, header.slot, index)) - pending_headers.append(PendingShardHeader( - slot=header.slot, - shard=header.shard, + initial_votes = Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length) + pending_header = PendingShardHeader( commitment=body_summary.commitment, root=header_root, - votes=Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length), - confirmed=False, - )) + votes=initial_votes, + weight=0, + update_slot=state.slot, + ) + + # Include it in the pending list + state.shard_buffer[header.slot % SHARD_STATE_MEMORY_SLOTS][header.shard].append(pending_header) ``` The degree proof works as follows. For a block `B` with length `l` (so `l` values in `[0...l - 1]`, seen as a polynomial `B(X)` which takes these values), @@ -613,7 +648,7 @@ the length proof is the commitment to the polynomial `B(X) * X**(MAX_DEGREE + 1 where `MAX_DEGREE` is the maximum power of `s` available in the setup, which is `MAX_DEGREE = len(G2_SETUP) - 1`. The goal is to ensure that a proof can only be constructed if `deg(B) < l` (there are not hidden higher-order terms in the polynomial, which would thwart reconstruction). -##### Shard Proposer slashings +##### `process_shard_proposer_slashing` ```python def process_shard_proposer_slashing(state: BeaconState, proposer_slashing: ShardProposerSlashing) -> None: @@ -646,19 +681,18 @@ This epoch transition overrides the Merge epoch transition: ```python def process_epoch(state: BeaconState) -> None: + # Sharding + process_pending_shard_confirmations(state) + charge_confirmed_shard_fees(state) + reset_pending_shard_work(state) + + # Phase0 process_justification_and_finalization(state) process_rewards_and_penalties(state) process_registry_updates(state) - process_slashings(state) - # Sharding - process_pending_headers(state) - charge_confirmed_header_fees(state) - reset_pending_headers(state) - # Final updates - # Phase 0 process_eth1_data_reset(state) process_effective_balance_updates(state) process_slashings_reset(state) @@ -669,10 +703,10 @@ def process_epoch(state: BeaconState) -> None: process_shard_epoch_increment(state) ``` -#### Pending headers +#### `process_pending_shard_confirmations` ```python -def process_pending_headers(state: BeaconState) -> None: +def process_pending_shard_confirmations(state: BeaconState) -> None: # Pending header processing applies to the previous epoch. # Skip if `GENESIS_EPOCH` because no prior epoch to process. if get_current_epoch(state) == GENESIS_EPOCH: @@ -680,108 +714,93 @@ def process_pending_headers(state: BeaconState) -> None: previous_epoch = get_previous_epoch(state) previous_epoch_start_slot = compute_start_slot_at_epoch(previous_epoch) - for slot in range(previous_epoch_start_slot, previous_epoch_start_slot + SLOTS_PER_EPOCH): - for shard_index in range(get_active_shard_count(state, previous_epoch)): - shard = Shard(shard_index) - # Pending headers for this (slot, shard) combo - candidates = [ - c for c in state.previous_epoch_pending_shard_headers - if (c.slot, c.shard) == (slot, shard) - ] - # If any candidates already confirmed, skip - if True in [c.confirmed for c in candidates]: - continue - # The entire committee (and its balance) - index = compute_committee_index_from_shard(state, slot, shard) - full_committee = get_beacon_committee(state, slot, index) - # The set of voters who voted for each header (and their total balances) - voting_sets = [ - set(v for i, v in enumerate(full_committee) if c.votes[i]) - for c in candidates - ] - voting_balances = [ - get_total_balance(state, voters) - for voters in voting_sets - ] - # Get the index with the most total balance voting for them. - # NOTE: if two choices get exactly the same voting balance, - # the candidate earlier in the list wins - if max(voting_balances) > 0: - winning_index = voting_balances.index(max(voting_balances)) - else: - # If no votes, zero wins - winning_index = [c.root for c in candidates].index(Root()) - candidates[winning_index].confirmed = True - for slot_index in range(SLOTS_PER_EPOCH): - for shard in range(MAX_SHARDS): - state.grandparent_epoch_confirmed_commitments[shard][slot_index] = DataCommitment() - confirmed_headers = [candidate for candidate in state.previous_epoch_pending_shard_headers if candidate.confirmed] - for header in confirmed_headers: - state.grandparent_epoch_confirmed_commitments[header.shard][header.slot % SLOTS_PER_EPOCH] = header.commitment + # Mark stale headers as unconfirmed + for slot in range(previous_epoch_start_slot, previous_epoch_start_slot + SLOTS_PER_EPOCH): + buffer_index = slot % SHARD_STATE_MEMORY_SLOTS + for shard_index in range(len(state.shard_buffer[buffer_index])): + committee_work = state.shard_buffer[buffer_index][shard_index] + if committee_work.status.selector == SHARD_WORK_PENDING: + winning_header = max(committee_work.status.value, key=lambda header: header.weight) + # TODO In Altair: set participation bit flag of voters for winning header + if winning_header.commitment == DataCommitment(): + committee_work.status.change(selector=SHARD_WORK_UNCONFIRMED, value=None) + else: + committee_work.status.change(selector=SHARD_WORK_CONFIRMED, value=winning_header.commitment) ``` +#### `charge_confirmed_shard_fees` + ```python -def charge_confirmed_header_fees(state: BeaconState) -> None: +def charge_confirmed_shard_fees(state: BeaconState) -> None: new_gasprice = state.shard_gasprice previous_epoch = get_previous_epoch(state) + previous_epoch_start_slot = compute_start_slot_at_epoch(previous_epoch) adjustment_quotient = ( get_active_shard_count(state, previous_epoch) * SLOTS_PER_EPOCH * GASPRICE_ADJUSTMENT_COEFFICIENT ) - previous_epoch_start_slot = compute_start_slot_at_epoch(previous_epoch) + # Iterate through confirmed shard-headers for slot in range(previous_epoch_start_slot, previous_epoch_start_slot + SLOTS_PER_EPOCH): - for shard_index in range(get_active_shard_count(state, previous_epoch)): - shard = Shard(shard_index) - confirmed_candidates = [ - c for c in state.previous_epoch_pending_shard_headers - if (c.slot, c.shard, c.confirmed) == (slot, shard, True) - ] - if not any(confirmed_candidates): - continue - candidate = confirmed_candidates[0] + buffer_index = slot % SHARD_STATE_MEMORY_SLOTS + for shard_index in range(len(state.shard_buffer[buffer_index])): + committee_work = state.shard_buffer[buffer_index][shard_index] + if committee_work.status.selector == SHARD_WORK_CONFIRMED: + commitment: DataCommitment = committee_work.status.value + # Charge EIP 1559 fee + proposer = get_shard_proposer_index(state, slot, Shard(shard_index)) + fee = ( + (state.shard_gasprice * commitment.length) + // TARGET_SAMPLES_PER_BLOCK + ) + decrease_balance(state, proposer, fee) - # Charge EIP 1559 fee - proposer = get_shard_proposer_index(state, slot, shard) - fee = ( - (state.shard_gasprice * candidate.commitment.length) - // TARGET_SAMPLES_PER_BLOCK - ) - decrease_balance(state, proposer, fee) - - # Track updated gas price - new_gasprice = compute_updated_gasprice( - new_gasprice, - candidate.commitment.length, - adjustment_quotient, - ) + # Track updated gas price + new_gasprice = compute_updated_gasprice( + new_gasprice, + commitment.length, + adjustment_quotient, + ) state.shard_gasprice = new_gasprice ``` +#### `reset_pending_shard_work` + ```python -def reset_pending_headers(state: BeaconState) -> None: - state.previous_epoch_pending_shard_headers = state.current_epoch_pending_shard_headers - state.current_epoch_pending_shard_headers = [] - # Add dummy "empty" PendingShardHeader (default vote for if no shard header available) +def reset_pending_shard_work(state: BeaconState) -> None: + # Add dummy "empty" PendingShardHeader (default vote if no shard header is available) next_epoch = get_current_epoch(state) + 1 next_epoch_start_slot = compute_start_slot_at_epoch(next_epoch) committees_per_slot = get_committee_count_per_slot(state, next_epoch) + active_shards = get_active_shard_count(state, next_epoch) + for slot in range(next_epoch_start_slot, next_epoch_start_slot + SLOTS_PER_EPOCH): - for index in range(committees_per_slot): - committee_index = CommitteeIndex(index) - shard = compute_shard_from_committee_index(state, slot, committee_index) + buffer_index = slot % SHARD_STATE_MEMORY_SLOTS + + # Reset the shard work tracking + state.shard_buffer[buffer_index] = [ShardWork() for _ in range(active_shards)] + + start_shard = get_start_shard(state, slot) + for committee_index in range(committees_per_slot): + shard = (start_shard + committee_index) % active_shards + # a committee is available, initialize a pending shard-header list committee_length = len(get_beacon_committee(state, slot, committee_index)) - state.current_epoch_pending_shard_headers.append(PendingShardHeader( - slot=slot, - shard=shard, - commitment=DataCommitment(), - root=Root(), - votes=Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length), - confirmed=False, - )) + state.shard_buffer[buffer_index][shard].change( + selector=SHARD_WORK_PENDING, + value=List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD]( + PendingShardHeader( + commitment=DataCommitment(), + root=Root(), + votes=Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length), + weight=0, + update_slot=slot, + ) + ) + ) + # a shard without committee available defaults to SHARD_WORK_UNCONFIRMED. ``` -#### Shard epoch increment +#### `process_shard_epoch_increment` ```python def process_shard_epoch_increment(state: BeaconState) -> None: