From e5521af1f8186d2a624948bdbaa753181695e92e Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 May 2021 20:18:29 +0200 Subject: [PATCH 01/12] new union-based shard headers/commitments representation --- specs/sharding/beacon-chain.md | 247 +++++++++++++++------------------ 1 file changed, 115 insertions(+), 132 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 5522e044d..38007e478 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -13,6 +13,7 @@ - [Constants](#constants) - [Misc](#misc) - [Domain types](#domain-types) + - [Shard Header Status](#shard-header-status) - [Preset](#preset) - [Misc](#misc-1) - [Shard block samples](#shard-block-samples) @@ -99,6 +100,14 @@ The following values are (non-configurable) constants used throughout the specif | `DOMAIN_SHARD_PROPOSER` | `DomainType('0x80000000')` | | `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | +### Shard Header Status + +| Name | Value | Notes | +| - | - | - | +| `UNCONFIRMED_SHARD_DATA` | `0` | Unconfirmed, nullified after confirmation time elapses | +| `CONFIRMED_SHARD_DATA` | `1` | Confirmed, reduced to just the commitment | +| `PENDING_SHARD_DATA` | `2` | Pending, a list of competing headers | + ## Preset ### Misc @@ -109,6 +118,7 @@ The following values are (non-configurable) constants used throughout the specif | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `uint64(2**3)` (= 8) | Gasprice may decrease/increase by at most exp(1 / this value) *per epoch* | | `MAX_SHARD_PROPOSER_SLASHINGS` | `2**4` (= 16) | Maximum amount of shard proposer slashing operations per block | | `MAX_SHARD_HEADERS_PER_SHARD` | `4` | | +| `SHARD_STATE_MEMORY_SLOTS` | `uint64(2**8)` (=256) | Number of slots for which shard commitments and confirmation status is directly available in the state | ### Shard block samples @@ -169,13 +179,21 @@ class BeaconBlockBody(merge.BeaconBlockBody): # [extends The Merge block body] ```python class BeaconState(merge.BeaconState): # [extends The Merge state] - # [Updated fields] + # [Updated fields] (Warning: this changes with Altair, Sharding will rebase to use participation-flags) previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] # [New fields] - previous_epoch_pending_shard_headers: List[PendingShardHeader, MAX_SHARDS * MAX_SHARD_HEADERS_PER_SHARD * SLOTS_PER_EPOCH] - current_epoch_pending_shard_headers: List[PendingShardHeader, MAX_SHARDS * MAX_SHARD_HEADERS_PER_SHARD * SLOTS_PER_EPOCH] - grandparent_epoch_confirmed_commitments: Vector[Vector[DataCommitment, SLOTS_PER_EPOCH], MAX_SHARDS] + # A ring buffer of the latest shard headers per slot. Upon confirmation the data is reduced to just the header. + shard_headers: Vector[ + List[ + Union[ # See Shard Header Status enum + None, # UNCONFIRMED_SHARD_DATA + DataCommitment, # CONFIRMED_SHARD_DATA + List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD] # PENDING_SHARD_DATA + ], + MAX_SHARDS + ], + SHARD_STATE_MEMORY_SLOTS] shard_gasprice: uint64 current_epoch_start_shard: Shard ``` @@ -233,17 +251,12 @@ class SignedShardBlobHeader(Container): ```python class PendingShardHeader(Container): - # Slot and shard that this header is intended for - slot: Slot - shard: Shard # KZG10 commitment to the data commitment: DataCommitment # hash_tree_root of the ShardHeader (stored so that attestations can be checked against it) root: Root # Who voted for the header votes: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] - # Has this header been confirmed? - confirmed: boolean ``` ### `ShardBlobReference` @@ -346,7 +359,7 @@ def get_active_shard_count(state: BeaconState, epoch: Epoch) -> uint64: Return the number of active shards. Note that this puts an upper bound on the number of committees per slot. """ - return INITIAL_ACTIVE_SHARDS + return INITIAL_ACTIVE_SHARDS # TODO: use shard_headers from state instead? ``` #### `get_shard_committee` @@ -511,53 +524,46 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: ```python def update_pending_votes(state: BeaconState, attestation: Attestation) -> None: - # Find and update the PendingShardHeader object, invalid block if pending header not in state - if compute_epoch_at_slot(attestation.data.slot) == get_current_epoch(state): - pending_headers = state.current_epoch_pending_shard_headers - else: - pending_headers = state.previous_epoch_pending_shard_headers - attestation_shard = compute_shard_from_committee_index( state, attestation.data.slot, attestation.data.index, ) - pending_header = None - for header in pending_headers: - if ( - header.root == attestation.data.shard_header_root - and header.slot == attestation.data.slot - and header.shard == attestation_shard - ): - pending_header = header - assert pending_header is not None + buffer_index = attestation.data.slot % SHARD_STATE_MEMORY_SLOTS + shard_header_status = state.shard_headers[buffer_index][attestation_shard] - for i in range(len(pending_header.votes)): - pending_header.votes[i] = pending_header.votes[i] or attestation.aggregation_bits[i] - - # Check if the PendingShardHeader is eligible for expedited confirmation - # Requirement 1: nothing else confirmed - all_candidates = [ - c for c in pending_headers if - (c.slot, c.shard) == (pending_header.slot, pending_header.shard) - ] - if True in [c.confirmed for c in all_candidates]: + # Skip attestation vote accounting if the header is already confirmed + if shard_header_status.selector == CONFIRMED_SHARD_DATA: return - # Requirement 2: >= 2/3 of balance attesting + assert shard_header_status.selector == PENDING_SHARD_DATA + current_headers: Sequence[PendingShardHeader] = shard_header_status.value + + # Find the corresponding header, abort if it cannot be found + header_index = [header.root for header in current_headers].index(attestation.data.shard_header_root) + + # Update votes bitfield in the state + pending_header = state.shard_headers[buffer_index][attestation_shard][header_index] + for i, bit in enumerate(attestation.aggregation_bits): + if bit: + pending_header.votes[i] = True + + # Check if the PendingShardHeader is eligible for expedited confirmation, requiring 2/3 of balance attesting participants = get_attesting_indices(state, attestation.data, pending_header.votes) participants_balance = get_total_balance(state, participants) full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) full_committee_balance = get_total_balance(state, set(full_committee)) if participants_balance * 3 >= full_committee_balance * 2: - pending_header.confirmed = True + state.shard_headers[buffer_index][attestation_shard].change( + selector=CONFIRMED_SHARD_DATA, + value=pending_header.commitment, + ) ``` #### `process_shard_header` ```python -def process_shard_header(state: BeaconState, - signed_header: SignedShardBlobHeader) -> None: +def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeader) -> None: header = signed_header.message # Verify the header is not 0, and not from the future. assert Slot(0) < header.slot <= state.slot @@ -569,6 +575,15 @@ def process_shard_header(state: BeaconState, # Verify that the block root matches, # to ensure the header will only be included in this specific Beacon Chain sub-tree. assert header.body_summary.beacon_block_root == get_block_root_at_slot(state, header.slot - 1) + + # Check that this data is still pending + shard_header_status = state.shard_headers[header.slot % SHARD_STATE_MEMORY_SLOTS][header.slot] + assert shard_header_status.selector == PENDING_SHARD_DATA + + # Check that this header is not yet in the pending list + current_headers: Sequence[PendingShardHeader] = shard_header_status.value + assert header_root not in [pending_header.root for pending_header in current_headers] + # Verify proposer assert header.proposer_index == get_shard_proposer_index(state, header.slot, header.shard) # Verify signature @@ -584,27 +599,18 @@ def process_shard_header(state: BeaconState, == bls.Pairing(body_summary.commitment.point, G2_SETUP[-body_summary.commitment.length]) ) - # Get the correct pending header list - if header_epoch == get_current_epoch(state): - pending_headers = state.current_epoch_pending_shard_headers - else: - pending_headers = state.previous_epoch_pending_shard_headers - - header_root = hash_tree_root(header) - # Check that this header is not yet in the pending list - assert header_root not in [pending_header.root for pending_header in pending_headers] - - # Include it in the pending list + # Initialize the pending header index = compute_committee_index_from_shard(state, header.slot, header.shard) committee_length = len(get_beacon_committee(state, header.slot, index)) - pending_headers.append(PendingShardHeader( - slot=header.slot, - shard=header.shard, + initial_votes = Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length) + pending_header = PendingShardHeader( commitment=body_summary.commitment, root=header_root, - votes=Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length), - confirmed=False, - )) + votes=initial_votes, + ) + + # Include it in the pending list + state.shard_headers[header.slot % SHARD_STATE_MEMORY_SLOTS][header.slot].append(pending_header) ``` The degree proof works as follows. For a block `B` with length `l` (so `l` values in `[0...l - 1]`, seen as a polynomial `B(X)` which takes these values), @@ -679,105 +685,82 @@ def process_pending_headers(state: BeaconState) -> None: previous_epoch = get_previous_epoch(state) previous_epoch_start_slot = compute_start_slot_at_epoch(previous_epoch) - for slot in range(previous_epoch_start_slot, previous_epoch_start_slot + SLOTS_PER_EPOCH): - for shard_index in range(get_active_shard_count(state, previous_epoch)): - shard = Shard(shard_index) - # Pending headers for this (slot, shard) combo - candidates = [ - c for c in state.previous_epoch_pending_shard_headers - if (c.slot, c.shard) == (slot, shard) - ] - # If any candidates already confirmed, skip - if True in [c.confirmed for c in candidates]: - continue - # The entire committee (and its balance) - index = compute_committee_index_from_shard(state, slot, shard) - full_committee = get_beacon_committee(state, slot, index) - # The set of voters who voted for each header (and their total balances) - voting_sets = [ - set(v for i, v in enumerate(full_committee) if c.votes[i]) - for c in candidates - ] - voting_balances = [ - get_total_balance(state, voters) - for voters in voting_sets - ] - # Get the index with the most total balance voting for them. - # NOTE: if two choices get exactly the same voting balance, - # the candidate earlier in the list wins - if max(voting_balances) > 0: - winning_index = voting_balances.index(max(voting_balances)) - else: - # If no votes, zero wins - winning_index = [c.root for c in candidates].index(Root()) - candidates[winning_index].confirmed = True - for slot_index in range(SLOTS_PER_EPOCH): - for shard in range(MAX_SHARDS): - state.grandparent_epoch_confirmed_commitments[shard][slot_index] = DataCommitment() - confirmed_headers = [candidate for candidate in state.previous_epoch_pending_shard_headers if candidate.confirmed] - for header in confirmed_headers: - state.grandparent_epoch_confirmed_commitments[header.shard][header.slot % SLOTS_PER_EPOCH] = header.commitment + # Mark stale headers as unconfirmed + for slot in range(previous_epoch_start_slot, previous_epoch_start_slot + SLOTS_PER_EPOCH): + buffer_index = slot % SHARD_STATE_MEMORY_SLOTS + for shard_index in range(len(state.shard_headers[buffer_index])): + if state.shard_headers[buffer_index][shard_index].selector == PENDING_SHARD_DATA: + state.shard_headers[buffer_index][shard_index].change(selector=UNCONFIRMED_SHARD_DATA, value=None) ``` ```python def charge_confirmed_header_fees(state: BeaconState) -> None: new_gasprice = state.shard_gasprice previous_epoch = get_previous_epoch(state) + previous_epoch_start_slot = compute_start_slot_at_epoch(previous_epoch) adjustment_quotient = ( get_active_shard_count(state, previous_epoch) * SLOTS_PER_EPOCH * GASPRICE_ADJUSTMENT_COEFFICIENT ) - previous_epoch_start_slot = compute_start_slot_at_epoch(previous_epoch) + # Iterate through confirmed shard-headers for slot in range(previous_epoch_start_slot, previous_epoch_start_slot + SLOTS_PER_EPOCH): - for shard_index in range(get_active_shard_count(state, previous_epoch)): - shard = Shard(shard_index) - confirmed_candidates = [ - c for c in state.previous_epoch_pending_shard_headers - if (c.slot, c.shard, c.confirmed) == (slot, shard, True) - ] - if not any(confirmed_candidates): - continue - candidate = confirmed_candidates[0] + buffer_index = slot % SHARD_STATE_MEMORY_SLOTS + for shard_index in range(len(state.shard_headers[buffer_index])): + shard_header_status = state.shard_headers[buffer_index][shard_index] + if shard_header_status.selector == CONFIRMED_SHARD_DATA: + # Charge EIP 1559 fee + proposer = get_shard_proposer_index(state, slot, Shard(shard_index)) + fee = ( + (state.shard_gasprice * candidate.commitment.length) + // TARGET_SAMPLES_PER_BLOCK + ) + decrease_balance(state, proposer, fee) - # Charge EIP 1559 fee - proposer = get_shard_proposer_index(state, slot, shard) - fee = ( - (state.shard_gasprice * candidate.commitment.length) - // TARGET_SAMPLES_PER_BLOCK - ) - decrease_balance(state, proposer, fee) - - # Track updated gas price - new_gasprice = compute_updated_gasprice( - new_gasprice, - candidate.commitment.length, - adjustment_quotient, - ) + # Track updated gas price + new_gasprice = compute_updated_gasprice( + new_gasprice, + candidate.commitment.length, + adjustment_quotient, + ) state.shard_gasprice = new_gasprice ``` ```python def reset_pending_headers(state: BeaconState) -> None: - state.previous_epoch_pending_shard_headers = state.current_epoch_pending_shard_headers - state.current_epoch_pending_shard_headers = [] # Add dummy "empty" PendingShardHeader (default vote for if no shard header available) next_epoch = get_current_epoch(state) + 1 next_epoch_start_slot = compute_start_slot_at_epoch(next_epoch) committees_per_slot = get_committee_count_per_slot(state, next_epoch) + active_shards = get_active_shard_count(state, next_epoch) + for slot in range(next_epoch_start_slot, next_epoch_start_slot + SLOTS_PER_EPOCH): - for index in range(committees_per_slot): - committee_index = CommitteeIndex(index) - shard = compute_shard_from_committee_index(state, slot, committee_index) - committee_length = len(get_beacon_committee(state, slot, committee_index)) - state.current_epoch_pending_shard_headers.append(PendingShardHeader( - slot=slot, - shard=shard, - commitment=DataCommitment(), - root=Root(), - votes=Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length), - confirmed=False, - )) + buffer_index = slot % SHARD_STATE_MEMORY_SLOTS + if len(state.shard_headers[buffer_index]) < active_shards: + state.shard_headers[buffer_index].extend() + + start_shard = get_start_shard(state, slot) + for shard_index in range(state.shard_headers[buffer_index]): + if start_shard <= shard_index < start_shard + committees_per_slot: + # a committee is available, initialize a pending shard-header list + committee_index = CommitteeIndex(shard_index - start_shard) + committee_length = len(get_beacon_committee(state, slot, committee_index)) + state.shard_headers[buffer_index][shard_index].change( + selector=PENDING_SHARD_DATA, + value=List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD]( + PendingShardHeader( + commitment=DataCommitment(), + root=Root(), + votes=Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length), + ) + ) + ) + else: + # shard is inactive, no committee available. + state.shard_headers[buffer_index][shard_index].change( + selector=UNCONFIRMED_SHARD_DATA, + value=None, + ) ``` #### Shard epoch increment From 38a0f4f21144b5e56179eb39c2887de22089c928 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 May 2021 21:26:06 +0200 Subject: [PATCH 02/12] update to wrap the union, clean up initialization and typing --- specs/sharding/beacon-chain.md | 102 ++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 45 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 38007e478..ba3a5ab50 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -33,6 +33,7 @@ - [`ShardBlobReference`](#shardblobreference) - [`SignedShardBlobReference`](#signedshardblobreference) - [`ShardProposerSlashing`](#shardproposerslashing) + - [`ShardCommitteeWork`](#shardcommitteework) - [Helper functions](#helper-functions) - [Misc](#misc-2) - [`next_power_of_two`](#next_power_of_two) @@ -183,17 +184,8 @@ class BeaconState(merge.BeaconState): # [extends The Merge state] previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] # [New fields] - # A ring buffer of the latest shard headers per slot. Upon confirmation the data is reduced to just the header. - shard_headers: Vector[ - List[ - Union[ # See Shard Header Status enum - None, # UNCONFIRMED_SHARD_DATA - DataCommitment, # CONFIRMED_SHARD_DATA - List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD] # PENDING_SHARD_DATA - ], - MAX_SHARDS - ], - SHARD_STATE_MEMORY_SLOTS] + # A ring buffer of the latest slots, with information per active shard. + shard_buffer: Vector[List[ShardCommitteeWork, MAX_SHARDS], SHARD_STATE_MEMORY_SLOTS] shard_gasprice: uint64 current_epoch_start_shard: Shard ``` @@ -288,6 +280,18 @@ class ShardProposerSlashing(Container): signed_reference_2: SignedShardBlobReference ``` +### `ShardCommitteeWork` + +```python +class ShardWork(Container): + # Upon confirmation the data is reduced to just the header. + status: Union[ # See Shard Header Status enum + None, # UNCONFIRMED_SHARD_DATA + DataCommitment, # CONFIRMED_SHARD_DATA + List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD] # PENDING_SHARD_DATA + ] +``` + ## Helper functions ### Misc @@ -359,7 +363,7 @@ def get_active_shard_count(state: BeaconState, epoch: Epoch) -> uint64: Return the number of active shards. Note that this puts an upper bound on the number of committees per slot. """ - return INITIAL_ACTIVE_SHARDS # TODO: use shard_headers from state instead? + return INITIAL_ACTIVE_SHARDS ``` #### `get_shard_committee` @@ -461,6 +465,7 @@ def get_start_shard(state: BeaconState, slot: Slot) -> Shard: ```python def compute_shard_from_committee_index(state: BeaconState, slot: Slot, index: CommitteeIndex) -> Shard: active_shards = get_active_shard_count(state, compute_epoch_at_slot(slot)) + assert index < active_shards return Shard((index + get_start_shard(state, slot)) % active_shards) ``` @@ -468,8 +473,11 @@ def compute_shard_from_committee_index(state: BeaconState, slot: Slot, index: Co ```python def compute_committee_index_from_shard(state: BeaconState, slot: Slot, shard: Shard) -> CommitteeIndex: - active_shards = get_active_shard_count(state, compute_epoch_at_slot(slot)) - return CommitteeIndex((active_shards + shard - get_start_shard(state, slot)) % active_shards) + epoch = compute_epoch_at_slot(slot) + active_shards = get_active_shard_count(state, epoch) + index = CommitteeIndex((active_shards + shard - get_start_shard(state, slot)) % active_shards) + assert index >= get_committee_count_per_slot(state, epoch) + return index ``` @@ -530,20 +538,21 @@ def update_pending_votes(state: BeaconState, attestation: Attestation) -> None: attestation.data.index, ) buffer_index = attestation.data.slot % SHARD_STATE_MEMORY_SLOTS - shard_header_status = state.shard_headers[buffer_index][attestation_shard] + committee_work = state.shard_buffer[buffer_index][attestation_shard] # Skip attestation vote accounting if the header is already confirmed - if shard_header_status.selector == CONFIRMED_SHARD_DATA: + if committee_work.status.selector == CONFIRMED_SHARD_DATA: return - assert shard_header_status.selector == PENDING_SHARD_DATA - current_headers: Sequence[PendingShardHeader] = shard_header_status.value + # Note that shard-slot combinations without an assigned committee do not have a pending state + assert shard_info.status.selector == PENDING_SHARD_DATA + current_headers: Sequence[PendingShardHeader] = committee_work.status.value # Find the corresponding header, abort if it cannot be found header_index = [header.root for header in current_headers].index(attestation.data.shard_header_root) # Update votes bitfield in the state - pending_header = state.shard_headers[buffer_index][attestation_shard][header_index] + pending_header: PendingShardHeader = state.shard_buffer[buffer_index][attestation_shard][header_index] for i, bit in enumerate(attestation.aggregation_bits): if bit: pending_header.votes[i] = True @@ -554,10 +563,17 @@ def update_pending_votes(state: BeaconState, attestation: Attestation) -> None: full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) full_committee_balance = get_total_balance(state, set(full_committee)) if participants_balance * 3 >= full_committee_balance * 2: - state.shard_headers[buffer_index][attestation_shard].change( - selector=CONFIRMED_SHARD_DATA, - value=pending_header.commitment, - ) + if pending_header.commitment == DataCommitment(): + # The committee voted to not confirm anything + state.shard_buffer[buffer_index][attestation_shard].change( + selector=UNCONFIRMED_SHARD_DATA, + value=None, + ) + else: + state.shard_buffer[buffer_index][attestation_shard].change( + selector=CONFIRMED_SHARD_DATA, + value=pending_header.commitment, + ) ``` #### `process_shard_header` @@ -577,11 +593,11 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade assert header.body_summary.beacon_block_root == get_block_root_at_slot(state, header.slot - 1) # Check that this data is still pending - shard_header_status = state.shard_headers[header.slot % SHARD_STATE_MEMORY_SLOTS][header.slot] - assert shard_header_status.selector == PENDING_SHARD_DATA + committee_work = state.shard_buffer[header.slot % SHARD_STATE_MEMORY_SLOTS][header.slot] + assert committee_work.status.selector == PENDING_SHARD_DATA # Check that this header is not yet in the pending list - current_headers: Sequence[PendingShardHeader] = shard_header_status.value + current_headers: Sequence[PendingShardHeader] = committee_work.status.value assert header_root not in [pending_header.root for pending_header in current_headers] # Verify proposer @@ -610,7 +626,7 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade ) # Include it in the pending list - state.shard_headers[header.slot % SHARD_STATE_MEMORY_SLOTS][header.slot].append(pending_header) + state.shard_buffer[header.slot % SHARD_STATE_MEMORY_SLOTS][header.slot].append(pending_header) ``` The degree proof works as follows. For a block `B` with length `l` (so `l` values in `[0...l - 1]`, seen as a polynomial `B(X)` which takes these values), @@ -689,9 +705,9 @@ def process_pending_headers(state: BeaconState) -> None: # Mark stale headers as unconfirmed for slot in range(previous_epoch_start_slot, previous_epoch_start_slot + SLOTS_PER_EPOCH): buffer_index = slot % SHARD_STATE_MEMORY_SLOTS - for shard_index in range(len(state.shard_headers[buffer_index])): - if state.shard_headers[buffer_index][shard_index].selector == PENDING_SHARD_DATA: - state.shard_headers[buffer_index][shard_index].change(selector=UNCONFIRMED_SHARD_DATA, value=None) + for shard_index in range(len(state.shard_buffer[buffer_index])): + if state.shard_buffer[buffer_index][shard_index].selector == PENDING_SHARD_DATA: + state.shard_buffer[buffer_index][shard_index].change(selector=UNCONFIRMED_SHARD_DATA, value=None) ``` ```python @@ -706,9 +722,9 @@ def charge_confirmed_header_fees(state: BeaconState) -> None: # Iterate through confirmed shard-headers for slot in range(previous_epoch_start_slot, previous_epoch_start_slot + SLOTS_PER_EPOCH): buffer_index = slot % SHARD_STATE_MEMORY_SLOTS - for shard_index in range(len(state.shard_headers[buffer_index])): - shard_header_status = state.shard_headers[buffer_index][shard_index] - if shard_header_status.selector == CONFIRMED_SHARD_DATA: + for shard_index in range(len(state.shard_buffer[buffer_index])): + committee_work = state.shard_buffer[buffer_index][shard_index] + if committee_work.status.selector == CONFIRMED_SHARD_DATA: # Charge EIP 1559 fee proposer = get_shard_proposer_index(state, slot, Shard(shard_index)) fee = ( @@ -728,24 +744,25 @@ def charge_confirmed_header_fees(state: BeaconState) -> None: ```python def reset_pending_headers(state: BeaconState) -> None: - # Add dummy "empty" PendingShardHeader (default vote for if no shard header available) + # Add dummy "empty" PendingShardHeader (default vote if no shard header is available) next_epoch = get_current_epoch(state) + 1 next_epoch_start_slot = compute_start_slot_at_epoch(next_epoch) committees_per_slot = get_committee_count_per_slot(state, next_epoch) active_shards = get_active_shard_count(state, next_epoch) - + for slot in range(next_epoch_start_slot, next_epoch_start_slot + SLOTS_PER_EPOCH): buffer_index = slot % SHARD_STATE_MEMORY_SLOTS - if len(state.shard_headers[buffer_index]) < active_shards: - state.shard_headers[buffer_index].extend() + + # Reset the shard work tracking + state.shard_buffer[buffer_index] = [ShardCommitteeWork() for _ in range(active_shards)] start_shard = get_start_shard(state, slot) - for shard_index in range(state.shard_headers[buffer_index]): + for shard_index in range(state.shard_buffer[buffer_index]): if start_shard <= shard_index < start_shard + committees_per_slot: # a committee is available, initialize a pending shard-header list committee_index = CommitteeIndex(shard_index - start_shard) committee_length = len(get_beacon_committee(state, slot, committee_index)) - state.shard_headers[buffer_index][shard_index].change( + state.shard_buffer[buffer_index][shard_index].change( selector=PENDING_SHARD_DATA, value=List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD]( PendingShardHeader( @@ -755,12 +772,7 @@ def reset_pending_headers(state: BeaconState) -> None: ) ) ) - else: - # shard is inactive, no committee available. - state.shard_headers[buffer_index][shard_index].change( - selector=UNCONFIRMED_SHARD_DATA, - value=None, - ) + # the shard is inactive for this slot otherwise, no committee available, default to UNCONFIRMED_SHARD_DATA. ``` #### Shard epoch increment From 72215412fa1a89f5155e8b47e8ac479d21b2d686 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 28 May 2021 21:56:13 +0200 Subject: [PATCH 03/12] For slow shard confirmation, assuming it is available via DAS, we could confirm the best pending header --- specs/sharding/beacon-chain.md | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index ba3a5ab50..ba6da197f 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -249,6 +249,8 @@ class PendingShardHeader(Container): root: Root # Who voted for the header votes: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] + # Sum of effective balances of votes + weight: Gwei ``` ### `ShardBlobReference` @@ -552,15 +554,18 @@ def update_pending_votes(state: BeaconState, attestation: Attestation) -> None: header_index = [header.root for header in current_headers].index(attestation.data.shard_header_root) # Update votes bitfield in the state - pending_header: PendingShardHeader = state.shard_buffer[buffer_index][attestation_shard][header_index] + pending_header: PendingShardHeader = state.shard_buffer[buffer_index][attestation_shard][header_index] + full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) + participants_balance = Gwei(0) for i, bit in enumerate(attestation.aggregation_bits): + weight = state.validators[full_committee[i]].effective_balance if bit: + if not pending_header.votes[i]: + pending_header.weight += weight pending_header.votes[i] = True + participants_balance += weight # Check if the PendingShardHeader is eligible for expedited confirmation, requiring 2/3 of balance attesting - participants = get_attesting_indices(state, attestation.data, pending_header.votes) - participants_balance = get_total_balance(state, participants) - full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) full_committee_balance = get_total_balance(state, set(full_committee)) if participants_balance * 3 >= full_committee_balance * 2: if pending_header.commitment == DataCommitment(): @@ -623,6 +628,7 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade commitment=body_summary.commitment, root=header_root, votes=initial_votes, + weight=0, ) # Include it in the pending list @@ -706,8 +712,13 @@ def process_pending_headers(state: BeaconState) -> None: for slot in range(previous_epoch_start_slot, previous_epoch_start_slot + SLOTS_PER_EPOCH): buffer_index = slot % SHARD_STATE_MEMORY_SLOTS for shard_index in range(len(state.shard_buffer[buffer_index])): - if state.shard_buffer[buffer_index][shard_index].selector == PENDING_SHARD_DATA: - state.shard_buffer[buffer_index][shard_index].change(selector=UNCONFIRMED_SHARD_DATA, value=None) + committee_work = state.shard_buffer[buffer_index][shard_index] + if committee_work.selector == PENDING_SHARD_DATA: + winning_header = max(committee_work.value, key=lambda header: header.weight) + if winning_header.commitment == DataCommitment(): + committee_work.change(selector=UNCONFIRMED_SHARD_DATA, value=None) + else: + committee_work.change(selector=CONFIRMED_SHARD_DATA, value=winning_header.commitment) ``` ```python @@ -769,6 +780,7 @@ def reset_pending_headers(state: BeaconState) -> None: commitment=DataCommitment(), root=Root(), votes=Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length), + weight=0, ) ) ) From 3665dbea8b65d08da4953ac5fc26f7f39e77f746 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 May 2021 21:28:00 +0200 Subject: [PATCH 04/12] name (slot, shard) union wrapper ShardWork --- specs/sharding/beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index ba6da197f..d5e43623b 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -33,7 +33,7 @@ - [`ShardBlobReference`](#shardblobreference) - [`SignedShardBlobReference`](#signedshardblobreference) - [`ShardProposerSlashing`](#shardproposerslashing) - - [`ShardCommitteeWork`](#shardcommitteework) + - [`ShardWork`](#shardwork) - [Helper functions](#helper-functions) - [Misc](#misc-2) - [`next_power_of_two`](#next_power_of_two) @@ -185,7 +185,7 @@ class BeaconState(merge.BeaconState): # [extends The Merge state] current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] # [New fields] # A ring buffer of the latest slots, with information per active shard. - shard_buffer: Vector[List[ShardCommitteeWork, MAX_SHARDS], SHARD_STATE_MEMORY_SLOTS] + shard_buffer: Vector[List[ShardWork, MAX_SHARDS], SHARD_STATE_MEMORY_SLOTS] shard_gasprice: uint64 current_epoch_start_shard: Shard ``` @@ -282,7 +282,7 @@ class ShardProposerSlashing(Container): signed_reference_2: SignedShardBlobReference ``` -### `ShardCommitteeWork` +### `ShardWork` ```python class ShardWork(Container): @@ -765,7 +765,7 @@ def reset_pending_headers(state: BeaconState) -> None: buffer_index = slot % SHARD_STATE_MEMORY_SLOTS # Reset the shard work tracking - state.shard_buffer[buffer_index] = [ShardCommitteeWork() for _ in range(active_shards)] + state.shard_buffer[buffer_index] = [ShardWork() for _ in range(active_shards)] start_shard = get_start_shard(state, slot) for shard_index in range(state.shard_buffer[buffer_index]): From 31f48b7b3bdc0c415a3cf50f3366402efb24d2f1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 May 2021 22:39:25 +0200 Subject: [PATCH 05/12] update sharding presets --- presets/mainnet/sharding.yaml | 2 ++ presets/minimal/sharding.yaml | 2 ++ specs/sharding/beacon-chain.md | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/presets/mainnet/sharding.yaml b/presets/mainnet/sharding.yaml index 9a81c8cdc..2b78855fc 100644 --- a/presets/mainnet/sharding.yaml +++ b/presets/mainnet/sharding.yaml @@ -15,6 +15,8 @@ MAX_SHARD_PROPOSER_SLASHINGS: 16 # Shard block configs # --------------------------------------------------------------- MAX_SHARD_HEADERS_PER_SHARD: 4 +# 2**8 (= 256) +SHARD_STATE_MEMORY_SLOTS: 256 # 2**11 (= 2,048) MAX_SAMPLES_PER_BLOCK: 2048 # 2**10 (= 1,1024) diff --git a/presets/minimal/sharding.yaml b/presets/minimal/sharding.yaml index 7dedbc926..10f79c96e 100644 --- a/presets/minimal/sharding.yaml +++ b/presets/minimal/sharding.yaml @@ -15,6 +15,8 @@ MAX_SHARD_PROPOSER_SLASHINGS: 4 # Shard block configs # --------------------------------------------------------------- MAX_SHARD_HEADERS_PER_SHARD: 4 +# 2**8 (= 256) +SHARD_STATE_MEMORY_SLOTS: 256 # 2**11 (= 2,048) MAX_SAMPLES_PER_BLOCK: 2048 # 2**10 (= 1,1024) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index d5e43623b..a562050bf 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -119,7 +119,7 @@ The following values are (non-configurable) constants used throughout the specif | `GASPRICE_ADJUSTMENT_COEFFICIENT` | `uint64(2**3)` (= 8) | Gasprice may decrease/increase by at most exp(1 / this value) *per epoch* | | `MAX_SHARD_PROPOSER_SLASHINGS` | `2**4` (= 16) | Maximum amount of shard proposer slashing operations per block | | `MAX_SHARD_HEADERS_PER_SHARD` | `4` | | -| `SHARD_STATE_MEMORY_SLOTS` | `uint64(2**8)` (=256) | Number of slots for which shard commitments and confirmation status is directly available in the state | +| `SHARD_STATE_MEMORY_SLOTS` | `uint64(2**8)` (= 256) | Number of slots for which shard commitments and confirmation status is directly available in the state | ### Shard block samples From bb3d581b13170c74f5afa5661b21c1d920e83f21 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 May 2021 23:55:16 +0200 Subject: [PATCH 06/12] update shard processing naming and doc structure --- specs/sharding/beacon-chain.md | 48 +++++++++++++++++----------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index a562050bf..be4520a31 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -51,14 +51,14 @@ - [`compute_committee_index_from_shard`](#compute_committee_index_from_shard) - [Block processing](#block-processing) - [Operations](#operations) - - [New Attestation processing](#new-attestation-processing) - - [Updated `process_attestation`](#updated-process_attestation) - - [`update_pending_votes`](#update_pending_votes) - - [`process_shard_header`](#process_shard_header) - - [Shard Proposer slashings](#shard-proposer-slashings) + - [Extended Attestation processing](#extended-attestation-processing) + - [`process_shard_header`](#process_shard_header) + - [`process_shard_proposer_slashing`](#process_shard_proposer_slashing) - [Epoch transition](#epoch-transition) - - [Pending headers](#pending-headers) - - [Shard epoch increment](#shard-epoch-increment) + - [`process_pending_shard_confirmations`](#process_pending_shard_confirmations) + - [`charge_confirmed_shard_fees`](#charge_confirmed_shard_fees) + - [`reset_pending_shard_work`](#reset_pending_shard_work) + - [`process_shard_epoch_increment`](#process_shard_epoch_increment) @@ -520,20 +520,16 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: for_ops(body.voluntary_exits, process_voluntary_exit) ``` -### New Attestation processing - -#### Updated `process_attestation` +##### Extended Attestation processing ```python def process_attestation(state: BeaconState, attestation: Attestation) -> None: phase0.process_attestation(state, attestation) - update_pending_votes(state, attestation) + update_pending_shard_work(state, attestation) ``` -#### `update_pending_votes` - ```python -def update_pending_votes(state: BeaconState, attestation: Attestation) -> None: +def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> None: attestation_shard = compute_shard_from_committee_index( state, attestation.data.slot, @@ -581,7 +577,7 @@ def update_pending_votes(state: BeaconState, attestation: Attestation) -> None: ) ``` -#### `process_shard_header` +##### `process_shard_header` ```python def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeader) -> None: @@ -640,7 +636,7 @@ the length proof is the commitment to the polynomial `B(X) * X**(MAX_DEGREE + 1 where `MAX_DEGREE` is the maximum power of `s` available in the setup, which is `MAX_DEGREE = len(G2_SETUP) - 1`. The goal is to ensure that a proof can only be constructed if `deg(B) < l` (there are not hidden higher-order terms in the polynomial, which would thwart reconstruction). -##### Shard Proposer slashings +##### `process_shard_proposer_slashing` ```python def process_shard_proposer_slashing(state: BeaconState, proposer_slashing: ShardProposerSlashing) -> None: @@ -680,9 +676,9 @@ def process_epoch(state: BeaconState) -> None: process_slashings(state) # Sharding - process_pending_headers(state) - charge_confirmed_header_fees(state) - reset_pending_headers(state) + process_pending_shard_confirmations(state) + charge_confirmed_shard_fees(state) + reset_pending_shard_work(state) # Final updates # Phase 0 @@ -696,10 +692,10 @@ def process_epoch(state: BeaconState) -> None: process_shard_epoch_increment(state) ``` -#### Pending headers +#### `process_pending_shard_confirmations` ```python -def process_pending_headers(state: BeaconState) -> None: +def process_pending_shard_confirmations(state: BeaconState) -> None: # Pending header processing applies to the previous epoch. # Skip if `GENESIS_EPOCH` because no prior epoch to process. if get_current_epoch(state) == GENESIS_EPOCH: @@ -721,8 +717,10 @@ def process_pending_headers(state: BeaconState) -> None: committee_work.change(selector=CONFIRMED_SHARD_DATA, value=winning_header.commitment) ``` +#### `charge_confirmed_shard_fees` + ```python -def charge_confirmed_header_fees(state: BeaconState) -> None: +def charge_confirmed_shard_fees(state: BeaconState) -> None: new_gasprice = state.shard_gasprice previous_epoch = get_previous_epoch(state) previous_epoch_start_slot = compute_start_slot_at_epoch(previous_epoch) @@ -753,8 +751,10 @@ def charge_confirmed_header_fees(state: BeaconState) -> None: state.shard_gasprice = new_gasprice ``` +#### `reset_pending_shard_work` + ```python -def reset_pending_headers(state: BeaconState) -> None: +def reset_pending_shard_work(state: BeaconState) -> None: # Add dummy "empty" PendingShardHeader (default vote if no shard header is available) next_epoch = get_current_epoch(state) + 1 next_epoch_start_slot = compute_start_slot_at_epoch(next_epoch) @@ -787,7 +787,7 @@ def reset_pending_headers(state: BeaconState) -> None: # the shard is inactive for this slot otherwise, no committee available, default to UNCONFIRMED_SHARD_DATA. ``` -#### Shard epoch increment +#### `process_shard_epoch_increment` ```python def process_shard_epoch_increment(state: BeaconState) -> None: From 5a235d02516c270fa1703a4d78073df6d413d70f Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Mon, 31 May 2021 17:03:06 +0200 Subject: [PATCH 07/12] Review suggestions Co-authored-by: Anton Nashatyrev --- specs/sharding/beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index be4520a31..930d6a08e 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -550,7 +550,7 @@ def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> N header_index = [header.root for header in current_headers].index(attestation.data.shard_header_root) # Update votes bitfield in the state - pending_header: PendingShardHeader = state.shard_buffer[buffer_index][attestation_shard][header_index] + pending_header: PendingShardHeader = current_headers[header_index] full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) participants_balance = Gwei(0) for i, bit in enumerate(attestation.aggregation_bits): @@ -594,7 +594,7 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade assert header.body_summary.beacon_block_root == get_block_root_at_slot(state, header.slot - 1) # Check that this data is still pending - committee_work = state.shard_buffer[header.slot % SHARD_STATE_MEMORY_SLOTS][header.slot] + committee_work = state.shard_buffer[header.slot % SHARD_STATE_MEMORY_SLOTS][header.shard] assert committee_work.status.selector == PENDING_SHARD_DATA # Check that this header is not yet in the pending list @@ -628,7 +628,7 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade ) # Include it in the pending list - state.shard_buffer[header.slot % SHARD_STATE_MEMORY_SLOTS][header.slot].append(pending_header) + state.shard_buffer[header.slot % SHARD_STATE_MEMORY_SLOTS][header.shard].append(pending_header) ``` The degree proof works as follows. For a block `B` with length `l` (so `l` values in `[0...l - 1]`, seen as a polynomial `B(X)` which takes these values), From 2545c3e2d07b6d78b486e7232a6608b927f9e6da Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 31 May 2021 18:55:08 +0200 Subject: [PATCH 08/12] update pending header weights after epoch transition, fix committee index loop, fix header processing status assert, add todos for Altair-like shard attestation rewards --- specs/sharding/beacon-chain.md | 74 +++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 930d6a08e..84ce94867 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -251,6 +251,8 @@ class PendingShardHeader(Container): votes: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] # Sum of effective balances of votes weight: Gwei + # When the header was last updated, as reference for weight accuracy + update_slot: Slot ``` ### `ShardBlobReference` @@ -538,32 +540,39 @@ def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> N buffer_index = attestation.data.slot % SHARD_STATE_MEMORY_SLOTS committee_work = state.shard_buffer[buffer_index][attestation_shard] - # Skip attestation vote accounting if the header is already confirmed - if committee_work.status.selector == CONFIRMED_SHARD_DATA: + # Skip attestation vote accounting if the header is not pending + if committee_work.status.selector != PENDING_SHARD_DATA: + # TODO In Altair: set participation bit flag, if attestation matches winning header. return - # Note that shard-slot combinations without an assigned committee do not have a pending state - assert shard_info.status.selector == PENDING_SHARD_DATA current_headers: Sequence[PendingShardHeader] = committee_work.status.value # Find the corresponding header, abort if it cannot be found header_index = [header.root for header in current_headers].index(attestation.data.shard_header_root) - # Update votes bitfield in the state pending_header: PendingShardHeader = current_headers[header_index] full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) - participants_balance = Gwei(0) + full_committee_balance = Gwei(0) + + # The weight may be outdated if it is not the initial weight, and from a previous epoch + if pending_header.weight != 0 and compute_epoch_at_slot(pending_header.update_slot) < get_current_epoch(state): + pending_header.weight = sum(state.validators[index].effective_balance for index, bit + in zip(full_committee, pending_header.votes) if bit) + + pending_header.update_slot = state.slot + + # Update votes bitfield in the state, update weights for i, bit in enumerate(attestation.aggregation_bits): weight = state.validators[full_committee[i]].effective_balance + full_committee_balance += weight if bit: if not pending_header.votes[i]: pending_header.weight += weight - pending_header.votes[i] = True - participants_balance += weight + pending_header.votes[i] = True # Check if the PendingShardHeader is eligible for expedited confirmation, requiring 2/3 of balance attesting - full_committee_balance = get_total_balance(state, set(full_committee)) - if participants_balance * 3 >= full_committee_balance * 2: + if pending_header.weight * 3 >= full_committee_balance * 2: + # TODO In Altair: set participation bit flag for voters of this early winning header if pending_header.commitment == DataCommitment(): # The committee voted to not confirm anything state.shard_buffer[buffer_index][attestation_shard].change( @@ -625,6 +634,7 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade root=header_root, votes=initial_votes, weight=0, + update_slot=state.slot, ) # Include it in the pending list @@ -669,19 +679,18 @@ This epoch transition overrides the Merge epoch transition: ```python def process_epoch(state: BeaconState) -> None: - process_justification_and_finalization(state) - process_rewards_and_penalties(state) - process_registry_updates(state) - - process_slashings(state) - # Sharding process_pending_shard_confirmations(state) charge_confirmed_shard_fees(state) reset_pending_shard_work(state) + # Phase0 + process_justification_and_finalization(state) + process_rewards_and_penalties(state) + process_registry_updates(state) + process_slashings(state) + # Final updates - # Phase 0 process_eth1_data_reset(state) process_effective_balance_updates(state) process_slashings_reset(state) @@ -711,6 +720,7 @@ def process_pending_shard_confirmations(state: BeaconState) -> None: committee_work = state.shard_buffer[buffer_index][shard_index] if committee_work.selector == PENDING_SHARD_DATA: winning_header = max(committee_work.value, key=lambda header: header.weight) + # TODO In Altair: set participation bit flag of voters for winning header if winning_header.commitment == DataCommitment(): committee_work.change(selector=UNCONFIRMED_SHARD_DATA, value=None) else: @@ -768,23 +778,23 @@ def reset_pending_shard_work(state: BeaconState) -> None: state.shard_buffer[buffer_index] = [ShardWork() for _ in range(active_shards)] start_shard = get_start_shard(state, slot) - for shard_index in range(state.shard_buffer[buffer_index]): - if start_shard <= shard_index < start_shard + committees_per_slot: - # a committee is available, initialize a pending shard-header list - committee_index = CommitteeIndex(shard_index - start_shard) - committee_length = len(get_beacon_committee(state, slot, committee_index)) - state.shard_buffer[buffer_index][shard_index].change( - selector=PENDING_SHARD_DATA, - value=List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD]( - PendingShardHeader( - commitment=DataCommitment(), - root=Root(), - votes=Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length), - weight=0, - ) + for committee_index in range(committees_per_slot): + shard = (start_shard + committee_index) % active_shards + # a committee is available, initialize a pending shard-header list + committee_length = len(get_beacon_committee(state, slot, committee_index)) + state.shard_buffer[buffer_index][shard].change( + selector=PENDING_SHARD_DATA, + value=List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD]( + PendingShardHeader( + commitment=DataCommitment(), + root=Root(), + votes=Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length), + weight=0, + update_slot=slot, ) ) - # the shard is inactive for this slot otherwise, no committee available, default to UNCONFIRMED_SHARD_DATA. + ) + # a shard without committee available defaults to UNCONFIRMED_SHARD_DATA. ``` #### `process_shard_epoch_increment` From 417dda832601595aa8c066734a32bc319dc1d584 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 1 Jun 2021 22:38:36 +0200 Subject: [PATCH 09/12] fix committee index assertion --- specs/sharding/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 84ce94867..ce0dbd0f6 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -480,7 +480,7 @@ def compute_committee_index_from_shard(state: BeaconState, slot: Slot, shard: Sh epoch = compute_epoch_at_slot(slot) active_shards = get_active_shard_count(state, epoch) index = CommitteeIndex((active_shards + shard - get_start_shard(state, slot)) % active_shards) - assert index >= get_committee_count_per_slot(state, epoch) + assert index < get_committee_count_per_slot(state, epoch) return index ``` From c0af4201306e42393f6c4de84413dd293d3e58ac Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 3 Jun 2021 17:32:35 +0200 Subject: [PATCH 10/12] shard work status enum, prefix instead of suffix + move full committee balance init line --- specs/sharding/beacon-chain.md | 40 +++++++++++++++++----------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index ce0dbd0f6..7c9c6295d 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -13,7 +13,7 @@ - [Constants](#constants) - [Misc](#misc) - [Domain types](#domain-types) - - [Shard Header Status](#shard-header-status) + - [Shard Work Status](#shard-work-status) - [Preset](#preset) - [Misc](#misc-1) - [Shard block samples](#shard-block-samples) @@ -101,13 +101,13 @@ The following values are (non-configurable) constants used throughout the specif | `DOMAIN_SHARD_PROPOSER` | `DomainType('0x80000000')` | | `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | -### Shard Header Status +### Shard Work Status | Name | Value | Notes | | - | - | - | -| `UNCONFIRMED_SHARD_DATA` | `0` | Unconfirmed, nullified after confirmation time elapses | -| `CONFIRMED_SHARD_DATA` | `1` | Confirmed, reduced to just the commitment | -| `PENDING_SHARD_DATA` | `2` | Pending, a list of competing headers | +| `SHARD_WORK_UNCONFIRMED` | `0` | Unconfirmed, nullified after confirmation time elapses | +| `SHARD_WORK_CONFIRMED` | `1` | Confirmed, reduced to just the commitment | +| `SHARD_WORK_PENDING` | `2` | Pending, a list of competing headers | ## Preset @@ -289,10 +289,10 @@ class ShardProposerSlashing(Container): ```python class ShardWork(Container): # Upon confirmation the data is reduced to just the header. - status: Union[ # See Shard Header Status enum - None, # UNCONFIRMED_SHARD_DATA - DataCommitment, # CONFIRMED_SHARD_DATA - List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD] # PENDING_SHARD_DATA + status: Union[ # See Shard Work Status enum + None, # SHARD_WORK_UNCONFIRMED + DataCommitment, # SHARD_WORK_CONFIRMED + List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD] # SHARD_WORK_PENDING ] ``` @@ -541,7 +541,7 @@ def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> N committee_work = state.shard_buffer[buffer_index][attestation_shard] # Skip attestation vote accounting if the header is not pending - if committee_work.status.selector != PENDING_SHARD_DATA: + if committee_work.status.selector != SHARD_WORK_PENDING: # TODO In Altair: set participation bit flag, if attestation matches winning header. return @@ -552,7 +552,6 @@ def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> N pending_header: PendingShardHeader = current_headers[header_index] full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) - full_committee_balance = Gwei(0) # The weight may be outdated if it is not the initial weight, and from a previous epoch if pending_header.weight != 0 and compute_epoch_at_slot(pending_header.update_slot) < get_current_epoch(state): @@ -561,6 +560,7 @@ def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> N pending_header.update_slot = state.slot + full_committee_balance = Gwei(0) # Update votes bitfield in the state, update weights for i, bit in enumerate(attestation.aggregation_bits): weight = state.validators[full_committee[i]].effective_balance @@ -576,12 +576,12 @@ def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> N if pending_header.commitment == DataCommitment(): # The committee voted to not confirm anything state.shard_buffer[buffer_index][attestation_shard].change( - selector=UNCONFIRMED_SHARD_DATA, + selector=SHARD_WORK_UNCONFIRMED, value=None, ) else: state.shard_buffer[buffer_index][attestation_shard].change( - selector=CONFIRMED_SHARD_DATA, + selector=SHARD_WORK_CONFIRMED, value=pending_header.commitment, ) ``` @@ -604,7 +604,7 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade # Check that this data is still pending committee_work = state.shard_buffer[header.slot % SHARD_STATE_MEMORY_SLOTS][header.shard] - assert committee_work.status.selector == PENDING_SHARD_DATA + assert committee_work.status.selector == SHARD_WORK_PENDING # Check that this header is not yet in the pending list current_headers: Sequence[PendingShardHeader] = committee_work.status.value @@ -718,13 +718,13 @@ def process_pending_shard_confirmations(state: BeaconState) -> None: buffer_index = slot % SHARD_STATE_MEMORY_SLOTS for shard_index in range(len(state.shard_buffer[buffer_index])): committee_work = state.shard_buffer[buffer_index][shard_index] - if committee_work.selector == PENDING_SHARD_DATA: + if committee_work.selector == SHARD_WORK_PENDING: winning_header = max(committee_work.value, key=lambda header: header.weight) # TODO In Altair: set participation bit flag of voters for winning header if winning_header.commitment == DataCommitment(): - committee_work.change(selector=UNCONFIRMED_SHARD_DATA, value=None) + committee_work.change(selector=SHARD_WORK_UNCONFIRMED, value=None) else: - committee_work.change(selector=CONFIRMED_SHARD_DATA, value=winning_header.commitment) + committee_work.change(selector=SHARD_WORK_CONFIRMED, value=winning_header.commitment) ``` #### `charge_confirmed_shard_fees` @@ -743,7 +743,7 @@ def charge_confirmed_shard_fees(state: BeaconState) -> None: buffer_index = slot % SHARD_STATE_MEMORY_SLOTS for shard_index in range(len(state.shard_buffer[buffer_index])): committee_work = state.shard_buffer[buffer_index][shard_index] - if committee_work.status.selector == CONFIRMED_SHARD_DATA: + if committee_work.status.selector == SHARD_WORK_CONFIRMED: # Charge EIP 1559 fee proposer = get_shard_proposer_index(state, slot, Shard(shard_index)) fee = ( @@ -783,7 +783,7 @@ def reset_pending_shard_work(state: BeaconState) -> None: # a committee is available, initialize a pending shard-header list committee_length = len(get_beacon_committee(state, slot, committee_index)) state.shard_buffer[buffer_index][shard].change( - selector=PENDING_SHARD_DATA, + selector=SHARD_WORK_PENDING, value=List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD]( PendingShardHeader( commitment=DataCommitment(), @@ -794,7 +794,7 @@ def reset_pending_shard_work(state: BeaconState) -> None: ) ) ) - # a shard without committee available defaults to UNCONFIRMED_SHARD_DATA. + # a shard without committee available defaults to SHARD_WORK_UNCONFIRMED. ``` #### `process_shard_epoch_increment` From d4f6459108a06a2e4fc5b2870d9c2ee19463cf36 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Thu, 3 Jun 2021 18:00:51 +0200 Subject: [PATCH 11/12] Define missing header_root Co-authored-by: Anton Nashatyrev --- specs/sharding/beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 7c9c6295d..9a0117fb4 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -608,6 +608,7 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade # Check that this header is not yet in the pending list current_headers: Sequence[PendingShardHeader] = committee_work.status.value + header_root = hash_tree_root(header) assert header_root not in [pending_header.root for pending_header in current_headers] # Verify proposer From 9050897cb273cf772abb4ac97bc66215b1cf9740 Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 3 Jun 2021 19:11:47 +0200 Subject: [PATCH 12/12] fix committee work status and commitment references --- specs/sharding/beacon-chain.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 9a0117fb4..4f9f6d934 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -719,13 +719,13 @@ def process_pending_shard_confirmations(state: BeaconState) -> None: buffer_index = slot % SHARD_STATE_MEMORY_SLOTS for shard_index in range(len(state.shard_buffer[buffer_index])): committee_work = state.shard_buffer[buffer_index][shard_index] - if committee_work.selector == SHARD_WORK_PENDING: - winning_header = max(committee_work.value, key=lambda header: header.weight) + if committee_work.status.selector == SHARD_WORK_PENDING: + winning_header = max(committee_work.status.value, key=lambda header: header.weight) # TODO In Altair: set participation bit flag of voters for winning header if winning_header.commitment == DataCommitment(): - committee_work.change(selector=SHARD_WORK_UNCONFIRMED, value=None) + committee_work.status.change(selector=SHARD_WORK_UNCONFIRMED, value=None) else: - committee_work.change(selector=SHARD_WORK_CONFIRMED, value=winning_header.commitment) + committee_work.status.change(selector=SHARD_WORK_CONFIRMED, value=winning_header.commitment) ``` #### `charge_confirmed_shard_fees` @@ -745,10 +745,11 @@ def charge_confirmed_shard_fees(state: BeaconState) -> None: for shard_index in range(len(state.shard_buffer[buffer_index])): committee_work = state.shard_buffer[buffer_index][shard_index] if committee_work.status.selector == SHARD_WORK_CONFIRMED: + commitment: DataCommitment = committee_work.status.value # Charge EIP 1559 fee proposer = get_shard_proposer_index(state, slot, Shard(shard_index)) fee = ( - (state.shard_gasprice * candidate.commitment.length) + (state.shard_gasprice * commitment.length) // TARGET_SAMPLES_PER_BLOCK ) decrease_balance(state, proposer, fee) @@ -756,7 +757,7 @@ def charge_confirmed_shard_fees(state: BeaconState) -> None: # Track updated gas price new_gasprice = compute_updated_gasprice( new_gasprice, - candidate.commitment.length, + commitment.length, adjustment_quotient, ) state.shard_gasprice = new_gasprice