From bc936768c74cd8afb72fd383dd76750ebdc00c8e Mon Sep 17 00:00:00 2001 From: protolambda Date: Thu, 17 Jun 2021 23:17:47 +0200 Subject: [PATCH 01/23] global selection of shard proposers --- specs/sharding/beacon-chain.md | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 517892b8c..41497d8d8 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -43,7 +43,6 @@ - [Beacon state accessors](#beacon-state-accessors) - [Updated `get_committee_count_per_slot`](#updated-get_committee_count_per_slot) - [`get_active_shard_count`](#get_active_shard_count) - - [`get_shard_committee`](#get_shard_committee) - [`compute_proposer_index`](#compute_proposer_index) - [`get_shard_proposer_index`](#get_shard_proposer_index) - [`get_start_shard`](#get_start_shard) @@ -369,24 +368,6 @@ def get_active_shard_count(state: BeaconState, epoch: Epoch) -> uint64: return INITIAL_ACTIVE_SHARDS ``` -#### `get_shard_committee` - -```python -def get_shard_committee(beacon_state: BeaconState, epoch: Epoch, shard: Shard) -> Sequence[ValidatorIndex]: - """ - Return the shard committee of the given ``epoch`` of the given ``shard``. - """ - source_epoch = compute_committee_source_epoch(epoch, SHARD_COMMITTEE_PERIOD) - active_validator_indices = get_active_validator_indices(beacon_state, source_epoch) - seed = get_seed(beacon_state, source_epoch, DOMAIN_SHARD_COMMITTEE) - return compute_committee( - indices=active_validator_indices, - seed=seed, - index=shard, - count=get_active_shard_count(beacon_state, epoch), - ) -``` - #### `compute_proposer_index` Updated version to get a proposer index that will only allow proposers with a certain minimum balance, @@ -423,8 +404,7 @@ def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard Return the proposer's index of shard block at ``slot``. """ epoch = compute_epoch_at_slot(slot) - committee = get_shard_committee(beacon_state, epoch, shard) - seed = hash(get_seed(beacon_state, epoch, DOMAIN_SHARD_PROPOSER) + uint_to_bytes(slot)) + seed = hash(get_seed(beacon_state, epoch, DOMAIN_SHARD_PROPOSER) + uint_to_bytes(slot) + uint_to_bytes(shard)) # Proposer must have sufficient balance to pay for worst case fee burn EFFECTIVE_BALANCE_MAX_DOWNWARD_DEVIATION = ( @@ -435,7 +415,8 @@ def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard beacon_state.shard_gasprice * MAX_SAMPLES_PER_BLOCK // TARGET_SAMPLES_PER_BLOCK + EFFECTIVE_BALANCE_MAX_DOWNWARD_DEVIATION ) - return compute_proposer_index(beacon_state, committee, seed, min_effective_balance) + indices = get_active_validator_indices(state, epoch) + return compute_proposer_index(beacon_state, indices, seed, min_effective_balance) ``` #### `get_start_shard` From 4b2523961723717c49a61e6f5587d4802211a33d Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 18 Jun 2021 02:21:21 +0200 Subject: [PATCH 02/23] builders make blobs, proposers make blocks --- specs/sharding/beacon-chain.md | 264 +++++++++++++++++++++++++++----- specs/sharding/p2p-interface.md | 45 ------ 2 files changed, 224 insertions(+), 85 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 41497d8d8..acba8e2a5 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -9,6 +9,7 @@ - [Introduction](#introduction) + - [Glossary](#glossary) - [Custom types](#custom-types) - [Constants](#constants) - [Misc](#misc) @@ -23,15 +24,25 @@ - [Updated containers](#updated-containers) - [`AttestationData`](#attestationdata) - [`BeaconBlockBody`](#beaconblockbody) + - [`Builder`](#builder) - [`BeaconState`](#beaconstate) - [New containers](#new-containers) - [`DataCommitment`](#datacommitment) + - [ShardBlobBody](#shardblobbody) - [`ShardBlobBodySummary`](#shardblobbodysummary) + - [`ShardBlob`](#shardblob) - [`ShardBlobHeader`](#shardblobheader) + - [`SignedShardBlob`](#signedshardblob) - [`SignedShardBlobHeader`](#signedshardblobheader) + - [ShardBlock](#shardblock) + - [`ShardBlockHeader`](#shardblockheader) + - [`SignedShardBlock`](#signedshardblock) + - [`SignedShardBlockHeader`](#signedshardblockheader) - [`PendingShardHeader`](#pendingshardheader) - [`ShardBlobReference`](#shardblobreference) - [`SignedShardBlobReference`](#signedshardblobreference) + - [`ShardBlockReference`](#shardblockreference) + - [`SignedShardBlockReference`](#signedshardblockreference) - [`ShardProposerSlashing`](#shardproposerslashing) - [`ShardWork`](#shardwork) - [Helper functions](#helper-functions) @@ -51,6 +62,7 @@ - [Block processing](#block-processing) - [Operations](#operations) - [Extended Attestation processing](#extended-attestation-processing) + - [`charge_builder`](#charge_builder) - [`process_shard_header`](#process_shard_header) - [`process_shard_proposer_slashing`](#process_shard_proposer_slashing) - [Epoch transition](#epoch-transition) @@ -68,6 +80,13 @@ This document describes the extensions made to the Phase 0 design of The Beacon based on the ideas [here](https://hackmd.io/G-Iy5jqyT7CXWEz8Ssos8g) and more broadly [here](https://arxiv.org/abs/1809.09044), using KZG10 commitments to commit to data to remove any need for fraud proofs (and hence, safety-critical synchrony assumptions) in the design. +### Glossary + +- **Data**: A list of KZG points, to translate a byte string into +- **Blob**: Data with commitments and meta-data, like a flattened bundle of L2 transactions. +- **Builder**: Builds blobs and bids for proposal slots with fee-paying blob-headers, responsible for availability. +- **Shard proposer**: Validator, selects a signed blob-header, taking bids for shard data opportunity. +- **Shard block**: Unique per `(slot, shard, proposer)`, selected signed blob ## Custom types @@ -78,6 +97,7 @@ We define the following Python custom types for type hinting and readability: | `Shard` | `uint64` | A shard number | | `BLSCommitment` | `Bytes48` | A G1 curve point | | `BLSPoint` | `uint256` | A number `x` in the range `0 <= x < MODULUS` | +| `BuilderIndex` | `uint64` | Builder registry index | ## Constants @@ -97,7 +117,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | | `DOMAIN_SHARD_PROPOSER` | `DomainType('0x80000000')` | -| `DOMAIN_SHARD_COMMITTEE` | `DomainType('0x81000000')` | +| `DOMAIN_SHARD_BUILDER` | `DomainType('0x81000000')` | ### Shard Work Status @@ -118,6 +138,7 @@ The following values are (non-configurable) constants used throughout the specif | `MAX_SHARD_PROPOSER_SLASHINGS` | `2**4` (= 16) | Maximum amount of shard proposer slashing operations per block | | `MAX_SHARD_HEADERS_PER_SHARD` | `4` | | | `SHARD_STATE_MEMORY_SLOTS` | `uint64(2**8)` (= 256) | Number of slots for which shard commitments and confirmation status is directly available in the state | +| `BUILDER_REGISTRY_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | builders | ### Shard block samples @@ -162,8 +183,8 @@ class AttestationData(Container): # FFG vote source: Checkpoint target: Checkpoint - # Shard header root - shard_header_root: Root # [New in Sharding] + # Hash-tree-root of ShardBlock + shard_block_root: Root # [New in Sharding] ``` ### `BeaconBlockBody` @@ -171,7 +192,16 @@ class AttestationData(Container): ```python class BeaconBlockBody(merge.BeaconBlockBody): # [extends The Merge block body] shard_proposer_slashings: List[ShardProposerSlashing, MAX_SHARD_PROPOSER_SLASHINGS] - shard_headers: List[SignedShardBlobHeader, MAX_SHARDS * MAX_SHARD_HEADERS_PER_SHARD] + shard_headers: List[SignedShardBlockHeader, MAX_SHARDS * MAX_SHARD_HEADERS_PER_SHARD] +``` + +### `Builder` + +```python +class Builder(Container): + pubkey: BLSPubkey + # TODO: fields for either an expiry mechanism (refunding execution account with remaining balance) + # and/or a builder-transaction mechanism. ``` ### `BeaconState` @@ -182,6 +212,9 @@ class BeaconState(merge.BeaconState): previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] # [New fields] + # Builder registry. + builders: List[Builder, BUILDER_REGISTRY_LIMIT] + builder_balances: List[Gwei, BUILDER_REGISTRY_LIMIT] # A ring buffer of the latest slots, with information per active shard. shard_buffer: Vector[List[ShardWork, MAX_SHARDS], SHARD_STATE_MEMORY_SLOTS] shard_gasprice: uint64 @@ -189,9 +222,6 @@ class BeaconState(merge.BeaconState): ## New containers -The shard data itself is network-layer only, and can be found in the [P2P specification](./p2p-interface.md). -The beacon chain registers just the commitments of the shard data. - ### `DataCommitment` ```python @@ -202,8 +232,33 @@ class DataCommitment(Container): length: uint64 ``` +### ShardBlobBody + +Unsigned shard data, bundled by a shard-builder. +Unique, signing different bodies as shard proposer for the same `(slot, shard)` is slashable. + +```python +class ShardBlobBody(Container): + # The actual data commitment + commitment: DataCommitment + # Proof that the degree < commitment.length + degree_proof: BLSCommitment + # The actual data. Should match the commitment and degree proof. + data: List[BLSPoint, POINTS_PER_SAMPLE * MAX_SAMPLES_PER_BLOCK] + # Latest block root of the Beacon Chain, before shard_blob.slot + beacon_block_root: Root + # Builder of the data, pays data-fee to proposer + builder_index: BuilderIndex + # TODO: fee payment amount fields (EIP 1559 like) +``` + ### `ShardBlobBodySummary` +Summary version of the `ShardBlobBody`, omitting the data payload, while preserving the data-commitments. + +The commitments are not further collapsed to a single hash, +to avoid an extra network roundtrip between proposer and builder, to include the header on-chain more quickly. + ```python class ShardBlobBodySummary(Container): # The actual data commitment @@ -214,36 +269,110 @@ class ShardBlobBodySummary(Container): data_root: Root # Latest block root of the Beacon Chain, before shard_blob.slot beacon_block_root: Root + # Builder of the data, pays data-fee to proposer + builder_index: BuilderIndex + # TODO: fee payment amount fields (EIP 1559 like) +``` + +### `ShardBlob` + +`ShardBlobBody` wrapped with the header data that is unique to the shard blob proposal. + +```python +class ShardBlob(Container): + slot: Slot + shard: Shard + # Blob contents + body: ShardBlobBody ``` ### `ShardBlobHeader` +Header version of `ShardBlob`. Separates designation (slot, shard) and contents (blob). + ```python class ShardBlobHeader(Container): - # Slot and shard that this header is intended for slot: Slot shard: Shard - # SSZ-summary of ShardBlobBody + # Blob contents, without the full data body_summary: ShardBlobBodySummary - # Proposer of the shard-blob - proposer_index: ValidatorIndex +``` + +### `SignedShardBlob` + +Full blob data, signed by the shard builder, ensuring fee payment. + +```python +class SignedShardBlob(Container): + message: ShardBlob + signature: BLSSignature ``` ### `SignedShardBlobHeader` +Header of the blob, the signature is equally applicable to `SignedShardBlob`. +Shard proposers can accept `SignedShardBlobHeader` as a data-transaction. + ```python class SignedShardBlobHeader(Container): message: ShardBlobHeader signature: BLSSignature ``` +### ShardBlock + +Full blob data signed by builder, to be confirmed by proxy as `ShardBlockHeader`. + +```python +class ShardBlock(Container): + # Shard data with fee payment by bundle builder + signed_blob: SignedShardBlob + # Proposer of the shard-blob + proposer_index: ValidatorIndex +``` + +### `ShardBlockHeader` + +Header version of `ShardBlock`, selecting a `SignedShardBlobHeader`. + +```python +class ShardBlockHeader(Container): + # Shard commitments and fee payment by blob builder + signed_blob_header: SignedShardBlobHeader + # Proposer of the shard-blob + proposer_index: ValidatorIndex +``` + +### `SignedShardBlock` + +Shard blob, signed for payment, and signed for proposal. Propagated to attesters. + +```python +class SignedShardBlock(Container): + message: ShardBlock + signature: BLSSignature +``` + +### `SignedShardBlockHeader` + +Header version of `SignedShardBlock`, substituting the full data within the blob for just the hash-tree-root. + +The signature is equally applicable to `SignedShardBlock`, +which the builder can publish as soon as the signed header is seen. + +```python +class SignedShardBlockHeader(Container): + message: ShardBlockHeader + signature: BLSSignature +``` + ### `PendingShardHeader` ```python class PendingShardHeader(Container): # KZG10 commitment to the data commitment: DataCommitment - # hash_tree_root of the ShardHeader (stored so that attestations can be checked against it) + # hash_tree_root of the ShardBlockHeader (stored so that attestations can be checked against it) root: Root # Who voted for the header votes: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] @@ -255,31 +384,50 @@ class PendingShardHeader(Container): ### `ShardBlobReference` +Reference version of `ShardBlobHeader`, substituting the body for just a hash-tree-root. + ```python class ShardBlobReference(Container): - # Slot and shard that this reference is intended for slot: Slot shard: Shard - # Hash-tree-root of ShardBlobBody + # Blob hash-tree-root for reference, enough for uniqueness body_root: Root - # Proposer of the shard-blob - proposer_index: ValidatorIndex ``` ### `SignedShardBlobReference` +`ShardBlobReference`, signed by the blob builder. The builder-signature is part of the block identity. + ```python class SignedShardBlobReference(Container): message: ShardBlobReference signature: BLSSignature ``` +### `ShardBlockReference` + +```python +class ShardBlockReference(Container): + # Blob, minimized for efficient slashing + signed_blob_reference: SignedShardBlobReference + # Proposer of the shard-blob + proposer_index: ValidatorIndex +``` + +### `SignedShardBlockReference` + +```python +class SignedShardBlockReference(Container): + message: ShardBlockReference + signature: BLSSignature +``` + ### `ShardProposerSlashing` ```python class ShardProposerSlashing(Container): - signed_reference_1: SignedShardBlobReference - signed_reference_2: SignedShardBlobReference + signed_reference_1: SignedShardBlockReference + signed_reference_2: SignedShardBlockReference ``` ### `ShardWork` @@ -516,7 +664,7 @@ def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> N current_headers: Sequence[PendingShardHeader] = committee_work.status.value # Find the corresponding header, abort if it cannot be found - header_index = [header.root for header in current_headers].index(attestation.data.shard_header_root) + header_index = [header.root for header in current_headers].index(attestation.data.shard_block_root) pending_header: PendingShardHeader = current_headers[header_index] full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) @@ -554,36 +702,63 @@ def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> N ) ``` + +#### `charge_builder` + +```python +def charge_builder(state: BeaconState, index: BuilderIndex, fee: Gwei) -> None: + """ + Decrease the builder balance at index ``index`` by ``fee``, with underflow check. + """ + assert state.builder_balances[index] >= fee + state.builder_balances[index] -= fee +``` + ##### `process_shard_header` ```python -def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeader) -> None: - header = signed_header.message +def process_shard_header(state: BeaconState, signed_block_header: SignedShardBlockHeader) -> None: + block_header: ShardBlockHeader = signed_block_header.message + signed_blob_header: SignedShardBlobHeader = block_header.signed_blob_header + blob_header: ShardBlobHeader = signed_blob_header.message + slot = blob_header.slot + shard = blob_header.shard + # Verify the header is not 0, and not from the future. - assert Slot(0) < header.slot <= state.slot - header_epoch = compute_epoch_at_slot(header.slot) + assert Slot(0) < slot <= state.slot + header_epoch = compute_epoch_at_slot(slot) # Verify that the header is within the processing time window assert header_epoch in [get_previous_epoch(state), get_current_epoch(state)] # Verify that the shard is active - assert header.shard < get_active_shard_count(state, header_epoch) + assert shard < get_active_shard_count(state, header_epoch) # Verify that the block root matches, # to ensure the header will only be included in this specific Beacon Chain sub-tree. - assert header.body_summary.beacon_block_root == get_block_root_at_slot(state, header.slot - 1) + assert blob_header.body_summary.beacon_block_root == get_block_root_at_slot(state, slot - 1) # Check that this data is still pending - committee_work = state.shard_buffer[header.slot % SHARD_STATE_MEMORY_SLOTS][header.shard] + committee_work = state.shard_buffer[slot % SHARD_STATE_MEMORY_SLOTS][shard] assert committee_work.status.selector == SHARD_WORK_PENDING # Check that this header is not yet in the pending list current_headers: List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD] = committee_work.status.value - header_root = hash_tree_root(header) + header_root = hash_tree_root(block_header) assert header_root not in [pending_header.root for pending_header in current_headers] # Verify proposer - assert header.proposer_index == get_shard_proposer_index(state, header.slot, header.shard) - # Verify signature - signing_root = compute_signing_root(header, get_domain(state, DOMAIN_SHARD_PROPOSER)) - assert bls.Verify(state.validators[header.proposer_index].pubkey, signing_root, signed_header.signature) + assert block_header.proposer_index == get_shard_proposer_index(state, slot, shard) + # Verify proposer signature + block_signing_root = compute_signing_root(block_header, get_domain(state, DOMAIN_SHARD_PROPOSER)) + proposer_pubkey = state.validators[block_header.proposer_index].pubkey + assert bls.Verify(proposer_pubkey, block_signing_root, signed_block_header.signature) + + # Verify builder requirements + blob_summary: ShardBlobBodySummary = blob_header.body_summary + builder_index = blob_summary.builder_index + + # Verify builder signature + builder = state.builders[builder_index] + blob_signing_root = compute_signing_root(blob_header, get_domain(state, DOMAIN_SHARD_BUILDER)) # TODO new constant + assert bls.Verify(builder.pubkey, blob_signing_root, signed_blob_header.signature) # Verify the length by verifying the degree. body_summary = header.body_summary @@ -594,12 +769,16 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade == bls.Pairing(body_summary.commitment.point, G2_SETUP[-body_summary.commitment.length]) ) + # Charge builder, with hard balance requirement + fee = Gwei(123) # TODO + charge_builder(state, builder_index, fee) + # Initialize the pending header - index = compute_committee_index_from_shard(state, header.slot, header.shard) - committee_length = len(get_beacon_committee(state, header.slot, index)) + index = compute_committee_index_from_shard(state, slot, shard) + committee_length = len(get_beacon_committee(state, slot, index)) initial_votes = Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length) pending_header = PendingShardHeader( - commitment=body_summary.commitment, + commitment=blob_summary.commitment, root=header_root, votes=initial_votes, weight=0, @@ -619,17 +798,22 @@ The goal is to ensure that a proof can only be constructed if `deg(B) < l` (ther ```python def process_shard_proposer_slashing(state: BeaconState, proposer_slashing: ShardProposerSlashing) -> None: - reference_1 = proposer_slashing.signed_reference_1.message - reference_2 = proposer_slashing.signed_reference_2.message + reference_1: ShardBlockReference = proposer_slashing.signed_reference_1.message + reference_2 : ShardBlockReference = proposer_slashing.signed_reference_2.message + blob_1 = reference_1.signed_blob_reference.message + blob_2 = reference_2.signed_blob_reference.message # Verify header slots match - assert reference_1.slot == reference_2.slot + assert blob_1.slot == blob_2.slot # Verify header shards match - assert reference_1.shard == reference_2.shard + assert blob_1.shard == blob_2.shard # Verify header proposer indices match assert reference_1.proposer_index == reference_2.proposer_index - # Verify the headers are different (i.e. different body) - assert reference_1 != reference_2 + # Verify the headers are different (i.e. different body, or different builder signature) + assert ( + blob_1.body_root != blob_2.body_root + or reference_1.signed_blob_reference.signature != reference_2.signed_blob_reference.signature + ) # Verify the proposer is slashable proposer = state.validators[reference_1.proposer_index] assert is_slashable_validator(proposer, get_current_epoch(state)) diff --git a/specs/sharding/p2p-interface.md b/specs/sharding/p2p-interface.md index baf9494f2..d00321f36 100644 --- a/specs/sharding/p2p-interface.md +++ b/specs/sharding/p2p-interface.md @@ -11,10 +11,6 @@ - [Introduction](#introduction) - [Constants](#constants) - [Misc](#misc) -- [New containers](#new-containers) - - [ShardBlobBody](#shardblobbody) - - [ShardBlob](#shardblob) - - [SignedShardBlob](#signedshardblob) - [Gossip domain](#gossip-domain) - [Topics and messages](#topics-and-messages) - [Shard blob subnets](#shard-blob-subnets) @@ -40,47 +36,6 @@ The adjustments and additions for Shards are outlined in this document. | ---- | ----- | ----------- | | `SHARD_BLOB_SUBNET_COUNT` | `64` | The number of `shard_blob_{subnet_id}` subnets used in the gossipsub protocol. | -## New containers - -### ShardBlobBody - -```python -class ShardBlobBody(Container): - # The actual data commitment - commitment: DataCommitment - # Proof that the degree < commitment.length - degree_proof: BLSCommitment - # The actual data. Should match the commitment and degree proof. - data: List[BLSPoint, POINTS_PER_SAMPLE * MAX_SAMPLES_PER_BLOCK] - # Latest block root of the Beacon Chain, before shard_blob.slot - beacon_block_root: Root -``` - -The user MUST always verify the commitments in the `body` are valid for the `data` in the `body`. - -### ShardBlob - -```python -class ShardBlob(Container): - # Slot and shard that this blob is intended for - slot: Slot - shard: Shard - # Shard data with related commitments and beacon anchor - body: ShardBlobBody - # Proposer of the shard-blob - proposer_index: ValidatorIndex -``` - -This is the expanded form of the `ShardBlobHeader` type. - -### SignedShardBlob - -```python -class SignedShardBlob(Container): - message: ShardBlob - signature: BLSSignature -``` - ## Gossip domain ### Topics and messages From 9e10f582993fc1f48f01c6b57e737c34b84ce4c5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 18 Jun 2021 02:48:58 +0200 Subject: [PATCH 03/23] update networking spec --- specs/sharding/p2p-interface.md | 85 ++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 32 deletions(-) diff --git a/specs/sharding/p2p-interface.md b/specs/sharding/p2p-interface.md index d00321f36..94dd75128 100644 --- a/specs/sharding/p2p-interface.md +++ b/specs/sharding/p2p-interface.md @@ -14,7 +14,7 @@ - [Gossip domain](#gossip-domain) - [Topics and messages](#topics-and-messages) - [Shard blob subnets](#shard-blob-subnets) - - [`shard_blob_{subnet_id}`](#shard_blob_subnet_id) + - [`shard_block_{subnet_id}`](#shard_block_subnet_id) - [Global topics](#global-topics) - [`shard_header`](#shard_header) - [`shard_proposer_slashing`](#shard_proposer_slashing) @@ -34,7 +34,7 @@ The adjustments and additions for Shards are outlined in this document. | Name | Value | Description | | ---- | ----- | ----------- | -| `SHARD_BLOB_SUBNET_COUNT` | `64` | The number of `shard_blob_{subnet_id}` subnets used in the gossipsub protocol. | +| `SHARD_BLOCK_SUBNET_COUNT` | `64` | The number of `shard_block_{subnet_id}` subnets used in the gossipsub protocol. | ## Gossip domain @@ -44,22 +44,22 @@ Following the same scheme as the [Phase0 gossip topics](../phase0/p2p-interface. | Name | Message Type | |----------------------------------|---------------------------| -| `shard_blob_{subnet_id}` | `SignedShardBlob` | -| `shard_header` | `SignedShardBlobHeader` | +| `shard_block_{subnet_id}` | `SignedShardBlock` | +| `shard_block_header` | `SignedShardBlockHeader` | | `shard_proposer_slashing` | `ShardProposerSlashing` | The [DAS network specification](./das-p2p.md) defines additional topics. -#### Shard blob subnets +#### Shard block subnets -Shard blob subnets are used to propagate shard blobs to subsections of the network. +Shard block subnets are used by builders to make their blobs available after selection by shard proposers. -##### `shard_blob_{subnet_id}` +##### `shard_block_{subnet_id}` -Shard block data, in the form of a `SignedShardBlob` is published to the `shard_blob_{subnet_id}` subnets. +Shard block data, in the form of a `SignedShardBlock` is published to the `shard_block_{subnet_id}` subnets. ```python -def compute_subnet_for_shard_blob(state: BeaconState, slot: Slot, shard: Shard) -> uint64: +def compute_subnet_for_shard_block(state: BeaconState, slot: Slot, shard: Shard) -> uint64: """ Compute the correct subnet for a shard blob publication. Note, this mimics compute_subnet_for_attestation(). @@ -69,11 +69,19 @@ def compute_subnet_for_shard_blob(state: BeaconState, slot: Slot, shard: Shard) slots_since_epoch_start = Slot(slot % SLOTS_PER_EPOCH) committees_since_epoch_start = committees_per_slot * slots_since_epoch_start - return uint64((committees_since_epoch_start + committee_index) % SHARD_BLOB_SUBNET_COUNT) + return uint64((committees_since_epoch_start + committee_index) % SHARD_BLOCK_SUBNET_COUNT) ``` -The following validations MUST pass before forwarding the `signed_blob` (with inner `message` as `blob`) on the horizontal subnet or creating samples for it. -- _[IGNORE]_ The `blob` is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- +The following validations MUST pass before forwarding the `signed_block` on the horizontal subnet or creating samples for it. + +We define some aliases to the nested contents of `signed_block`: +```python +block: ShardBlock = signed_block.message +signed_blob: SignedShardBlob = block.signed_blob +blob: ShardBlob = signed_blob.message +``` + +- _[IGNORE]_ The `block` is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `blob.slot <= current_slot` (a client MAY queue future blobs for processing at the appropriate slot). - _[IGNORE]_ The `blob` is new enough to be still be processed -- @@ -81,36 +89,49 @@ The following validations MUST pass before forwarding the `signed_blob` (with in - _[REJECT]_ The shard should have a committee at slot -- i.e. validate that `compute_committee_index_from_shard(state, blob.slot, blob.shard)` doesn't raise an error - _[REJECT]_ The shard blob is for the correct subnet -- - i.e. `compute_subnet_for_shard_blob(state, blob.slot, blob.shard) == subnet_id` -- _[IGNORE]_ The blob is the first blob with valid signature received for the `(blob.proposer_index, blob.slot, blob.shard)` combination. -- _[REJECT]_ As already limited by the SSZ list-limit, it is important the blob is well-formatted and not too large. + i.e. `compute_subnet_for_shard_block(state, blob.slot, blob.shard) == subnet_id` +- _[IGNORE]_ The block is the first block with valid signature received for the `(block.proposer_index, blob.slot, blob.shard)` combination. +- _[REJECT]_ The blob is not too large, the data MUST NOT be larger than the SSZ list-limit, and a client MAY be more strict. - _[REJECT]_ The `blob.body.data` MUST NOT contain any point `p >= MODULUS`. Although it is a `uint256`, not the full 256 bit range is valid. -- _[REJECT]_ The proposer signature, `signed_blob.signature`, is valid with respect to the `proposer_index` pubkey. -- _[REJECT]_ The blob is proposed by the expected `proposer_index` for the blob's slot +- _[REJECT]_ The block proposer signature, `signed_block.signature`, is valid with respect to the `proposer_index` pubkey. +- _[REJECT]_ The blob builder exists and has sufficient balance to back the fee payment. +- _[REJECT]_ The blob builder signature, `signed_blob.signature`, is valid with respect to the `builder_index` pubkey. +- _[REJECT]_ The block is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `blob.body.beacon_block_root`/`slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, - the block MAY be queued for later processing while proposers for the blob's branch are calculated -- + the block MAY be queued for later processing while proposers for the block's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message. #### Global topics -There are two additional global topics for Sharding, one is used to propagate shard blob headers (`shard_header`) to -all nodes on the network. Another one is used to propagate validator message (`shard_proposer_slashing`). +There are two additional global topics for Sharding. -##### `shard_header` +One is used to propagate shard block headers (`shard_block_header`) to all nodes on the network. +Another one is used to propagate shard proposer slashings (`shard_proposer_slashing`). -Shard header data, in the form of a `SignedShardBlobHeader` is published to the global `shard_header` subnet. +##### `shard_block_header` -The following validations MUST pass before forwarding the `signed_shard_blob_header` (with inner `message` as `header`) on the network. -- _[IGNORE]_ The `header` is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- - i.e. validate that `header.slot <= current_slot` +Shard header data, in the form of a `SignedShardBlockHeader` is published to the global `shard_block_header` subnet. +Shard block headers select shard blob bids by builders, and should be timely to ensure builders can publish the full shard block timely. + +The following validations MUST pass before forwarding the `signed_block_header` (with inner `message` as `header`) on the network. + +We define some aliases to the nested contents of `signed_block_header`: +```python +block_header: ShardBlockHeader = signed_block_header.message +signed_blob_header: SignedShardBlobHeader = header.signed_blob_header +blob_header: ShardBlobHeader = signed_blob_header.message +``` + +- _[IGNORE]_ The header is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- + i.e. validate that `blob_header.slot <= current_slot` (a client MAY queue future headers for processing at the appropriate slot). -- _[IGNORE]_ The `header` is new enough to be still be processed -- - i.e. validate that `compute_epoch_at_slot(header.slot) >= get_previous_epoch(state)` -- _[IGNORE]_ The header is the first header with valid signature received for the `(header.proposer_index, header.slot, header.shard)` combination. -- _[REJECT]_ The shard should have a committee at slot -- - i.e. validate that `compute_committee_index_from_shard(state, header.slot, header.shard)` doesn't raise an error -- _[REJECT]_ The proposer signature, `signed_shard_blob_header.signature`, is valid with respect to the `proposer_index` pubkey. +- _[IGNORE]_ The header is new enough to be still be processed -- + i.e. validate that `compute_epoch_at_slot(blob_header.slot) >= get_previous_epoch(state)` +- _[IGNORE]_ The header is the first header with valid signature received for the `(block_header.proposer_index, blob_header.slot, blob_header.shard)` combination. +- _[REJECT]_ The `shard` MUST have a committee at the `slot` -- + i.e. validate that `compute_committee_index_from_shard(state, blob_header.slot, blob_header.shard)` doesn't raise an error +- _[REJECT]_ The proposer signature, `signed_shard_block_header.signature`, is valid with respect to the `block_header.proposer_index` pubkey. - _[REJECT]_ The header is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `header.body_summary.beacon_block_root`/`slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, @@ -124,6 +145,6 @@ Shard proposer slashings, in the form of `ShardProposerSlashing`, are published The following validations MUST pass before forwarding the `shard_proposer_slashing` on to the network. - _[IGNORE]_ The shard proposer slashing is the first valid shard proposer slashing received - for the proposer with index `proposer_slashing.signed_header_1.message.proposer_index`. + for the proposer with index `proposer_slashing.signed_reference_1.message.proposer_index`. The `slot` and `shard` are ignored, there are no per-shard slashings. - _[REJECT]_ All of the conditions within `process_shard_proposer_slashing` pass validation. From b3d5858cc9c42db65005206e6f3f1f31ce983209 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 18 Jun 2021 04:02:06 +0200 Subject: [PATCH 04/23] update data fee payment, todo --- specs/sharding/beacon-chain.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index acba8e2a5..fe0c53bc4 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -770,8 +770,11 @@ def process_shard_header(state: BeaconState, signed_block_header: SignedShardBlo ) # Charge builder, with hard balance requirement - fee = Gwei(123) # TODO + fee = Gwei(123) # TODO EIP 1559 like fee? Burn some of it? charge_builder(state, builder_index, fee) + # TODO: proposer is charged for confirmed headers (see charge_confirmed_shard_fees). + # Need to align incentive, so proposer does not gain from including unconfirmed headers + increase_balance(state, block_header.proposer_index, fee) # Initialize the pending header index = compute_committee_index_from_shard(state, slot, shard) From 2a105f45819e262f27a3ea14279c5a5ca2a3cdd7 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 18 Jun 2021 04:02:34 +0200 Subject: [PATCH 05/23] fix toc --- specs/sharding/p2p-interface.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/sharding/p2p-interface.md b/specs/sharding/p2p-interface.md index 94dd75128..e9b561312 100644 --- a/specs/sharding/p2p-interface.md +++ b/specs/sharding/p2p-interface.md @@ -13,10 +13,10 @@ - [Misc](#misc) - [Gossip domain](#gossip-domain) - [Topics and messages](#topics-and-messages) - - [Shard blob subnets](#shard-blob-subnets) + - [Shard block subnets](#shard-block-subnets) - [`shard_block_{subnet_id}`](#shard_block_subnet_id) - [Global topics](#global-topics) - - [`shard_header`](#shard_header) + - [`shard_block_header`](#shard_block_header) - [`shard_proposer_slashing`](#shard_proposer_slashing) From 5726cb9374bf31b8bc208f4703ee9109fb709367 Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 21 Jun 2021 00:55:45 +0200 Subject: [PATCH 06/23] aggregate builder and proposer for simplified typing and optimized verification --- specs/sharding/beacon-chain.md | 201 +++++++++++---------------------- 1 file changed, 63 insertions(+), 138 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index fe0c53bc4..ebfaa89eb 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -34,15 +34,8 @@ - [`ShardBlobHeader`](#shardblobheader) - [`SignedShardBlob`](#signedshardblob) - [`SignedShardBlobHeader`](#signedshardblobheader) - - [ShardBlock](#shardblock) - - [`ShardBlockHeader`](#shardblockheader) - - [`SignedShardBlock`](#signedshardblock) - - [`SignedShardBlockHeader`](#signedshardblockheader) - [`PendingShardHeader`](#pendingshardheader) - [`ShardBlobReference`](#shardblobreference) - - [`SignedShardBlobReference`](#signedshardblobreference) - - [`ShardBlockReference`](#shardblockreference) - - [`SignedShardBlockReference`](#signedshardblockreference) - [`ShardProposerSlashing`](#shardproposerslashing) - [`ShardWork`](#shardwork) - [Helper functions](#helper-functions) @@ -116,8 +109,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | | - | - | -| `DOMAIN_SHARD_PROPOSER` | `DomainType('0x80000000')` | -| `DOMAIN_SHARD_BUILDER` | `DomainType('0x81000000')` | +| `DOMAIN_SHARD_BLOB` | `DomainType('0x80000000')` | ### Shard Work Status @@ -247,8 +239,6 @@ class ShardBlobBody(Container): data: List[BLSPoint, POINTS_PER_SAMPLE * MAX_SAMPLES_PER_BLOCK] # Latest block root of the Beacon Chain, before shard_blob.slot beacon_block_root: Root - # Builder of the data, pays data-fee to proposer - builder_index: BuilderIndex # TODO: fee payment amount fields (EIP 1559 like) ``` @@ -269,8 +259,6 @@ class ShardBlobBodySummary(Container): data_root: Root # Latest block root of the Beacon Chain, before shard_blob.slot beacon_block_root: Root - # Builder of the data, pays data-fee to proposer - builder_index: BuilderIndex # TODO: fee payment amount fields (EIP 1559 like) ``` @@ -282,18 +270,26 @@ class ShardBlobBodySummary(Container): class ShardBlob(Container): slot: Slot shard: Shard + # Proposer of the shard-blob + proposer_index: ValidatorIndex + # Builder of the data, pays data-fee to proposer + builder_index: BuilderIndex # Blob contents body: ShardBlobBody ``` ### `ShardBlobHeader` -Header version of `ShardBlob`. Separates designation (slot, shard) and contents (blob). +Header version of `ShardBlob`. ```python class ShardBlobHeader(Container): slot: Slot shard: Shard + # Proposer of the shard-blob + proposer_index: ValidatorIndex + # Builder of the data, pays data-fee to proposer + builder_index: BuilderIndex # Blob contents, without the full data body_summary: ShardBlobBodySummary ``` @@ -316,53 +312,8 @@ Shard proposers can accept `SignedShardBlobHeader` as a data-transaction. ```python class SignedShardBlobHeader(Container): message: ShardBlobHeader - signature: BLSSignature -``` - -### ShardBlock - -Full blob data signed by builder, to be confirmed by proxy as `ShardBlockHeader`. - -```python -class ShardBlock(Container): - # Shard data with fee payment by bundle builder - signed_blob: SignedShardBlob - # Proposer of the shard-blob - proposer_index: ValidatorIndex -``` - -### `ShardBlockHeader` - -Header version of `ShardBlock`, selecting a `SignedShardBlobHeader`. - -```python -class ShardBlockHeader(Container): - # Shard commitments and fee payment by blob builder - signed_blob_header: SignedShardBlobHeader - # Proposer of the shard-blob - proposer_index: ValidatorIndex -``` - -### `SignedShardBlock` - -Shard blob, signed for payment, and signed for proposal. Propagated to attesters. - -```python -class SignedShardBlock(Container): - message: ShardBlock - signature: BLSSignature -``` - -### `SignedShardBlockHeader` - -Header version of `SignedShardBlock`, substituting the full data within the blob for just the hash-tree-root. - -The signature is equally applicable to `SignedShardBlock`, -which the builder can publish as soon as the signed header is seen. - -```python -class SignedShardBlockHeader(Container): - message: ShardBlockHeader + # Signature by builder. + # Once accepted by proposer, the signatures is the aggregate of both. signature: BLSSignature ``` @@ -390,44 +341,26 @@ Reference version of `ShardBlobHeader`, substituting the body for just a hash-tr class ShardBlobReference(Container): slot: Slot shard: Shard - # Blob hash-tree-root for reference, enough for uniqueness - body_root: Root -``` - -### `SignedShardBlobReference` - -`ShardBlobReference`, signed by the blob builder. The builder-signature is part of the block identity. - -```python -class SignedShardBlobReference(Container): - message: ShardBlobReference - signature: BLSSignature -``` - -### `ShardBlockReference` - -```python -class ShardBlockReference(Container): - # Blob, minimized for efficient slashing - signed_blob_reference: SignedShardBlobReference # Proposer of the shard-blob proposer_index: ValidatorIndex -``` - -### `SignedShardBlockReference` - -```python -class SignedShardBlockReference(Container): - message: ShardBlockReference - signature: BLSSignature + # Builder of the data + builder_index: BuilderIndex + # Blob hash-tree-root for slashing reference + body_root: Root ``` ### `ShardProposerSlashing` ```python class ShardProposerSlashing(Container): - signed_reference_1: SignedShardBlockReference - signed_reference_2: SignedShardBlockReference + slot: Slot + shard: Shard + proposer_index: ValidatorIndex + builder_index: BuilderIndex + body_root_1: Root + body_root_2: Root + signature_1: BLSSignature + signature_2: BLSSignature ``` ### `ShardWork` @@ -552,7 +485,7 @@ def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard Return the proposer's index of shard block at ``slot``. """ epoch = compute_epoch_at_slot(slot) - seed = hash(get_seed(beacon_state, epoch, DOMAIN_SHARD_PROPOSER) + uint_to_bytes(slot) + uint_to_bytes(shard)) + seed = hash(get_seed(beacon_state, epoch, DOMAIN_SHARD_BLOB) + uint_to_bytes(slot) + uint_to_bytes(shard)) # Proposer must have sufficient balance to pay for worst case fee burn EFFECTIVE_BALANCE_MAX_DOWNWARD_DEVIATION = ( @@ -717,12 +650,10 @@ def charge_builder(state: BeaconState, index: BuilderIndex, fee: Gwei) -> None: ##### `process_shard_header` ```python -def process_shard_header(state: BeaconState, signed_block_header: SignedShardBlockHeader) -> None: - block_header: ShardBlockHeader = signed_block_header.message - signed_blob_header: SignedShardBlobHeader = block_header.signed_blob_header - blob_header: ShardBlobHeader = signed_blob_header.message - slot = blob_header.slot - shard = blob_header.shard +def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeader) -> None: + header: ShardBlobHeader = signed_header.message + slot = header.slot + shard = header.shard # Verify the header is not 0, and not from the future. assert Slot(0) < slot <= state.slot @@ -733,7 +664,7 @@ def process_shard_header(state: BeaconState, signed_block_header: SignedShardBlo assert shard < get_active_shard_count(state, header_epoch) # Verify that the block root matches, # to ensure the header will only be included in this specific Beacon Chain sub-tree. - assert blob_header.body_summary.beacon_block_root == get_block_root_at_slot(state, slot - 1) + assert header.body_summary.beacon_block_root == get_block_root_at_slot(state, slot - 1) # Check that this data is still pending committee_work = state.shard_buffer[slot % SHARD_STATE_MEMORY_SLOTS][shard] @@ -741,24 +672,17 @@ def process_shard_header(state: BeaconState, signed_block_header: SignedShardBlo # Check that this header is not yet in the pending list current_headers: List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD] = committee_work.status.value - header_root = hash_tree_root(block_header) + header_root = hash_tree_root(header) assert header_root not in [pending_header.root for pending_header in current_headers] - # Verify proposer - assert block_header.proposer_index == get_shard_proposer_index(state, slot, shard) - # Verify proposer signature - block_signing_root = compute_signing_root(block_header, get_domain(state, DOMAIN_SHARD_PROPOSER)) - proposer_pubkey = state.validators[block_header.proposer_index].pubkey - assert bls.Verify(proposer_pubkey, block_signing_root, signed_block_header.signature) + # Verify proposer matches + assert header.proposer_index == get_shard_proposer_index(state, slot, shard) - # Verify builder requirements - blob_summary: ShardBlobBodySummary = blob_header.body_summary - builder_index = blob_summary.builder_index - - # Verify builder signature - builder = state.builders[builder_index] - blob_signing_root = compute_signing_root(blob_header, get_domain(state, DOMAIN_SHARD_BUILDER)) # TODO new constant - assert bls.Verify(builder.pubkey, blob_signing_root, signed_blob_header.signature) + # Verify builder and proposer aggregate signature + blob_signing_root = compute_signing_root(header, get_domain(state, DOMAIN_SHARD_BLOB)) + builder_pubkey = state.builders[header.builder_index].pubkey + proposer_pubkey = state.validators[header.proposer_index].pubkey + assert bls.FastAggregateVerify([builder_pubkey, proposer_pubkey], blob_signing_root, signed_header.signature) # Verify the length by verifying the degree. body_summary = header.body_summary @@ -771,10 +695,10 @@ def process_shard_header(state: BeaconState, signed_block_header: SignedShardBlo # Charge builder, with hard balance requirement fee = Gwei(123) # TODO EIP 1559 like fee? Burn some of it? - charge_builder(state, builder_index, fee) + charge_builder(state, header.builder_index, fee) # TODO: proposer is charged for confirmed headers (see charge_confirmed_shard_fees). # Need to align incentive, so proposer does not gain from including unconfirmed headers - increase_balance(state, block_header.proposer_index, fee) + increase_balance(state, blob_header.proposer_index, fee) # Initialize the pending header index = compute_committee_index_from_shard(state, slot, shard) @@ -801,32 +725,33 @@ The goal is to ensure that a proof can only be constructed if `deg(B) < l` (ther ```python def process_shard_proposer_slashing(state: BeaconState, proposer_slashing: ShardProposerSlashing) -> None: - reference_1: ShardBlockReference = proposer_slashing.signed_reference_1.message - reference_2 : ShardBlockReference = proposer_slashing.signed_reference_2.message - blob_1 = reference_1.signed_blob_reference.message - blob_2 = reference_2.signed_blob_reference.message + # Verify the headers are different + assert proposer_slashing.body_root_1 != proposer_slashing.body_root_2 + + slot = proposer_slashing.slot + shard = proposer_slashing.shard + proposer_index = proposer_slashing.proposer_index + builder_index = proposer_slashing.builder_index - # Verify header slots match - assert blob_1.slot == blob_2.slot - # Verify header shards match - assert blob_1.shard == blob_2.shard - # Verify header proposer indices match - assert reference_1.proposer_index == reference_2.proposer_index - # Verify the headers are different (i.e. different body, or different builder signature) - assert ( - blob_1.body_root != blob_2.body_root - or reference_1.signed_blob_reference.signature != reference_2.signed_blob_reference.signature - ) # Verify the proposer is slashable - proposer = state.validators[reference_1.proposer_index] + proposer = state.validators[proposer_index] assert is_slashable_validator(proposer, get_current_epoch(state)) - # Verify signatures - for signed_header in (proposer_slashing.signed_reference_1, proposer_slashing.signed_reference_2): - domain = get_domain(state, DOMAIN_SHARD_PROPOSER, compute_epoch_at_slot(signed_header.message.slot)) - signing_root = compute_signing_root(signed_header.message, domain) - assert bls.Verify(proposer.pubkey, signing_root, signed_header.signature) - slash_validator(state, reference_1.proposer_index) + reference_1 = ShardBlobReference(slot=slot, shard=shard, + proposer_index=proposer_index, builder_index=builder_index, + body_root= proposer_slashing.body_root_1) + reference_2 = ShardBlobReference(slot=slot, shard=shard, + proposer_index=proposer_index, builder_index=builder_index, + body_root= proposer_slashing.body_root_2) + proposer_pubkey = proposer.pubkey + builder_pubkey = state.builders[builder_index].pubkey + domain = get_domain(state, DOMAIN_SHARD_PROPOSER, compute_epoch_at_slot(slot)) + signing_root_1 = compute_signing_root(reference_1, domain) + signing_root_2 = compute_signing_root(reference_2, domain) + assert bls.FastAggregateVerify([builder_pubkey, proposer_pubkey], signing_root_1, proposer_slashing.signature_1) + assert bls.FastAggregateVerify([builder_pubkey, proposer_pubkey], signing_root_2, proposer_slashing.signature_2) + + slash_validator(state, proposer_index) ``` ### Epoch transition From 5034e2d7bc194613a11e63359f4dcff2d1dd993a Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 21 Jun 2021 23:33:10 +0200 Subject: [PATCH 07/23] update shard spec wording + fix shard slashings --- specs/sharding/beacon-chain.md | 54 ++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index ebfaa89eb..4bf5f7a03 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -78,8 +78,7 @@ using KZG10 commitments to commit to data to remove any need for fraud proofs (a - **Data**: A list of KZG points, to translate a byte string into - **Blob**: Data with commitments and meta-data, like a flattened bundle of L2 transactions. - **Builder**: Builds blobs and bids for proposal slots with fee-paying blob-headers, responsible for availability. -- **Shard proposer**: Validator, selects a signed blob-header, taking bids for shard data opportunity. -- **Shard block**: Unique per `(slot, shard, proposer)`, selected signed blob +- **Shard proposer**: Validator, taking bids for shard data opportunity, co-signs with builder to propose the blob. ## Custom types @@ -132,12 +131,12 @@ The following values are (non-configurable) constants used throughout the specif | `SHARD_STATE_MEMORY_SLOTS` | `uint64(2**8)` (= 256) | Number of slots for which shard commitments and confirmation status is directly available in the state | | `BUILDER_REGISTRY_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | builders | -### Shard block samples +### Shard blob samples | Name | Value | Notes | | - | - | - | -| `MAX_SAMPLES_PER_BLOCK` | `uint64(2**11)` (= 2,048) | 248 * 2,048 = 507,904 bytes | -| `TARGET_SAMPLES_PER_BLOCK` | `uint64(2**10)` (= 1,024) | 248 * 1,024 = 253,952 bytes | +| `MAX_SAMPLES_PER_BLOB` | `uint64(2**11)` (= 2,048) | 248 * 2,048 = 507,904 bytes | +| `TARGET_SAMPLES_PER_BLOB` | `uint64(2**10)` (= 1,024) | 248 * 1,024 = 253,952 bytes | ### Precomputed size verification points @@ -145,7 +144,7 @@ The following values are (non-configurable) constants used throughout the specif | - | - | | `G1_SETUP` | Type `List[G1]`. The G1-side trusted setup `[G, G*s, G*s**2....]`; note that the first point is the generator. | | `G2_SETUP` | Type `List[G2]`. The G2-side trusted setup `[G, G*s, G*s**2....]` | -| `ROOT_OF_UNITY` | `pow(PRIMITIVE_ROOT_OF_UNITY, (MODULUS - 1) // int(MAX_SAMPLES_PER_BLOCK * POINTS_PER_SAMPLE), MODULUS)` | +| `ROOT_OF_UNITY` | `pow(PRIMITIVE_ROOT_OF_UNITY, (MODULUS - 1) // int(MAX_SAMPLES_PER_BLOB * POINTS_PER_SAMPLE), MODULUS)` | ### Gwei values @@ -176,7 +175,7 @@ class AttestationData(Container): source: Checkpoint target: Checkpoint # Hash-tree-root of ShardBlock - shard_block_root: Root # [New in Sharding] + shard_blob_root: Root # [New in Sharding] ``` ### `BeaconBlockBody` @@ -236,7 +235,7 @@ class ShardBlobBody(Container): # Proof that the degree < commitment.length degree_proof: BLSCommitment # The actual data. Should match the commitment and degree proof. - data: List[BLSPoint, POINTS_PER_SAMPLE * MAX_SAMPLES_PER_BLOCK] + data: List[BLSPoint, POINTS_PER_SAMPLE * MAX_SAMPLES_PER_BLOB] # Latest block root of the Beacon Chain, before shard_blob.slot beacon_block_root: Root # TODO: fee payment amount fields (EIP 1559 like) @@ -296,7 +295,7 @@ class ShardBlobHeader(Container): ### `SignedShardBlob` -Full blob data, signed by the shard builder, ensuring fee payment. +Full blob data, signed by the shard builder (ensuring fee payment) and shard proposer (ensuring a single proposal). ```python class SignedShardBlob(Container): @@ -307,7 +306,7 @@ class SignedShardBlob(Container): ### `SignedShardBlobHeader` Header of the blob, the signature is equally applicable to `SignedShardBlob`. -Shard proposers can accept `SignedShardBlobHeader` as a data-transaction. +Shard proposers can accept `SignedShardBlobHeader` as a data-transaction by co-signing the header. ```python class SignedShardBlobHeader(Container): @@ -356,7 +355,8 @@ class ShardProposerSlashing(Container): slot: Slot shard: Shard proposer_index: ValidatorIndex - builder_index: BuilderIndex + builder_index_1: BuilderIndex + builder_index_2: BuilderIndex body_root_1: Root body_root_2: Root signature_1: BLSSignature @@ -400,13 +400,13 @@ def compute_previous_slot(slot: Slot) -> Slot: ```python def compute_updated_gasprice(prev_gasprice: Gwei, shard_block_length: uint64, adjustment_quotient: uint64) -> Gwei: - if shard_block_length > TARGET_SAMPLES_PER_BLOCK: - delta = max(1, prev_gasprice * (shard_block_length - TARGET_SAMPLES_PER_BLOCK) - // TARGET_SAMPLES_PER_BLOCK // adjustment_quotient) + if shard_block_length > TARGET_SAMPLES_PER_BLOB: + delta = max(1, prev_gasprice * (shard_block_length - TARGET_SAMPLES_PER_BLOB) + // TARGET_SAMPLES_PER_BLOB // adjustment_quotient) return min(prev_gasprice + delta, MAX_GASPRICE) else: - delta = max(1, prev_gasprice * (TARGET_SAMPLES_PER_BLOCK - shard_block_length) - // TARGET_SAMPLES_PER_BLOCK // adjustment_quotient) + delta = max(1, prev_gasprice * (TARGET_SAMPLES_PER_BLOB - shard_block_length) + // TARGET_SAMPLES_PER_BLOB // adjustment_quotient) return max(prev_gasprice, MIN_GASPRICE + delta) - delta ``` @@ -493,7 +493,7 @@ def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard * HYSTERESIS_DOWNWARD_MULTIPLIER // HYSTERESIS_QUOTIENT ) min_effective_balance = ( - beacon_state.shard_gasprice * MAX_SAMPLES_PER_BLOCK // TARGET_SAMPLES_PER_BLOCK + beacon_state.shard_gasprice * MAX_SAMPLES_PER_BLOB // TARGET_SAMPLES_PER_BLOB + EFFECTIVE_BALANCE_MAX_DOWNWARD_DEVIATION ) indices = get_active_validator_indices(state, epoch) @@ -597,7 +597,7 @@ def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> N current_headers: Sequence[PendingShardHeader] = committee_work.status.value # Find the corresponding header, abort if it cannot be found - header_index = [header.root for header in current_headers].index(attestation.data.shard_block_root) + header_index = [header.root for header in current_headers].index(attestation.data.shard_blob_root) pending_header: PendingShardHeader = current_headers[header_index] full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) @@ -731,25 +731,27 @@ def process_shard_proposer_slashing(state: BeaconState, proposer_slashing: Shard slot = proposer_slashing.slot shard = proposer_slashing.shard proposer_index = proposer_slashing.proposer_index - builder_index = proposer_slashing.builder_index # Verify the proposer is slashable proposer = state.validators[proposer_index] assert is_slashable_validator(proposer, get_current_epoch(state)) reference_1 = ShardBlobReference(slot=slot, shard=shard, - proposer_index=proposer_index, builder_index=builder_index, + proposer_index=proposer_index, + builder_index=proposer_slashing.builder_index_1, body_root= proposer_slashing.body_root_1) reference_2 = ShardBlobReference(slot=slot, shard=shard, - proposer_index=proposer_index, builder_index=builder_index, + proposer_index=proposer_index, + builder_index=proposer_slashing.builder_index_1, body_root= proposer_slashing.body_root_2) - proposer_pubkey = proposer.pubkey - builder_pubkey = state.builders[builder_index].pubkey + # The builders are not slashed, the proposer co-signed with them + builder_pubkey_1 = state.builders[proposer_slashing.builder_index_1].pubkey + builder_pubkey_2 = state.builders[proposer_slashing.builder_index_2].pubkey domain = get_domain(state, DOMAIN_SHARD_PROPOSER, compute_epoch_at_slot(slot)) signing_root_1 = compute_signing_root(reference_1, domain) signing_root_2 = compute_signing_root(reference_2, domain) - assert bls.FastAggregateVerify([builder_pubkey, proposer_pubkey], signing_root_1, proposer_slashing.signature_1) - assert bls.FastAggregateVerify([builder_pubkey, proposer_pubkey], signing_root_2, proposer_slashing.signature_2) + assert bls.FastAggregateVerify([builder_pubkey_1, proposer.pubkey], signing_root_1, proposer_slashing.signature_1) + assert bls.FastAggregateVerify([builder_pubkey_2, proposer.pubkey], signing_root_2, proposer_slashing.signature_2) slash_validator(state, proposer_index) ``` @@ -828,7 +830,7 @@ def charge_confirmed_shard_fees(state: BeaconState) -> None: proposer = get_shard_proposer_index(state, slot, Shard(shard_index)) fee = ( (state.shard_gasprice * commitment.length) - // TARGET_SAMPLES_PER_BLOCK + // TARGET_SAMPLES_PER_BLOB ) decrease_balance(state, proposer, fee) From b25afc88fd2686f9d9e94447b7db6f3e25d391da Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 22 Jun 2021 00:10:47 +0200 Subject: [PATCH 08/23] update networking spec with aggregate proposer/builder types, update TOCs --- specs/sharding/beacon-chain.md | 2 +- specs/sharding/p2p-interface.md | 140 +++++++++++++++++++------------- 2 files changed, 83 insertions(+), 59 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 4bf5f7a03..6378389a7 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -17,7 +17,7 @@ - [Shard Work Status](#shard-work-status) - [Preset](#preset) - [Misc](#misc-1) - - [Shard block samples](#shard-block-samples) + - [Shard blob samples](#shard-blob-samples) - [Precomputed size verification points](#precomputed-size-verification-points) - [Gwei values](#gwei-values) - [Configuration](#configuration) diff --git a/specs/sharding/p2p-interface.md b/specs/sharding/p2p-interface.md index e9b561312..4eb3f6f2e 100644 --- a/specs/sharding/p2p-interface.md +++ b/specs/sharding/p2p-interface.md @@ -13,10 +13,11 @@ - [Misc](#misc) - [Gossip domain](#gossip-domain) - [Topics and messages](#topics-and-messages) - - [Shard block subnets](#shard-block-subnets) - - [`shard_block_{subnet_id}`](#shard_block_subnet_id) + - [Shard blob subnets](#shard-blob-subnets) + - [`shard_blob_{subnet_id}`](#shard_blob_subnet_id) - [Global topics](#global-topics) - - [`shard_block_header`](#shard_block_header) + - [`shard_blob_header`](#shard_blob_header) + - [`shard_blob_tx`](#shard_blob_tx) - [`shard_proposer_slashing`](#shard_proposer_slashing) @@ -34,7 +35,7 @@ The adjustments and additions for Shards are outlined in this document. | Name | Value | Description | | ---- | ----- | ----------- | -| `SHARD_BLOCK_SUBNET_COUNT` | `64` | The number of `shard_block_{subnet_id}` subnets used in the gossipsub protocol. | +| `SHARD_BLOB_SUBNET_COUNT` | `64` | The number of `shard_blob_{subnet_id}` subnets used in the gossipsub protocol. | ## Gossip domain @@ -42,24 +43,25 @@ The adjustments and additions for Shards are outlined in this document. Following the same scheme as the [Phase0 gossip topics](../phase0/p2p-interface.md#topics-and-messages), names and payload types are: -| Name | Message Type | -|----------------------------------|---------------------------| -| `shard_block_{subnet_id}` | `SignedShardBlock` | -| `shard_block_header` | `SignedShardBlockHeader` | -| `shard_proposer_slashing` | `ShardProposerSlashing` | +| Name | Message Type | +|---------------------------------|--------------------------| +| `shard_blob_{subnet_id}` | `SignedShardBlob` | +| `shard_blob_header` | `SignedShardBlobHeader` | +| `shard_blob_tx` | `SignedShardBlobHeader` | +| `shard_proposer_slashing` | `ShardProposerSlashing` | The [DAS network specification](./das-p2p.md) defines additional topics. -#### Shard block subnets +#### Shard blob subnets -Shard block subnets are used by builders to make their blobs available after selection by shard proposers. +Shard blob subnets are used by builders to make their blobs available after selection by shard proposers. -##### `shard_block_{subnet_id}` +##### `shard_blob_{subnet_id}` -Shard block data, in the form of a `SignedShardBlock` is published to the `shard_block_{subnet_id}` subnets. +Shard blob data, in the form of a `SignedShardBlob` is published to the `shard_blob_{subnet_id}` subnets. ```python -def compute_subnet_for_shard_block(state: BeaconState, slot: Slot, shard: Shard) -> uint64: +def compute_subnet_for_shard_blob(state: BeaconState, slot: Slot, shard: Shard) -> uint64: """ Compute the correct subnet for a shard blob publication. Note, this mimics compute_subnet_for_attestation(). @@ -69,75 +71,97 @@ def compute_subnet_for_shard_block(state: BeaconState, slot: Slot, shard: Shard) slots_since_epoch_start = Slot(slot % SLOTS_PER_EPOCH) committees_since_epoch_start = committees_per_slot * slots_since_epoch_start - return uint64((committees_since_epoch_start + committee_index) % SHARD_BLOCK_SUBNET_COUNT) + return uint64((committees_since_epoch_start + committee_index) % SHARD_BLOB_SUBNET_COUNT) ``` -The following validations MUST pass before forwarding the `signed_block` on the horizontal subnet or creating samples for it. +The following validations MUST pass before forwarding the `signed_blob`, +on the horizontal subnet or creating samples for it. Alias `blob = signed_blob.message`. -We define some aliases to the nested contents of `signed_block`: -```python -block: ShardBlock = signed_block.message -signed_blob: SignedShardBlob = block.signed_blob -blob: ShardBlob = signed_blob.message -``` - -- _[IGNORE]_ The `block` is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- +- _[IGNORE]_ The `blob` is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `blob.slot <= current_slot` (a client MAY queue future blobs for processing at the appropriate slot). -- _[IGNORE]_ The `blob` is new enough to be still be processed -- +- _[IGNORE]_ The `blob` is new enough to still be processed -- i.e. validate that `compute_epoch_at_slot(blob.slot) >= get_previous_epoch(state)` -- _[REJECT]_ The shard should have a committee at slot -- +- _[REJECT]_ The shard blob is for an active shard -- + i.e. `blob.shard < get_active_shard_count(state, compute_epoch_at_slot(blob.slot))` +- _[REJECT]_ The `blob.shard` MUST have a committee at the `blob.slot` -- i.e. validate that `compute_committee_index_from_shard(state, blob.slot, blob.shard)` doesn't raise an error - _[REJECT]_ The shard blob is for the correct subnet -- - i.e. `compute_subnet_for_shard_block(state, blob.slot, blob.shard) == subnet_id` -- _[IGNORE]_ The block is the first block with valid signature received for the `(block.proposer_index, blob.slot, blob.shard)` combination. + i.e. `compute_subnet_for_shard_blob(state, blob.slot, blob.shard) == subnet_id` +- _[IGNORE]_ The blob is the first blob with valid signature received for the `(blob.proposer_index, blob.slot, blob.shard)` combination. - _[REJECT]_ The blob is not too large, the data MUST NOT be larger than the SSZ list-limit, and a client MAY be more strict. - _[REJECT]_ The `blob.body.data` MUST NOT contain any point `p >= MODULUS`. Although it is a `uint256`, not the full 256 bit range is valid. -- _[REJECT]_ The block proposer signature, `signed_block.signature`, is valid with respect to the `proposer_index` pubkey. - _[REJECT]_ The blob builder exists and has sufficient balance to back the fee payment. -- _[REJECT]_ The blob builder signature, `signed_blob.signature`, is valid with respect to the `builder_index` pubkey. -- _[REJECT]_ The block is proposed by the expected `proposer_index` for the block's slot +- _[REJECT]_ The blob signature is valid for the aggregate of proposer and builder, `signed_blob.signature`, + i.e. `bls.FastAggregateVerify([builder_pubkey, proposer_pubkey], blob_signing_root, signed_blob.signature)`. +- _[REJECT]_ The blob is proposed by the expected `proposer_index` for the blob's `slot` and `shard`, in the context of the current shuffling (defined by `blob.body.beacon_block_root`/`slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, - the block MAY be queued for later processing while proposers for the block's branch are calculated -- + the blob MAY be queued for later processing while proposers for the blob's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message. #### Global topics -There are two additional global topics for Sharding. +There are three additional global topics for Sharding. -One is used to propagate shard block headers (`shard_block_header`) to all nodes on the network. -Another one is used to propagate shard proposer slashings (`shard_proposer_slashing`). +- `shard_blob_header`: co-signed headers, to be included on-chain, and signaling builders to publish full data. +- `shard_blob_tx`: builder-signed headers, also known as "data transaction". +- `shard_proposer_slashing`: slashings of duplicate shard proposals -##### `shard_block_header` +##### `shard_blob_header` -Shard header data, in the form of a `SignedShardBlockHeader` is published to the global `shard_block_header` subnet. -Shard block headers select shard blob bids by builders, and should be timely to ensure builders can publish the full shard block timely. +Shard header data, in the form of a `SignedShardBlobHeader` is published to the global `shard_blob_header` subnet. +Shard blob headers select shard blob bids by builders, +and should be timely to ensure builders can publish the full shard blob before subsequent attestations. -The following validations MUST pass before forwarding the `signed_block_header` (with inner `message` as `header`) on the network. +The following validations MUST pass before forwarding the `signed_blob_header` on the network. Alias `header = signed_blob_header.message` -We define some aliases to the nested contents of `signed_block_header`: -```python -block_header: ShardBlockHeader = signed_block_header.message -signed_blob_header: SignedShardBlobHeader = header.signed_blob_header -blob_header: ShardBlobHeader = signed_blob_header.message -``` - -- _[IGNORE]_ The header is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- - i.e. validate that `blob_header.slot <= current_slot` +- _[IGNORE]_ The `header` is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- + i.e. validate that `header.slot <= current_slot` (a client MAY queue future headers for processing at the appropriate slot). -- _[IGNORE]_ The header is new enough to be still be processed -- - i.e. validate that `compute_epoch_at_slot(blob_header.slot) >= get_previous_epoch(state)` -- _[IGNORE]_ The header is the first header with valid signature received for the `(block_header.proposer_index, blob_header.slot, blob_header.shard)` combination. -- _[REJECT]_ The `shard` MUST have a committee at the `slot` -- - i.e. validate that `compute_committee_index_from_shard(state, blob_header.slot, blob_header.shard)` doesn't raise an error -- _[REJECT]_ The proposer signature, `signed_shard_block_header.signature`, is valid with respect to the `block_header.proposer_index` pubkey. -- _[REJECT]_ The header is proposed by the expected `proposer_index` for the block's slot +- _[IGNORE]_ The header is new enough to still be processed -- + i.e. validate that `compute_epoch_at_slot(header.slot) >= get_previous_epoch(state)` +- _[REJECT]_ The shard header is for an active shard -- + i.e. `header.shard < get_active_shard_count(state, compute_epoch_at_slot(header.slot))` +- _[REJECT]_ The `header.shard` MUST have a committee at the `header.slot` -- + i.e. validate that `compute_committee_index_from_shard(state, header.slot, header.shard)` doesn't raise an error +- _[IGNORE]_ The header is the first header with valid signature received for the `(header.proposer_index, header.slot, header.shard)` combination. +- _[REJECT]_ The blob builder exists and has sufficient balance to back the fee payment. +- _[REJECT]_ The header signature is valid for the aggregate of proposer and builder, `signed_blob_header.signature`, + i.e. `bls.FastAggregateVerify([builder_pubkey, proposer_pubkey], blob_signing_root, signed_blob_header.signature)`. +- _[REJECT]_ The header is proposed by the expected `proposer_index` for the blob's `header.slot` and `header.shard` in the context of the current shuffling (defined by `header.body_summary.beacon_block_root`/`slot`). If the `proposer_index` cannot immediately be verified against the expected shuffling, - the block MAY be queued for later processing while proposers for the block's branch are calculated -- + the blob MAY be queued for later processing while proposers for the blob's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message. +##### `shard_blob_tx` + +Shard data-transactions, in the form of a `SignedShardBlobHeader` is published to the global `shard_blob_tx` subnet. +These shard blob headers are signed solely by the blob-builder. + +The following validations MUST pass before forwarding the `signed_blob_header` on the network. Alias `header = signed_blob_header.message` + +- _[IGNORE]_ The `header` is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- + i.e. validate that `header.slot <= current_slot` + (a client MAY queue future headers for processing at the appropriate slot). +- _[IGNORE]_ The header is new enough to still be processed -- + i.e. validate that `compute_epoch_at_slot(header.slot) >= get_previous_epoch(state)` +- _[REJECT]_ The shard header is for an active shard -- + i.e. `header.shard < get_active_shard_count(state, compute_epoch_at_slot(header.slot))` +- _[REJECT]_ The `header.shard` MUST have a committee at the `header.slot` -- + i.e. validate that `compute_committee_index_from_shard(state, header.slot, header.shard)` doesn't raise an error +- _[IGNORE]_ The header is the first header with valid signature received for the `(header.builder_index, header.slot, header.shard)` combination. +- _[REJECT]_ The blob builder exists and has sufficient balance to back the fee payment. +- _[IGNORE]_ The header fee SHOULD be higher than previously seen headers for `(header.slot, header.shard)`, from any builder. + Propagating nodes MAY increase fee increments in case of spam. +- _[REJECT]_ The header signature is valid for ONLY the builder, `signed_blob_header.signature`, + i.e. `bls.Verify(builder_pubkey, blob_signing_root, signed_blob_header.signature)`. The signature is not an aggregate with the proposer. +- _[REJECT]_ The header is designated for proposal by the expected `proposer_index` for the blob's `header.slot` and `header.shard` + in the context of the current shuffling (defined by `header.body_summary.beacon_block_root`/`slot`). + If the `proposer_index` cannot immediately be verified against the expected shuffling, + the blob MAY be queued for later processing while proposers for the blob's branch are calculated -- + in such a case _do not_ `REJECT`, instead `IGNORE` this message. ##### `shard_proposer_slashing` @@ -145,6 +169,6 @@ Shard proposer slashings, in the form of `ShardProposerSlashing`, are published The following validations MUST pass before forwarding the `shard_proposer_slashing` on to the network. - _[IGNORE]_ The shard proposer slashing is the first valid shard proposer slashing received - for the proposer with index `proposer_slashing.signed_reference_1.message.proposer_index`. - The `slot` and `shard` are ignored, there are no per-shard slashings. + for the proposer with index `proposer_slashing.proposer_index`. + The `proposer_slashing.slot` and `proposer_slashing.shard` are ignored, there are no repeated or per-shard slashings. - _[REJECT]_ All of the conditions within `process_shard_proposer_slashing` pass validation. From f791fe7d1c322f3e8f2121f6c7a1abd675652085 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 23 Jun 2021 23:33:46 +0200 Subject: [PATCH 09/23] implement review suggestions Co-authored-by: Anton Nashatyrev --- specs/sharding/beacon-chain.md | 89 ++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 6378389a7..63b9457c4 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -24,9 +24,9 @@ - [Updated containers](#updated-containers) - [`AttestationData`](#attestationdata) - [`BeaconBlockBody`](#beaconblockbody) - - [`Builder`](#builder) - [`BeaconState`](#beaconstate) - [New containers](#new-containers) + - [`Builder`](#builder) - [`DataCommitment`](#datacommitment) - [ShardBlobBody](#shardblobbody) - [`ShardBlobBodySummary`](#shardblobbodysummary) @@ -77,8 +77,8 @@ using KZG10 commitments to commit to data to remove any need for fraud proofs (a - **Data**: A list of KZG points, to translate a byte string into - **Blob**: Data with commitments and meta-data, like a flattened bundle of L2 transactions. -- **Builder**: Builds blobs and bids for proposal slots with fee-paying blob-headers, responsible for availability. -- **Shard proposer**: Validator, taking bids for shard data opportunity, co-signs with builder to propose the blob. +- **Builder**: Independent actor that builds blobs and bids for proposal slots via fee-paying blob-headers, responsible for availability. +- **Shard proposer**: Validator taking bids from blob builders for shard data opportunity, co-signs with builder to propose the blob. ## Custom types @@ -129,7 +129,7 @@ The following values are (non-configurable) constants used throughout the specif | `MAX_SHARD_PROPOSER_SLASHINGS` | `2**4` (= 16) | Maximum amount of shard proposer slashing operations per block | | `MAX_SHARD_HEADERS_PER_SHARD` | `4` | | | `SHARD_STATE_MEMORY_SLOTS` | `uint64(2**8)` (= 256) | Number of slots for which shard commitments and confirmation status is directly available in the state | -| `BUILDER_REGISTRY_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | builders | +| `BLOB_BUILDER_REGISTRY_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | shard blob builders | ### Shard blob samples @@ -174,7 +174,7 @@ class AttestationData(Container): # FFG vote source: Checkpoint target: Checkpoint - # Hash-tree-root of ShardBlock + # Hash-tree-root of ShardBlob shard_blob_root: Root # [New in Sharding] ``` @@ -183,16 +183,7 @@ class AttestationData(Container): ```python class BeaconBlockBody(merge.BeaconBlockBody): # [extends The Merge block body] shard_proposer_slashings: List[ShardProposerSlashing, MAX_SHARD_PROPOSER_SLASHINGS] - shard_headers: List[SignedShardBlockHeader, MAX_SHARDS * MAX_SHARD_HEADERS_PER_SHARD] -``` - -### `Builder` - -```python -class Builder(Container): - pubkey: BLSPubkey - # TODO: fields for either an expiry mechanism (refunding execution account with remaining balance) - # and/or a builder-transaction mechanism. + shard_headers: List[SignedShardBlobHeader, MAX_SHARDS * MAX_SHARD_HEADERS_PER_SHARD] ``` ### `BeaconState` @@ -203,9 +194,9 @@ class BeaconState(merge.BeaconState): previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] # [New fields] - # Builder registry. - builders: List[Builder, BUILDER_REGISTRY_LIMIT] - builder_balances: List[Gwei, BUILDER_REGISTRY_LIMIT] + # Blob builder registry. + blob_builders: List[Builder, BLOB_BUILDER_REGISTRY_LIMIT] + blob_builder_balances: List[Gwei, BLOB_BUILDER_REGISTRY_LIMIT] # A ring buffer of the latest slots, with information per active shard. shard_buffer: Vector[List[ShardWork, MAX_SHARDS], SHARD_STATE_MEMORY_SLOTS] shard_gasprice: uint64 @@ -213,6 +204,15 @@ class BeaconState(merge.BeaconState): ## New containers +### `Builder` + +```python +class Builder(Container): + pubkey: BLSPubkey + # TODO: fields for either an expiry mechanism (refunding execution account with remaining balance) + # and/or a builder-transaction mechanism. +``` + ### `DataCommitment` ```python @@ -269,10 +269,10 @@ class ShardBlobBodySummary(Container): class ShardBlob(Container): slot: Slot shard: Shard - # Proposer of the shard-blob - proposer_index: ValidatorIndex # Builder of the data, pays data-fee to proposer builder_index: BuilderIndex + # Proposer of the shard-blob + proposer_index: ValidatorIndex # Blob contents body: ShardBlobBody ``` @@ -285,10 +285,10 @@ Header version of `ShardBlob`. class ShardBlobHeader(Container): slot: Slot shard: Shard - # Proposer of the shard-blob - proposer_index: ValidatorIndex # Builder of the data, pays data-fee to proposer builder_index: BuilderIndex + # Proposer of the shard-blob + proposer_index: ValidatorIndex # Blob contents, without the full data body_summary: ShardBlobBodySummary ``` @@ -322,7 +322,7 @@ class SignedShardBlobHeader(Container): class PendingShardHeader(Container): # KZG10 commitment to the data commitment: DataCommitment - # hash_tree_root of the ShardBlockHeader (stored so that attestations can be checked against it) + # hash_tree_root of the ShardBlobHeader (stored so that attestations can be checked against it) root: Root # Who voted for the header votes: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] @@ -340,10 +340,10 @@ Reference version of `ShardBlobHeader`, substituting the body for just a hash-tr class ShardBlobReference(Container): slot: Slot shard: Shard - # Proposer of the shard-blob - proposer_index: ValidatorIndex # Builder of the data builder_index: BuilderIndex + # Proposer of the shard-blob + proposer_index: ValidatorIndex # Blob hash-tree-root for slashing reference body_root: Root ``` @@ -643,8 +643,12 @@ def charge_builder(state: BeaconState, index: BuilderIndex, fee: Gwei) -> None: """ Decrease the builder balance at index ``index`` by ``fee``, with underflow check. """ - assert state.builder_balances[index] >= fee - state.builder_balances[index] -= fee + # TODO: apply stricter requirement to protect against fee-acceptance race conditions, e.g.: + # - balance per shard (or builders per shard) + # - balance / shard_count > fee + # TODO: also consider requirement to pay for base-fee of shard-data + assert state.blob_builder_balances[index] >= fee + state.blob_builder_balances[index] -= fee ``` ##### `process_shard_header` @@ -680,7 +684,7 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade # Verify builder and proposer aggregate signature blob_signing_root = compute_signing_root(header, get_domain(state, DOMAIN_SHARD_BLOB)) - builder_pubkey = state.builders[header.builder_index].pubkey + builder_pubkey = state.blob_builders[header.builder_index].pubkey proposer_pubkey = state.validators[header.proposer_index].pubkey assert bls.FastAggregateVerify([builder_pubkey, proposer_pubkey], blob_signing_root, signed_header.signature) @@ -698,7 +702,7 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade charge_builder(state, header.builder_index, fee) # TODO: proposer is charged for confirmed headers (see charge_confirmed_shard_fees). # Need to align incentive, so proposer does not gain from including unconfirmed headers - increase_balance(state, blob_header.proposer_index, fee) + increase_balance(state, header.proposer_index, fee) # Initialize the pending header index = compute_committee_index_from_shard(state, slot, shard) @@ -725,28 +729,29 @@ The goal is to ensure that a proof can only be constructed if `deg(B) < l` (ther ```python def process_shard_proposer_slashing(state: BeaconState, proposer_slashing: ShardProposerSlashing) -> None: - # Verify the headers are different - assert proposer_slashing.body_root_1 != proposer_slashing.body_root_2 - slot = proposer_slashing.slot shard = proposer_slashing.shard proposer_index = proposer_slashing.proposer_index + reference_1 = ShardBlobReference(slot=slot, shard=shard, + proposer_index=proposer_index, + builder_index=proposer_slashing.builder_index_1, + body_root=proposer_slashing.body_root_1) + reference_2 = ShardBlobReference(slot=slot, shard=shard, + proposer_index=proposer_index, + builder_index=proposer_slashing.builder_index_2, + body_root=proposer_slashing.body_root_2) + + # Verify the signed messages are different + assert reference_1 != reference_2 + # Verify the proposer is slashable proposer = state.validators[proposer_index] assert is_slashable_validator(proposer, get_current_epoch(state)) - reference_1 = ShardBlobReference(slot=slot, shard=shard, - proposer_index=proposer_index, - builder_index=proposer_slashing.builder_index_1, - body_root= proposer_slashing.body_root_1) - reference_2 = ShardBlobReference(slot=slot, shard=shard, - proposer_index=proposer_index, - builder_index=proposer_slashing.builder_index_1, - body_root= proposer_slashing.body_root_2) # The builders are not slashed, the proposer co-signed with them - builder_pubkey_1 = state.builders[proposer_slashing.builder_index_1].pubkey - builder_pubkey_2 = state.builders[proposer_slashing.builder_index_2].pubkey + builder_pubkey_1 = state.blob_builders[proposer_slashing.builder_index_1].pubkey + builder_pubkey_2 = state.blob_builders[proposer_slashing.builder_index_2].pubkey domain = get_domain(state, DOMAIN_SHARD_PROPOSER, compute_epoch_at_slot(slot)) signing_root_1 = compute_signing_root(reference_1, domain) signing_root_2 = compute_signing_root(reference_2, domain) From a7f58ef08acb81a88b279b449335faebc6c717b2 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 14 Jul 2021 13:19:00 +0200 Subject: [PATCH 10/23] fix comment + handle missing pending headers --- specs/sharding/beacon-chain.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 63b9457c4..2ab0f68cd 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -367,7 +367,7 @@ class ShardProposerSlashing(Container): ```python class ShardWork(Container): - # Upon confirmation the data is reduced to just the header. + # Upon confirmation the data is reduced to just the commitment. status: Union[ # See Shard Work Status enum None, # SHARD_WORK_UNCONFIRMED DataCommitment, # SHARD_WORK_CONFIRMED @@ -597,7 +597,15 @@ def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> N current_headers: Sequence[PendingShardHeader] = committee_work.status.value # Find the corresponding header, abort if it cannot be found - header_index = [header.root for header in current_headers].index(attestation.data.shard_blob_root) + header_index = len(current_headers) + for i, header in enumerate(current_headers): + if attestation.data.shard_blob_root == header.root: + header_index = i + break + # Attestations for an unknown header do not count towards shard confirmations, but can otherwise be valid. + if header_index == len(current_headers): + # TODO: Attestations may be re-included if headers are included late. + return pending_header: PendingShardHeader = current_headers[header_index] full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) From 1a966d1e378dc719aa12f6eacae854f531eab9fe Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 26 Jul 2021 15:26:55 +0200 Subject: [PATCH 11/23] work in progress new sharding fee mechanism --- specs/sharding/beacon-chain.md | 297 +++++++++++++++++---------------- 1 file changed, 156 insertions(+), 141 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 2ab0f68cd..40eabe548 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -15,8 +15,11 @@ - [Misc](#misc) - [Domain types](#domain-types) - [Shard Work Status](#shard-work-status) -- [Preset](#preset) - [Misc](#misc-1) + - [Participation flag indices](#participation-flag-indices) + - [Incentivization weights](#incentivization-weights) +- [Preset](#preset) + - [Misc](#misc-2) - [Shard blob samples](#shard-blob-samples) - [Precomputed size verification points](#precomputed-size-verification-points) - [Gwei values](#gwei-values) @@ -28,6 +31,7 @@ - [New containers](#new-containers) - [`Builder`](#builder) - [`DataCommitment`](#datacommitment) + - [`AttestedDataCommitment`](#attesteddatacommitment) - [ShardBlobBody](#shardblobbody) - [`ShardBlobBodySummary`](#shardblobbodysummary) - [`ShardBlob`](#shardblob) @@ -39,15 +43,15 @@ - [`ShardProposerSlashing`](#shardproposerslashing) - [`ShardWork`](#shardwork) - [Helper functions](#helper-functions) - - [Misc](#misc-2) + - [Misc](#misc-3) - [`next_power_of_two`](#next_power_of_two) - [`compute_previous_slot`](#compute_previous_slot) - - [`compute_updated_gasprice`](#compute_updated_gasprice) + - [`compute_updated_sample_price`](#compute_updated_sample_price) - [`compute_committee_source_epoch`](#compute_committee_source_epoch) + - [`batch_apply_participation_flag`](#batch_apply_participation_flag) - [Beacon state accessors](#beacon-state-accessors) - [Updated `get_committee_count_per_slot`](#updated-get_committee_count_per_slot) - [`get_active_shard_count`](#get_active_shard_count) - - [`compute_proposer_index`](#compute_proposer_index) - [`get_shard_proposer_index`](#get_shard_proposer_index) - [`get_start_shard`](#get_start_shard) - [`compute_shard_from_committee_index`](#compute_shard_from_committee_index) @@ -55,12 +59,10 @@ - [Block processing](#block-processing) - [Operations](#operations) - [Extended Attestation processing](#extended-attestation-processing) - - [`charge_builder`](#charge_builder) - [`process_shard_header`](#process_shard_header) - [`process_shard_proposer_slashing`](#process_shard_proposer_slashing) - [Epoch transition](#epoch-transition) - [`process_pending_shard_confirmations`](#process_pending_shard_confirmations) - - [`charge_confirmed_shard_fees`](#charge_confirmed_shard_fees) - [`reset_pending_shard_work`](#reset_pending_shard_work) @@ -118,6 +120,30 @@ The following values are (non-configurable) constants used throughout the specif | `SHARD_WORK_CONFIRMED` | `1` | Confirmed, reduced to just the commitment | | `SHARD_WORK_PENDING` | `2` | Pending, a list of competing headers | +### Misc + +TODO: `PARTICIPATION_FLAG_WEIGHTS` backwards-compatibility is difficult, depends on usage. + +| Name | Value | +| - | - | +| `PARTICIPATION_FLAG_WEIGHTS` | `[TIMELY_SOURCE_WEIGHT, TIMELY_TARGET_WEIGHT, TIMELY_HEAD_WEIGHT, TIMELY_SHARD_WEIGHT]` | + +### Participation flag indices + +| Name | Value | +| - | - | +| `TIMELY_SHARD_FLAG_INDEX` | `3` | + +### Incentivization weights + +TODO: determine weight for shard attestations + +| Name | Value | +| - | - | +| `TIMELY_SHARD_WEIGHT` | `uint64(8)` | + +TODO: `WEIGHT_DENOMINATOR` needs to be adjusted, but this breaks a lot of Altair code. + ## Preset ### Misc @@ -125,7 +151,7 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | Notes | | - | - | - | | `MAX_SHARDS` | `uint64(2**10)` (= 1,024) | Theoretical max shard count (used to determine data structure sizes) | -| `GASPRICE_ADJUSTMENT_COEFFICIENT` | `uint64(2**3)` (= 8) | Gasprice may decrease/increase by at most exp(1 / this value) *per epoch* | +| `SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT` | `uint64(2**3)` (= 8) | Sample price may decrease/increase by at most exp(1 / this value) *per epoch* | | `MAX_SHARD_PROPOSER_SLASHINGS` | `2**4` (= 16) | Maximum amount of shard proposer slashing operations per block | | `MAX_SHARD_HEADERS_PER_SHARD` | `4` | | | `SHARD_STATE_MEMORY_SLOTS` | `uint64(2**8)` (= 256) | Number of slots for which shard commitments and confirmation status is directly available in the state | @@ -150,8 +176,8 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | Unit | Description | | - | - | - | - | -| `MAX_GASPRICE` | `Gwei(2**33)` (= 8,589,934,592) | Gwei | Max gasprice charged for a TARGET-sized shard block | -| `MIN_GASPRICE` | `Gwei(2**3)` (= 8) | Gwei | Min gasprice charged for a TARGET-sized shard block | +| `MAX_SAMPLE_PRICE` | `Gwei(2**33)` (= 8,589,934,592) | Gwei | Max sample charged for a TARGET-sized shard blob | +| `MIN_SAMPLE_PRICE` | `Gwei(2**3)` (= 8) | Gwei | Min sample price charged for a TARGET-sized shard blob | ## Configuration @@ -199,7 +225,7 @@ class BeaconState(merge.BeaconState): blob_builder_balances: List[Gwei, BLOB_BUILDER_REGISTRY_LIMIT] # A ring buffer of the latest slots, with information per active shard. shard_buffer: Vector[List[ShardWork, MAX_SHARDS], SHARD_STATE_MEMORY_SLOTS] - shard_gasprice: uint64 + shard_sample_price: uint64 ``` ## New containers @@ -223,6 +249,18 @@ class DataCommitment(Container): length: uint64 ``` +### `AttestedDataCommitment` + +```python +class AttestedDataCommitment(Container): + # KZG10 commitment to the data, and length + commitment: DataCommitment + # hash_tree_root of the ShardBlobHeader (stored so that attestations can be checked against it) + root: Root + # The proposer who included the shard-header + includer_index: ValidatorIndex +``` + ### ShardBlobBody Unsigned shard data, bundled by a shard-builder. @@ -238,7 +276,10 @@ class ShardBlobBody(Container): data: List[BLSPoint, POINTS_PER_SAMPLE * MAX_SAMPLES_PER_BLOB] # Latest block root of the Beacon Chain, before shard_blob.slot beacon_block_root: Root - # TODO: fee payment amount fields (EIP 1559 like) + # fee payment fields (EIP 1559 like) + # TODO: express in MWei instead? + max_priority_fee_per_sample: Gwei + max_fee_per_sample: Gwei ``` ### `ShardBlobBodySummary` @@ -258,7 +299,10 @@ class ShardBlobBodySummary(Container): data_root: Root # Latest block root of the Beacon Chain, before shard_blob.slot beacon_block_root: Root - # TODO: fee payment amount fields (EIP 1559 like) + # fee payment fields (EIP 1559 like) + # TODO: express in MWei instead? + max_priority_fee_per_sample: Gwei + max_fee_per_sample: Gwei ``` ### `ShardBlob` @@ -320,10 +364,8 @@ class SignedShardBlobHeader(Container): ```python class PendingShardHeader(Container): - # KZG10 commitment to the data - commitment: DataCommitment - # hash_tree_root of the ShardBlobHeader (stored so that attestations can be checked against it) - root: Root + # The commitment that is attested + attested: AttestedDataCommitment # Who voted for the header votes: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] # Sum of effective balances of votes @@ -370,7 +412,7 @@ class ShardWork(Container): # Upon confirmation the data is reduced to just the commitment. status: Union[ # See Shard Work Status enum None, # SHARD_WORK_UNCONFIRMED - DataCommitment, # SHARD_WORK_CONFIRMED + ConfirmedDataCommitment, # SHARD_WORK_CONFIRMED List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD] # SHARD_WORK_PENDING ] ``` @@ -396,18 +438,16 @@ def compute_previous_slot(slot: Slot) -> Slot: return Slot(0) ``` -#### `compute_updated_gasprice` +#### `compute_updated_sample_price` ```python -def compute_updated_gasprice(prev_gasprice: Gwei, shard_block_length: uint64, adjustment_quotient: uint64) -> Gwei: - if shard_block_length > TARGET_SAMPLES_PER_BLOB: - delta = max(1, prev_gasprice * (shard_block_length - TARGET_SAMPLES_PER_BLOB) - // TARGET_SAMPLES_PER_BLOB // adjustment_quotient) - return min(prev_gasprice + delta, MAX_GASPRICE) +def compute_updated_sample_price(prev_price: Gwei, samples: uint64, adjustment_quotient: uint64) -> Gwei: + if samples > TARGET_SAMPLES_PER_BLOB: + delta = max(1, prev_price * (samples - TARGET_SAMPLES_PER_BLOB) // TARGET_SAMPLES_PER_BLOB // adjustment_quotient) + return min(prev_price + delta, MAX_SAMPLE_PRICE) else: - delta = max(1, prev_gasprice * (TARGET_SAMPLES_PER_BLOB - shard_block_length) - // TARGET_SAMPLES_PER_BLOB // adjustment_quotient) - return max(prev_gasprice, MIN_GASPRICE + delta) - delta + delta = max(1, prev_price * (TARGET_SAMPLES_PER_BLOB - samples) // TARGET_SAMPLES_PER_BLOB // adjustment_quotient) + return max(prev_price, MIN_SAMPLE_PRICE + delta) - delta ``` #### `compute_committee_source_epoch` @@ -423,6 +463,20 @@ def compute_committee_source_epoch(epoch: Epoch, period: uint64) -> Epoch: return source_epoch ``` +#### `batch_apply_participation_flag` + +```python +def batch_apply_participation_flag(state: BeaconState, bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE], + epoch: Epoch, full_committee: Sequence[ValidatorIndex], flag_index: int): + if epoch == get_current_epoch(state): + epoch_participation = state.current_epoch_participation + else: + epoch_participation = state.previous_epoch_participation + for bit, index in zip(bits, full_committee): + if bit: + epoch_participation[index] = add_flag(epoch_participation[index], flag_index) +``` + ### Beacon state accessors #### Updated `get_committee_count_per_slot` @@ -449,34 +503,6 @@ def get_active_shard_count(state: BeaconState, epoch: Epoch) -> uint64: return INITIAL_ACTIVE_SHARDS ``` -#### `compute_proposer_index` - -Updated version to get a proposer index that will only allow proposers with a certain minimum balance, -ensuring that the balance is always sufficient to cover gas costs. - -```python -def compute_proposer_index(beacon_state: BeaconState, - indices: Sequence[ValidatorIndex], - seed: Bytes32, - min_effective_balance: Gwei = Gwei(0)) -> ValidatorIndex: - """ - Return from ``indices`` a random index sampled by effective balance. - """ - assert len(indices) > 0 - MAX_RANDOM_BYTE = 2**8 - 1 - i = uint64(0) - total = uint64(len(indices)) - while True: - candidate_index = indices[compute_shuffled_index(i % total, total, seed)] - random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32] - effective_balance = beacon_state.validators[candidate_index].effective_balance - if effective_balance <= min_effective_balance: - continue - if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: - return candidate_index - i += 1 -``` - #### `get_shard_proposer_index` ```python @@ -486,18 +512,8 @@ def get_shard_proposer_index(beacon_state: BeaconState, slot: Slot, shard: Shard """ epoch = compute_epoch_at_slot(slot) seed = hash(get_seed(beacon_state, epoch, DOMAIN_SHARD_BLOB) + uint_to_bytes(slot) + uint_to_bytes(shard)) - - # Proposer must have sufficient balance to pay for worst case fee burn - EFFECTIVE_BALANCE_MAX_DOWNWARD_DEVIATION = ( - EFFECTIVE_BALANCE_INCREMENT - EFFECTIVE_BALANCE_INCREMENT - * HYSTERESIS_DOWNWARD_MULTIPLIER // HYSTERESIS_QUOTIENT - ) - min_effective_balance = ( - beacon_state.shard_gasprice * MAX_SAMPLES_PER_BLOB // TARGET_SAMPLES_PER_BLOB - + EFFECTIVE_BALANCE_MAX_DOWNWARD_DEVIATION - ) indices = get_active_validator_indices(state, epoch) - return compute_proposer_index(beacon_state, indices, seed, min_effective_balance) + return compute_proposer_index(beacon_state, indices, seed) ``` #### `get_start_shard` @@ -564,6 +580,12 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: for_ops(body.shard_proposer_slashings, process_shard_proposer_slashing) # Limit is dynamic based on active shard count assert len(body.shard_headers) <= MAX_SHARD_HEADERS_PER_SHARD * get_active_shard_count(state, get_current_epoch(state)) + # Included shard headers must be sorted by shard index, the base-fee is adjusted in sequence (capacity is staggered) + # Duplicates (same slot and shard) are allowed, although slashable, only the first affects capacity. + if len(body.shard_headers) > 0: + shard = 0 + for i, header in body.shard_headers[1:] + for_ops(body.shard_headers, process_shard_header) # New attestation processing for_ops(body.attestations, process_attestation) @@ -576,22 +598,30 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: ```python def process_attestation(state: BeaconState, attestation: Attestation) -> None: altair.process_attestation(state, attestation) - update_pending_shard_work(state, attestation) + process_attested_shard_work(state, attestation) ``` ```python -def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> None: +def process_attested_shard_work(state: BeaconState, attestation: Attestation) -> None: attestation_shard = compute_shard_from_committee_index( state, attestation.data.slot, attestation.data.index, ) + full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) + buffer_index = attestation.data.slot % SHARD_STATE_MEMORY_SLOTS committee_work = state.shard_buffer[buffer_index][attestation_shard] # Skip attestation vote accounting if the header is not pending if committee_work.status.selector != SHARD_WORK_PENDING: - # TODO In Altair: set participation bit flag, if attestation matches winning header. + # If the data was already confirmed, check if this matches, to apply the flag to the attesters. + if committee_work.status.selector == SHARD_WORK_CONFIRMED: + attested: AttestedDataCommitment = current_headers[header_index] + if attested.root == attestation.data.shard_blob_root: + batch_apply_participation_flag(state, attestation.aggregation_bits, + attestation.data.target.epoch, + full_committee, TIMELY_SHARD_FLAG_INDEX) return current_headers: Sequence[PendingShardHeader] = committee_work.status.value @@ -599,16 +629,16 @@ def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> N # Find the corresponding header, abort if it cannot be found header_index = len(current_headers) for i, header in enumerate(current_headers): - if attestation.data.shard_blob_root == header.root: + if attestation.data.shard_blob_root == header.attested.root: header_index = i break + # Attestations for an unknown header do not count towards shard confirmations, but can otherwise be valid. if header_index == len(current_headers): - # TODO: Attestations may be re-included if headers are included late. + # Note: Attestations may be re-included if headers are included late. return pending_header: PendingShardHeader = current_headers[header_index] - full_committee = get_beacon_committee(state, attestation.data.slot, attestation.data.index) # The weight may be outdated if it is not the initial weight, and from a previous epoch if pending_header.weight != 0 and compute_epoch_at_slot(pending_header.update_slot) < get_current_epoch(state): @@ -629,8 +659,11 @@ def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> N # Check if the PendingShardHeader is eligible for expedited confirmation, requiring 2/3 of balance attesting if pending_header.weight * 3 >= full_committee_balance * 2: - # TODO In Altair: set participation bit flag for voters of this early winning header - if pending_header.commitment == DataCommitment(): + # participants of the winning header are remembered with participation flags + batch_apply_participation_flag(state, pending_header.votes, attestation.data.target.epoch, + full_committee, TIMELY_SHARD_FLAG_INDEX) + + if pending_header.attested.commitment == DataCommitment(): # The committee voted to not confirm anything state.shard_buffer[buffer_index][attestation_shard].status.change( selector=SHARD_WORK_UNCONFIRMED, @@ -639,26 +672,10 @@ def update_pending_shard_work(state: BeaconState, attestation: Attestation) -> N else: state.shard_buffer[buffer_index][attestation_shard].status.change( selector=SHARD_WORK_CONFIRMED, - value=pending_header.commitment, + value=pending_header.attested, ) ``` - -#### `charge_builder` - -```python -def charge_builder(state: BeaconState, index: BuilderIndex, fee: Gwei) -> None: - """ - Decrease the builder balance at index ``index`` by ``fee``, with underflow check. - """ - # TODO: apply stricter requirement to protect against fee-acceptance race conditions, e.g.: - # - balance per shard (or builders per shard) - # - balance / shard_count > fee - # TODO: also consider requirement to pay for base-fee of shard-data - assert state.blob_builder_balances[index] >= fee - state.blob_builder_balances[index] -= fee -``` - ##### `process_shard_header` ```python @@ -673,7 +690,16 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade # Verify that the header is within the processing time window assert header_epoch in [get_previous_epoch(state), get_current_epoch(state)] # Verify that the shard is active - assert shard < get_active_shard_count(state, header_epoch) + start_shard = get_start_shard(state, slot) + committees_per_slot = get_committee_count_per_slot(state, header_epoch) + end_shard = start_shard + committees_per_slot + shard_count = get_active_shard_count(state, header_epoch) + # Per slot, there may be max. shard_count committees. + # If there are shard_count * 2/3 per slot, then wrap around. + if end_shard >= shard_count: + assert not (end_shard - shard_count <= shard < start_shard) + else: + assert start_shard <= shard < end_shard # Verify that the block root matches, # to ensure the header will only be included in this specific Beacon Chain sub-tree. assert header.body_summary.beacon_block_root == get_block_root_at_slot(state, slot - 1) @@ -685,7 +711,7 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade # Check that this header is not yet in the pending list current_headers: List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD] = committee_work.status.value header_root = hash_tree_root(header) - assert header_root not in [pending_header.root for pending_header in current_headers] + assert header_root not in [pending_header.attested.root for pending_header in current_headers] # Verify proposer matches assert header.proposer_index == get_shard_proposer_index(state, slot, shard) @@ -705,20 +731,46 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade == bls.Pairing(body_summary.commitment.point, G2_SETUP[-body_summary.commitment.length]) ) - # Charge builder, with hard balance requirement - fee = Gwei(123) # TODO EIP 1559 like fee? Burn some of it? - charge_builder(state, header.builder_index, fee) - # TODO: proposer is charged for confirmed headers (see charge_confirmed_shard_fees). - # Need to align incentive, so proposer does not gain from including unconfirmed headers - increase_balance(state, header.proposer_index, fee) + # Charge EIP 1559 fee, builder pays for opportunity, and is responsible for later availability, + # or fail to publish at their own expense. + samples = blob_summary.commitment.length + # TODO: overflows, need bigger int type + max_fee = blob_summary.max_fee_per_sample * samples + + # Builder must have sufficient balance, even if max_fee is not completely utilized + assert state.blob_builder_balances[header.builder_index] > max_fee + + base_fee = state.shard_sample_price * samples + # Base fee must be paid + assert max_fee >= base_fee + + # Remaining fee goes towards proposer for prioritizing, up to a maximum + max_priority_fee = blob_summary.max_priority_fee_per_sample * samples + priority_fee = min(max_fee - base_fee, max_priority_fee) + + # Burn base fee, take priority fee + decrease_balance(state, header.builder_index, base_fee + priority_fee) + # Pay out priority fee + increase_balance(state, header.proposer_index, priority_fee) + + # Track updated sample price + adjustment_quotient = ( + get_active_shard_count(state, previous_epoch) + * SLOTS_PER_EPOCH * SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT + ) + state.shard_sample_price = compute_updated_sample_price( + state.shard_sample_price, blob_summary.commitment.length, adjustment_quotient) # Initialize the pending header index = compute_committee_index_from_shard(state, slot, shard) committee_length = len(get_beacon_committee(state, slot, index)) initial_votes = Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length) pending_header = PendingShardHeader( - commitment=blob_summary.commitment, - root=header_root, + attested=AttestedDataCommitment( + commitment=blob_summary.commitment, + root=header_root, + includer_index=get_beacon_proposer_index(state), + ) votes=initial_votes, weight=0, update_slot=state.slot, @@ -777,13 +829,12 @@ This epoch transition overrides the Merge epoch transition: def process_epoch(state: BeaconState) -> None: # Sharding pre-processing process_pending_shard_confirmations(state) - charge_confirmed_shard_fees(state) reset_pending_shard_work(state) # Base functionality process_justification_and_finalization(state) process_inactivity_updates(state) - process_rewards_and_penalties(state) + process_rewards_and_penalties(state) # Note: modified, see new TIMELY_SHARD_FLAG_INDEX process_registry_updates(state) process_slashings(state) process_eth1_data_reset(state) @@ -815,45 +866,10 @@ def process_pending_shard_confirmations(state: BeaconState) -> None: if committee_work.status.selector == SHARD_WORK_PENDING: winning_header = max(committee_work.status.value, key=lambda header: header.weight) # TODO In Altair: set participation bit flag of voters for winning header - if winning_header.commitment == DataCommitment(): + if winning_header.attested.commitment == DataCommitment(): committee_work.status.change(selector=SHARD_WORK_UNCONFIRMED, value=None) else: - committee_work.status.change(selector=SHARD_WORK_CONFIRMED, value=winning_header.commitment) -``` - -#### `charge_confirmed_shard_fees` - -```python -def charge_confirmed_shard_fees(state: BeaconState) -> None: - new_gasprice = state.shard_gasprice - previous_epoch = get_previous_epoch(state) - previous_epoch_start_slot = compute_start_slot_at_epoch(previous_epoch) - adjustment_quotient = ( - get_active_shard_count(state, previous_epoch) - * SLOTS_PER_EPOCH * GASPRICE_ADJUSTMENT_COEFFICIENT - ) - # Iterate through confirmed shard-headers - for slot in range(previous_epoch_start_slot, previous_epoch_start_slot + SLOTS_PER_EPOCH): - buffer_index = slot % SHARD_STATE_MEMORY_SLOTS - for shard_index in range(len(state.shard_buffer[buffer_index])): - committee_work = state.shard_buffer[buffer_index][shard_index] - if committee_work.status.selector == SHARD_WORK_CONFIRMED: - commitment: DataCommitment = committee_work.status.value - # Charge EIP 1559 fee - proposer = get_shard_proposer_index(state, slot, Shard(shard_index)) - fee = ( - (state.shard_gasprice * commitment.length) - // TARGET_SAMPLES_PER_BLOB - ) - decrease_balance(state, proposer, fee) - - # Track updated gas price - new_gasprice = compute_updated_gasprice( - new_gasprice, - commitment.length, - adjustment_quotient, - ) - state.shard_gasprice = new_gasprice + committee_work.status.change(selector=SHARD_WORK_CONFIRMED, value=winning_header.attested) ``` #### `reset_pending_shard_work` @@ -881,8 +897,7 @@ def reset_pending_shard_work(state: BeaconState) -> None: selector=SHARD_WORK_PENDING, value=List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD]( PendingShardHeader( - commitment=DataCommitment(), - root=Root(), + attested=AttestedDataCommitment() votes=Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length), weight=0, update_slot=slot, From 756eb90bfed131f7c63b0416f3b8b14efabbd6da Mon Sep 17 00:00:00 2001 From: protolambda Date: Mon, 26 Jul 2021 16:27:19 +0200 Subject: [PATCH 12/23] consider per-slot sample target adjustment, to avoid racing and ordering problems --- specs/sharding/beacon-chain.md | 43 ++++++++++++++-------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 40eabe548..80e2ec85c 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -441,7 +441,8 @@ def compute_previous_slot(slot: Slot) -> Slot: #### `compute_updated_sample_price` ```python -def compute_updated_sample_price(prev_price: Gwei, samples: uint64, adjustment_quotient: uint64) -> Gwei: +def compute_updated_sample_price(prev_price: Gwei, samples: uint64, active_shards: uint64) -> Gwei: + adjustment_quotient = active_shards * SLOTS_PER_EPOCH * SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT if samples > TARGET_SAMPLES_PER_BLOB: delta = max(1, prev_price * (samples - TARGET_SAMPLES_PER_BLOB) // TARGET_SAMPLES_PER_BLOB // adjustment_quotient) return min(prev_price + delta, MAX_SAMPLE_PRICE) @@ -578,19 +579,19 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: for_ops(body.attester_slashings, process_attester_slashing) # New shard proposer slashing processing for_ops(body.shard_proposer_slashings, process_shard_proposer_slashing) - # Limit is dynamic based on active shard count + + # Limit is dynamic: based on active shard count assert len(body.shard_headers) <= MAX_SHARD_HEADERS_PER_SHARD * get_active_shard_count(state, get_current_epoch(state)) - # Included shard headers must be sorted by shard index, the base-fee is adjusted in sequence (capacity is staggered) - # Duplicates (same slot and shard) are allowed, although slashable, only the first affects capacity. - if len(body.shard_headers) > 0: - shard = 0 - for i, header in body.shard_headers[1:] - for_ops(body.shard_headers, process_shard_header) + # New attestation processing for_ops(body.attestations, process_attestation) for_ops(body.deposits, process_deposit) for_ops(body.voluntary_exits, process_voluntary_exit) + + # TODO: to avoid parallel shards racing, and avoid inclusion-order problems, + # update the fee price per slot, instead of per header. + # state.shard_sample_price = compute_updated_sample_price(state.shard_sample_price, ?, shard_count) ``` ##### Extended Attestation processing @@ -689,17 +690,15 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade header_epoch = compute_epoch_at_slot(slot) # Verify that the header is within the processing time window assert header_epoch in [get_previous_epoch(state), get_current_epoch(state)] - # Verify that the shard is active - start_shard = get_start_shard(state, slot) - committees_per_slot = get_committee_count_per_slot(state, header_epoch) - end_shard = start_shard + committees_per_slot + # Verify that the shard is valid shard_count = get_active_shard_count(state, header_epoch) - # Per slot, there may be max. shard_count committees. - # If there are shard_count * 2/3 per slot, then wrap around. - if end_shard >= shard_count: - assert not (end_shard - shard_count <= shard < start_shard) - else: - assert start_shard <= shard < end_shard + assert shard < shard_count + # Verify that a committee is able to attest this (slot, shard) + start_shard = get_start_shard(state, slot) + committee_index = (shard_count + shard - start_shard) % shard_count + committees_per_slot = get_committee_count_per_slot(state, header_epoch) + assert committee_index <= committees_per_slot + # Verify that the block root matches, # to ensure the header will only be included in this specific Beacon Chain sub-tree. assert header.body_summary.beacon_block_root == get_block_root_at_slot(state, slot - 1) @@ -753,14 +752,6 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade # Pay out priority fee increase_balance(state, header.proposer_index, priority_fee) - # Track updated sample price - adjustment_quotient = ( - get_active_shard_count(state, previous_epoch) - * SLOTS_PER_EPOCH * SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT - ) - state.shard_sample_price = compute_updated_sample_price( - state.shard_sample_price, blob_summary.commitment.length, adjustment_quotient) - # Initialize the pending header index = compute_committee_index_from_shard(state, slot, shard) committee_length = len(get_beacon_committee(state, slot, index)) From d47d2f92cc8fe630ba1f182634a0186bf76924d4 Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 27 Jul 2021 14:48:21 +0200 Subject: [PATCH 13/23] shard fees: implement review suggestions from @nashatyrev --- specs/sharding/beacon-chain.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 80e2ec85c..93ac2278a 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -412,7 +412,7 @@ class ShardWork(Container): # Upon confirmation the data is reduced to just the commitment. status: Union[ # See Shard Work Status enum None, # SHARD_WORK_UNCONFIRMED - ConfirmedDataCommitment, # SHARD_WORK_CONFIRMED + AttestedDataCommitment, # SHARD_WORK_CONFIRMED List[PendingShardHeader, MAX_SHARD_HEADERS_PER_SHARD] # SHARD_WORK_PENDING ] ``` @@ -737,7 +737,7 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade max_fee = blob_summary.max_fee_per_sample * samples # Builder must have sufficient balance, even if max_fee is not completely utilized - assert state.blob_builder_balances[header.builder_index] > max_fee + assert state.blob_builder_balances[header.builder_index] >= max_fee base_fee = state.shard_sample_price * samples # Base fee must be paid @@ -748,7 +748,8 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade priority_fee = min(max_fee - base_fee, max_priority_fee) # Burn base fee, take priority fee - decrease_balance(state, header.builder_index, base_fee + priority_fee) + # priority_fee <= max_fee - base_fee, thus priority_fee + base_fee <= max_fee, thus sufficient balance. + state.blob_builder_balances[header.builder_index] -= base_fee + priority_fee # Pay out priority fee increase_balance(state, header.proposer_index, priority_fee) From 0daaafbc98e32925120fb4a22665286548e70988 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 28 Jul 2021 22:00:23 +0200 Subject: [PATCH 14/23] fix union value retrieval, thanks @terencechain --- specs/sharding/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 93ac2278a..25c3423b7 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -618,7 +618,7 @@ def process_attested_shard_work(state: BeaconState, attestation: Attestation) -> if committee_work.status.selector != SHARD_WORK_PENDING: # If the data was already confirmed, check if this matches, to apply the flag to the attesters. if committee_work.status.selector == SHARD_WORK_CONFIRMED: - attested: AttestedDataCommitment = current_headers[header_index] + attested: AttestedDataCommitment = committee_work.status.value if attested.root == attestation.data.shard_blob_root: batch_apply_participation_flag(state, attestation.aggregation_bits, attestation.data.target.epoch, From ab78339350f9e2a66e5aa2dfe98a028393fb474b Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 30 Jul 2021 21:54:55 +0200 Subject: [PATCH 15/23] fix variable name of summary field --- specs/sharding/beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 25c3423b7..13a58e088 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -732,9 +732,9 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade # Charge EIP 1559 fee, builder pays for opportunity, and is responsible for later availability, # or fail to publish at their own expense. - samples = blob_summary.commitment.length + samples = body_summary.commitment.length # TODO: overflows, need bigger int type - max_fee = blob_summary.max_fee_per_sample * samples + max_fee = body_summary.max_fee_per_sample * samples # Builder must have sufficient balance, even if max_fee is not completely utilized assert state.blob_builder_balances[header.builder_index] >= max_fee @@ -744,7 +744,7 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade assert max_fee >= base_fee # Remaining fee goes towards proposer for prioritizing, up to a maximum - max_priority_fee = blob_summary.max_priority_fee_per_sample * samples + max_priority_fee = body_summary.max_priority_fee_per_sample * samples priority_fee = min(max_fee - base_fee, max_priority_fee) # Burn base fee, take priority fee @@ -759,7 +759,7 @@ def process_shard_header(state: BeaconState, signed_header: SignedShardBlobHeade initial_votes = Bitlist[MAX_VALIDATORS_PER_COMMITTEE]([0] * committee_length) pending_header = PendingShardHeader( attested=AttestedDataCommitment( - commitment=blob_summary.commitment, + commitment=body_summary.commitment, root=header_root, includer_index=get_beacon_proposer_index(state), ) From add5810d71ade5a80c65363336f1c1a43a0b4a2f Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 30 Jul 2021 22:06:04 +0200 Subject: [PATCH 16/23] remove unused pending attestation fields --- specs/sharding/beacon-chain.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 13a58e088..94495ef46 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -216,10 +216,6 @@ class BeaconBlockBody(merge.BeaconBlockBody): # [extends The Merge block body] ```python class BeaconState(merge.BeaconState): - # [Updated fields] (Warning: this changes with Altair, Sharding will rebase to use participation-flags) - previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] - current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] - # [New fields] # Blob builder registry. blob_builders: List[Builder, BLOB_BUILDER_REGISTRY_LIMIT] blob_builder_balances: List[Gwei, BLOB_BUILDER_REGISTRY_LIMIT] From 2d17c8c3c442c5a3c8d4b1e47e1c335f7fcacd4f Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 30 Jul 2021 22:22:43 +0200 Subject: [PATCH 17/23] move back INITIAL_ACTIVE_SHARDS to preset, avoid changing mainnet config --- specs/sharding/beacon-chain.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 94495ef46..e81e7e764 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -151,6 +151,7 @@ TODO: `WEIGHT_DENOMINATOR` needs to be adjusted, but this breaks a lot of Altair | Name | Value | Notes | | - | - | - | | `MAX_SHARDS` | `uint64(2**10)` (= 1,024) | Theoretical max shard count (used to determine data structure sizes) | +| `INITIAL_ACTIVE_SHARDS` | `uint64(2**6)` (= 64) | Initial shard count | | `SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT` | `uint64(2**3)` (= 8) | Sample price may decrease/increase by at most exp(1 / this value) *per epoch* | | `MAX_SHARD_PROPOSER_SLASHINGS` | `2**4` (= 16) | Maximum amount of shard proposer slashing operations per block | | `MAX_SHARD_HEADERS_PER_SHARD` | `4` | | @@ -181,9 +182,8 @@ TODO: `WEIGHT_DENOMINATOR` needs to be adjusted, but this breaks a lot of Altair ## Configuration -| Name | Value | Notes | -| - | - | - | -| `INITIAL_ACTIVE_SHARDS` | `uint64(2**6)` (= 64) | Initial shard count | +Note: some preset variables may become run-time configurable for testnets, but default to a preset while the spec is unstable. +E.g. `INITIAL_ACTIVE_SHARDS`, `MAX_SAMPLES_PER_BLOB` and `TARGET_SAMPLES_PER_BLOB`. ## Updated containers From 91968de423a8614e1fe20fac7b1b563c4123f7ca Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 30 Jul 2021 22:31:26 +0200 Subject: [PATCH 18/23] update sharding presets --- presets/mainnet/sharding.yaml | 20 +++++++++++--------- presets/minimal/sharding.yaml | 17 ++++++++++------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/presets/mainnet/sharding.yaml b/presets/mainnet/sharding.yaml index 2b78855fc..120a716c9 100644 --- a/presets/mainnet/sharding.yaml +++ b/presets/mainnet/sharding.yaml @@ -1,22 +1,24 @@ # Mainnet preset - Sharding -# Beacon-chain -# --------------------------------------------------------------- # Misc +# --------------------------------------------------------------- # 2**10 (= 1,024) MAX_SHARDS: 1024 -# 2**6 = 64 +# 2**6 (= 64) INITIAL_ACTIVE_SHARDS: 64 # 2**3 (= 8) -GASPRICE_ADJUSTMENT_COEFFICIENT: 8 +SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT: 8 # 2**4 (= 16) MAX_SHARD_PROPOSER_SLASHINGS: 16 - -# Shard block configs -# --------------------------------------------------------------- +# MAX_SHARD_HEADERS_PER_SHARD: 4 # 2**8 (= 256) SHARD_STATE_MEMORY_SLOTS: 256 +# 2**40 (= 1,099,511,627,776) +BLOB_BUILDER_REGISTRY_LIMIT: 1099511627776 + +# Shard blob samples +# --------------------------------------------------------------- # 2**11 (= 2,048) MAX_SAMPLES_PER_BLOCK: 2048 # 2**10 (= 1,1024) @@ -25,6 +27,6 @@ TARGET_SAMPLES_PER_BLOCK: 1024 # Gwei values # --------------------------------------------------------------- # 2**33 (= 8,589,934,592) Gwei -MAX_GASPRICE: 8589934592 +MAX_SAMPLE_PRICE: 8589934592 # 2**3 (= 8) Gwei -MIN_GASPRICE: 8 +MIN_SAMPLE_PRICE: 8 diff --git a/presets/minimal/sharding.yaml b/presets/minimal/sharding.yaml index 10f79c96e..6b8d223b4 100644 --- a/presets/minimal/sharding.yaml +++ b/presets/minimal/sharding.yaml @@ -1,6 +1,6 @@ # Minimal preset - Sharding -# Beacon-chain +# Misc # --------------------------------------------------------------- # Misc # [customized] reduced for testing @@ -8,15 +8,18 @@ MAX_SHARDS: 8 # [customized] reduced for testing INITIAL_ACTIVE_SHARDS: 2 # 2**3 (= 8) -GASPRICE_ADJUSTMENT_COEFFICIENT: 8 +SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT: 8 # [customized] reduced for testing MAX_SHARD_PROPOSER_SLASHINGS: 4 - -# Shard block configs -# --------------------------------------------------------------- +# MAX_SHARD_HEADERS_PER_SHARD: 4 # 2**8 (= 256) SHARD_STATE_MEMORY_SLOTS: 256 +# 2**40 (= 1,099,511,627,776) +BLOB_BUILDER_REGISTRY_LIMIT: 1099511627776 + +# Shard blob samples +# --------------------------------------------------------------- # 2**11 (= 2,048) MAX_SAMPLES_PER_BLOCK: 2048 # 2**10 (= 1,1024) @@ -25,6 +28,6 @@ TARGET_SAMPLES_PER_BLOCK: 1024 # Gwei values # --------------------------------------------------------------- # 2**33 (= 8,589,934,592) Gwei -MAX_GASPRICE: 8589934592 +MAX_SAMPLE_PRICE: 8589934592 # 2**3 (= 8) Gwei -MIN_GASPRICE: 8 +MIN_SAMPLE_PRICE: 8 From 322f072703636b7b70a459fe6af80d1fd42a20c1 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 31 Jul 2021 13:22:26 +0200 Subject: [PATCH 19/23] sharding: remove outdated comment, timely shard attesters are marked in attestation-processing, no need for epoch processing additions --- specs/sharding/beacon-chain.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index e81e7e764..f5b212c6e 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -853,7 +853,6 @@ def process_pending_shard_confirmations(state: BeaconState) -> None: committee_work = state.shard_buffer[buffer_index][shard_index] if committee_work.status.selector == SHARD_WORK_PENDING: winning_header = max(committee_work.status.value, key=lambda header: header.weight) - # TODO In Altair: set participation bit flag of voters for winning header if winning_header.attested.commitment == DataCommitment(): committee_work.status.change(selector=SHARD_WORK_UNCONFIRMED, value=None) else: From 424f8387473ef6b98ceac01704174472c917c194 Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Wed, 4 Aug 2021 12:44:42 +0200 Subject: [PATCH 20/23] Update specs/sharding/beacon-chain.md Co-authored-by: Danny Ryan --- specs/sharding/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index f5b212c6e..49ece701f 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -182,7 +182,7 @@ TODO: `WEIGHT_DENOMINATOR` needs to be adjusted, but this breaks a lot of Altair ## Configuration -Note: some preset variables may become run-time configurable for testnets, but default to a preset while the spec is unstable. +Note: Some preset variables may become run-time configurable for testnets, but default to a preset while the spec is unstable. E.g. `INITIAL_ACTIVE_SHARDS`, `MAX_SAMPLES_PER_BLOB` and `TARGET_SAMPLES_PER_BLOB`. ## Updated containers From fc3e651817e1775deae5f8252d546974ab24bd87 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 4 Aug 2021 23:25:25 +0200 Subject: [PATCH 21/23] samples -> samples_length --- specs/sharding/beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/sharding/beacon-chain.md b/specs/sharding/beacon-chain.md index 49ece701f..bbb4ae930 100644 --- a/specs/sharding/beacon-chain.md +++ b/specs/sharding/beacon-chain.md @@ -437,13 +437,13 @@ def compute_previous_slot(slot: Slot) -> Slot: #### `compute_updated_sample_price` ```python -def compute_updated_sample_price(prev_price: Gwei, samples: uint64, active_shards: uint64) -> Gwei: +def compute_updated_sample_price(prev_price: Gwei, samples_length: uint64, active_shards: uint64) -> Gwei: adjustment_quotient = active_shards * SLOTS_PER_EPOCH * SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT - if samples > TARGET_SAMPLES_PER_BLOB: - delta = max(1, prev_price * (samples - TARGET_SAMPLES_PER_BLOB) // TARGET_SAMPLES_PER_BLOB // adjustment_quotient) + if samples_length > TARGET_SAMPLES_PER_BLOB: + delta = max(1, prev_price * (samples_length - TARGET_SAMPLES_PER_BLOB) // TARGET_SAMPLES_PER_BLOB // adjustment_quotient) return min(prev_price + delta, MAX_SAMPLE_PRICE) else: - delta = max(1, prev_price * (TARGET_SAMPLES_PER_BLOB - samples) // TARGET_SAMPLES_PER_BLOB // adjustment_quotient) + delta = max(1, prev_price * (TARGET_SAMPLES_PER_BLOB - samples_length) // TARGET_SAMPLES_PER_BLOB // adjustment_quotient) return max(prev_price, MIN_SAMPLE_PRICE + delta) - delta ``` From d005fee67df0cdc9bed448ac44d6912c19f8dcbb Mon Sep 17 00:00:00 2001 From: Diederik Loerakker Date: Tue, 10 Aug 2021 13:48:26 +0200 Subject: [PATCH 22/23] sharding p2p code review fixes Co-authored-by: Danny Ryan --- specs/sharding/p2p-interface.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/specs/sharding/p2p-interface.md b/specs/sharding/p2p-interface.md index 4eb3f6f2e..e5394abc2 100644 --- a/specs/sharding/p2p-interface.md +++ b/specs/sharding/p2p-interface.md @@ -89,10 +89,10 @@ on the horizontal subnet or creating samples for it. Alias `blob = signed_blob.m - _[REJECT]_ The shard blob is for the correct subnet -- i.e. `compute_subnet_for_shard_blob(state, blob.slot, blob.shard) == subnet_id` - _[IGNORE]_ The blob is the first blob with valid signature received for the `(blob.proposer_index, blob.slot, blob.shard)` combination. -- _[REJECT]_ The blob is not too large, the data MUST NOT be larger than the SSZ list-limit, and a client MAY be more strict. +- _[REJECT]_ The blob is not too large -- the data MUST NOT be larger than the SSZ list-limit, and a client MAY apply stricter bounds. - _[REJECT]_ The `blob.body.data` MUST NOT contain any point `p >= MODULUS`. Although it is a `uint256`, not the full 256 bit range is valid. -- _[REJECT]_ The blob builder exists and has sufficient balance to back the fee payment. -- _[REJECT]_ The blob signature is valid for the aggregate of proposer and builder, `signed_blob.signature`, +- _[REJECT]_ The blob builder defined by `blob.builder_index` exists and has sufficient balance to back the fee payment. +- _[REJECT]_ The blob signature, `signed_blob.signature`, is valid for the aggregate of proposer and builder -- i.e. `bls.FastAggregateVerify([builder_pubkey, proposer_pubkey], blob_signing_root, signed_blob.signature)`. - _[REJECT]_ The blob is proposed by the expected `proposer_index` for the blob's `slot` and `shard`, in the context of the current shuffling (defined by `blob.body.beacon_block_root`/`slot`). @@ -104,17 +104,17 @@ on the horizontal subnet or creating samples for it. Alias `blob = signed_blob.m There are three additional global topics for Sharding. -- `shard_blob_header`: co-signed headers, to be included on-chain, and signaling builders to publish full data. +- `shard_blob_header`: co-signed headers to be included on-chain and to serve as a signal to the builder to publish full data. - `shard_blob_tx`: builder-signed headers, also known as "data transaction". -- `shard_proposer_slashing`: slashings of duplicate shard proposals +- `shard_proposer_slashing`: slashings of duplicate shard proposals. ##### `shard_blob_header` Shard header data, in the form of a `SignedShardBlobHeader` is published to the global `shard_blob_header` subnet. -Shard blob headers select shard blob bids by builders, +Shard blob headers select shard blob bids by builders and should be timely to ensure builders can publish the full shard blob before subsequent attestations. -The following validations MUST pass before forwarding the `signed_blob_header` on the network. Alias `header = signed_blob_header.message` +The following validations MUST pass before forwarding the `signed_blob_header` on the network. Alias `header = signed_blob_header.message`. - _[IGNORE]_ The `header` is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `header.slot <= current_slot` @@ -126,8 +126,8 @@ The following validations MUST pass before forwarding the `signed_blob_header` o - _[REJECT]_ The `header.shard` MUST have a committee at the `header.slot` -- i.e. validate that `compute_committee_index_from_shard(state, header.slot, header.shard)` doesn't raise an error - _[IGNORE]_ The header is the first header with valid signature received for the `(header.proposer_index, header.slot, header.shard)` combination. -- _[REJECT]_ The blob builder exists and has sufficient balance to back the fee payment. -- _[REJECT]_ The header signature is valid for the aggregate of proposer and builder, `signed_blob_header.signature`, +- _[REJECT]_ The blob builder defined by `blob.builder_index` exists and has sufficient balance to back the fee payment. +- _[REJECT]_ The header signature, `signed_blob_header.signature`, is valid for the aggregate of proposer and builder -- i.e. `bls.FastAggregateVerify([builder_pubkey, proposer_pubkey], blob_signing_root, signed_blob_header.signature)`. - _[REJECT]_ The header is proposed by the expected `proposer_index` for the blob's `header.slot` and `header.shard` in the context of the current shuffling (defined by `header.body_summary.beacon_block_root`/`slot`). @@ -137,12 +137,12 @@ The following validations MUST pass before forwarding the `signed_blob_header` o ##### `shard_blob_tx` -Shard data-transactions, in the form of a `SignedShardBlobHeader` is published to the global `shard_blob_tx` subnet. +Shard data-transactions in the form of a `SignedShardBlobHeader` are published to the global `shard_blob_tx` subnet. These shard blob headers are signed solely by the blob-builder. -The following validations MUST pass before forwarding the `signed_blob_header` on the network. Alias `header = signed_blob_header.message` +The following validations MUST pass before forwarding the `signed_blob_header` on the network. Alias `header = signed_blob_header.message`. -- _[IGNORE]_ The `header` is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- +- _[IGNORE]_ The header is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `header.slot <= current_slot` (a client MAY queue future headers for processing at the appropriate slot). - _[IGNORE]_ The header is new enough to still be processed -- @@ -152,10 +152,10 @@ The following validations MUST pass before forwarding the `signed_blob_header` o - _[REJECT]_ The `header.shard` MUST have a committee at the `header.slot` -- i.e. validate that `compute_committee_index_from_shard(state, header.slot, header.shard)` doesn't raise an error - _[IGNORE]_ The header is the first header with valid signature received for the `(header.builder_index, header.slot, header.shard)` combination. -- _[REJECT]_ The blob builder exists and has sufficient balance to back the fee payment. +- _[REJECT]_ The blob builder, define by `header.builder_index`, exists and has sufficient balance to back the fee payment. - _[IGNORE]_ The header fee SHOULD be higher than previously seen headers for `(header.slot, header.shard)`, from any builder. Propagating nodes MAY increase fee increments in case of spam. -- _[REJECT]_ The header signature is valid for ONLY the builder, `signed_blob_header.signature`, +- _[REJECT]_ The header signature, `signed_blob_header.signature`, is valid for ONLY the builder -- i.e. `bls.Verify(builder_pubkey, blob_signing_root, signed_blob_header.signature)`. The signature is not an aggregate with the proposer. - _[REJECT]_ The header is designated for proposal by the expected `proposer_index` for the blob's `header.slot` and `header.shard` in the context of the current shuffling (defined by `header.body_summary.beacon_block_root`/`slot`). From da893c123e933ace3663f0306e19fb79d85b7d8f Mon Sep 17 00:00:00 2001 From: protolambda Date: Tue, 10 Aug 2021 23:18:59 +0200 Subject: [PATCH 23/23] update p2p shard blob/header/tx propagation windows --- specs/sharding/p2p-interface.md | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/specs/sharding/p2p-interface.md b/specs/sharding/p2p-interface.md index e5394abc2..93ff1e26d 100644 --- a/specs/sharding/p2p-interface.md +++ b/specs/sharding/p2p-interface.md @@ -36,6 +36,9 @@ The adjustments and additions for Shards are outlined in this document. | Name | Value | Description | | ---- | ----- | ----------- | | `SHARD_BLOB_SUBNET_COUNT` | `64` | The number of `shard_blob_{subnet_id}` subnets used in the gossipsub protocol. | +| `SHARD_TX_PROPAGATION_GRACE_SLOTS` | `4` | The number of slots for a late transaction to propagate | +| `SHARD_TX_PROPAGATION_BUFFER_SLOTS` | `8` | The number of slots for an early transaction to propagate | + ## Gossip domain @@ -77,9 +80,9 @@ def compute_subnet_for_shard_blob(state: BeaconState, slot: Slot, shard: Shard) The following validations MUST pass before forwarding the `signed_blob`, on the horizontal subnet or creating samples for it. Alias `blob = signed_blob.message`. -- _[IGNORE]_ The `blob` is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- - i.e. validate that `blob.slot <= current_slot` - (a client MAY queue future blobs for processing at the appropriate slot). +- _[IGNORE]_ The `blob` is published 1 slot early or later (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- + i.e. validate that `blob.slot <= current_slot + 1` + (a client MAY queue future blobs for propagation at the appropriate slot). - _[IGNORE]_ The `blob` is new enough to still be processed -- i.e. validate that `compute_epoch_at_slot(blob.slot) >= get_previous_epoch(state)` - _[REJECT]_ The shard blob is for an active shard -- @@ -116,15 +119,15 @@ and should be timely to ensure builders can publish the full shard blob before s The following validations MUST pass before forwarding the `signed_blob_header` on the network. Alias `header = signed_blob_header.message`. -- _[IGNORE]_ The `header` is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- - i.e. validate that `header.slot <= current_slot` - (a client MAY queue future headers for processing at the appropriate slot). +- _[IGNORE]_ The `header` is published 1 slot early or later (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- + i.e. validate that `header.slot <= current_slot + 1` + (a client MAY queue future headers for propagation at the appropriate slot). - _[IGNORE]_ The header is new enough to still be processed -- i.e. validate that `compute_epoch_at_slot(header.slot) >= get_previous_epoch(state)` - _[REJECT]_ The shard header is for an active shard -- i.e. `header.shard < get_active_shard_count(state, compute_epoch_at_slot(header.slot))` - _[REJECT]_ The `header.shard` MUST have a committee at the `header.slot` -- - i.e. validate that `compute_committee_index_from_shard(state, header.slot, header.shard)` doesn't raise an error + i.e. validate that `compute_committee_index_from_shard(state, header.slot, header.shard)` doesn't raise an error. - _[IGNORE]_ The header is the first header with valid signature received for the `(header.proposer_index, header.slot, header.shard)` combination. - _[REJECT]_ The blob builder defined by `blob.builder_index` exists and has sufficient balance to back the fee payment. - _[REJECT]_ The header signature, `signed_blob_header.signature`, is valid for the aggregate of proposer and builder -- @@ -142,15 +145,15 @@ These shard blob headers are signed solely by the blob-builder. The following validations MUST pass before forwarding the `signed_blob_header` on the network. Alias `header = signed_blob_header.message`. -- _[IGNORE]_ The header is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- - i.e. validate that `header.slot <= current_slot` - (a client MAY queue future headers for processing at the appropriate slot). -- _[IGNORE]_ The header is new enough to still be processed -- - i.e. validate that `compute_epoch_at_slot(header.slot) >= get_previous_epoch(state)` +- _[IGNORE]_ The header is not propagating more than `SHARD_TX_PROPAGATION_BUFFER_SLOTS` slots ahead of time -- + i.e. validate that `header.slot <= current_slot + SHARD_TX_PROPAGATION_BUFFER_SLOTS`. +- _[IGNORE]_ The header is not propagating later than `SHARD_TX_PROPAGATION_GRACE_SLOTS` slots too late -- + i.e. validate that `header.slot + SHARD_TX_PROPAGATION_GRACE_SLOTS >= current_slot` - _[REJECT]_ The shard header is for an active shard -- i.e. `header.shard < get_active_shard_count(state, compute_epoch_at_slot(header.slot))` - _[REJECT]_ The `header.shard` MUST have a committee at the `header.slot` -- - i.e. validate that `compute_committee_index_from_shard(state, header.slot, header.shard)` doesn't raise an error + i.e. validate that `compute_committee_index_from_shard(state, header.slot, header.shard)` doesn't raise an error. +- _[IGNORE]_ The header is not stale -- i.e. the corresponding shard proposer has not already selected a header for `(header.slot, header.shard)`. - _[IGNORE]_ The header is the first header with valid signature received for the `(header.builder_index, header.slot, header.shard)` combination. - _[REJECT]_ The blob builder, define by `header.builder_index`, exists and has sufficient balance to back the fee payment. - _[IGNORE]_ The header fee SHOULD be higher than previously seen headers for `(header.slot, header.shard)`, from any builder.