mirror of
https://github.com/status-im/eth2.0-specs.git
synced 2025-02-25 08:45:19 +00:00
wip
wip
Add `TARGET_NUMBER_OF_PEERS`
Add networking spec draft
fix
simplification
Rename `DoYouHave` to `GetCustodyStatus`
Add DataLineSidecar design
Apply suggestions from code review
Co-authored-by: dankrad <mail@dankradfeist.de>
Co-authored-by: danny <dannyjryan@gmail.com>
Revamp after reviews and discussion
Remove `CustodyStatus`
minor fix
Change`DataColumn` to `List[DataCell, MAX_BLOBS_PER_BLOCK]`
Move folder
Replace `DataColumnByRootAndIndex` with `DataColumnSidecarByRoot` message. Add extended data description
Remove `DataRow`
Apply suggestions from @jacobkaufmann code review
Co-authored-by: Jacob Kaufmann <jacobkaufmann18@gmail.com>
Represent matrix in `BLSFieldElement` form
Add `assert time >= store.time` to `on_tick`
Revert the spec. Only handle it in tests
Remove extra tick
cleanup leftover
Add randomized block cases
Specify RPC byRoot blocks-sidecars elegibility
fix typo
Update specs/phase0/p2p-interface.md
Co-authored-by: Mikhail Kalinin <noblesse.knight@gmail.com>
Update specs/deneb/p2p-interface.md
Co-authored-by: Mikhail Kalinin <noblesse.knight@gmail.com>
add failed on_block condition
rephrase
Update specs/phase0/p2p-interface.md
Co-authored-by: Mikhail Kalinin <noblesse.knight@gmail.com>
apply suggestion
Update specs/deneb/p2p-interface.md
Co-authored-by: danny <dannyjryan@gmail.com>
Update specs/deneb/p2p-interface.md
Co-authored-by: danny <dannyjryan@gmail.com>
remove the last consider
from on_block to state_transition
simplify and add a new rule
Update specs/phase0/p2p-interface.md
Co-authored-by: Mikhail Kalinin <noblesse.knight@gmail.com>
Update specs/deneb/p2p-interface.md
Co-authored-by: Mikhail Kalinin <noblesse.knight@gmail.com>
Update specs/deneb/p2p-interface.md
Co-authored-by: danny <dannyjryan@gmail.com>
remove gossip failure rules
Apply suggestions from code review
bump version to v1.4.0-beta.5
Move `blob_sidecar_{subnet_id}` to `Blob subnets` section
Misc minor fix
Add linter support
Add column subnet validation. Split `verify_column_sidecar` into two functions
Fix `get_data_column_sidecars` by using `compute_samples_and_proofs`
Apply suggestions from code review
Co-authored-by: danny <dannyjryan@gmail.com>
Do not assign row custody
Apply suggestions from code review
Co-authored-by: danny <dannyjryan@gmail.com>
Revamp reconstruction section
Use depth as the primary preset for inclusion proof. Fix `get_data_column_sidecars` and add tests for merkle proof
Change `SAMPLES_PER_SLOT` to 8 and add tests (requirement TBD)
Apply PR feedback from @ppopth and @jtraglia
Fix `get_data_column_sidecars`
Co-authored-by: Pop Chunhapanya <haxx.pop@gmail.com>
Apply suggestions from code review
Co-authored-by: Pop Chunhapanya <haxx.pop@gmail.com>
Apply suggestions from code review
Co-authored-by: fradamt <104826920+fradamt@users.noreply.github.com>
Co-authored-by: Jacob Kaufmann <jacobkaufmann18@gmail.com>
Fix `get_data_column_sidecars` and `get_custody_lines`
Apply suggestions from code review
Co-authored-by: Jacob Kaufmann <jacobkaufmann18@gmail.com>
Enhance tests
fix typo
Co-authored-by: fradamt <104826920+fradamt@users.noreply.github.com>
Remove `epoch` from `get_custody_lines`
fix
fix
This commit is contained in:
Hsiao-Wei Wang
parent
d6a37ecfcb
commit
93dddd15f2
@ -59,6 +59,9 @@ EIP7002_FORK_EPOCH: 18446744073709551615
|
|||||||
# WHISK
|
# WHISK
|
||||||
WHISK_FORK_VERSION: 0x06000000 # temporary stub
|
WHISK_FORK_VERSION: 0x06000000 # temporary stub
|
||||||
WHISK_FORK_EPOCH: 18446744073709551615
|
WHISK_FORK_EPOCH: 18446744073709551615
|
||||||
|
# EIP7594
|
||||||
|
EIP7594_FORK_VERSION: 0x06000001
|
||||||
|
EIP7594_FORK_EPOCH: 18446744073709551615
|
||||||
|
|
||||||
|
|
||||||
# Time parameters
|
# Time parameters
|
||||||
@ -154,7 +157,3 @@ BLOB_SIDECAR_SUBNET_COUNT: 6
|
|||||||
WHISK_EPOCHS_PER_SHUFFLING_PHASE: 256
|
WHISK_EPOCHS_PER_SHUFFLING_PHASE: 256
|
||||||
# `Epoch(2)`
|
# `Epoch(2)`
|
||||||
WHISK_PROPOSER_SELECTION_GAP: 2
|
WHISK_PROPOSER_SELECTION_GAP: 2
|
||||||
|
|
||||||
# EIP7594
|
|
||||||
EIP7594_FORK_VERSION: 0x06000001
|
|
||||||
EIP7594_FORK_EPOCH: 18446744073709551615
|
|
||||||
|
|||||||
@ -58,6 +58,9 @@ EIP7002_FORK_EPOCH: 18446744073709551615
|
|||||||
# WHISK
|
# WHISK
|
||||||
WHISK_FORK_VERSION: 0x06000001
|
WHISK_FORK_VERSION: 0x06000001
|
||||||
WHISK_FORK_EPOCH: 18446744073709551615
|
WHISK_FORK_EPOCH: 18446744073709551615
|
||||||
|
# EIP7594
|
||||||
|
EIP7594_FORK_VERSION: 0x06000001
|
||||||
|
EIP7594_FORK_EPOCH: 18446744073709551615
|
||||||
|
|
||||||
|
|
||||||
# Time parameters
|
# Time parameters
|
||||||
@ -153,7 +156,3 @@ BLOB_SIDECAR_SUBNET_COUNT: 6
|
|||||||
# Whisk
|
# Whisk
|
||||||
WHISK_EPOCHS_PER_SHUFFLING_PHASE: 4
|
WHISK_EPOCHS_PER_SHUFFLING_PHASE: 4
|
||||||
WHISK_PROPOSER_SELECTION_GAP: 1
|
WHISK_PROPOSER_SELECTION_GAP: 1
|
||||||
|
|
||||||
# EIP7594
|
|
||||||
EIP7594_FORK_VERSION: 0x06000001
|
|
||||||
EIP7594_FORK_EPOCH: 18446744073709551615
|
|
||||||
|
|||||||
@ -4,3 +4,7 @@
|
|||||||
# ---------------------------------------------------------------
|
# ---------------------------------------------------------------
|
||||||
# `uint64(2**6)` (= 64)
|
# `uint64(2**6)` (= 64)
|
||||||
FIELD_ELEMENTS_PER_CELL: 64
|
FIELD_ELEMENTS_PER_CELL: 64
|
||||||
|
# uint64(floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments'))
|
||||||
|
KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH: 4
|
||||||
|
# `uint64((FIELD_ELEMENTS_PER_BLOB * 2) // FIELD_ELEMENTS_PER_CELL)` (= 128)
|
||||||
|
NUMBER_OF_COLUMNS: 128
|
||||||
|
|||||||
@ -4,3 +4,7 @@
|
|||||||
# ---------------------------------------------------------------
|
# ---------------------------------------------------------------
|
||||||
# `uint64(2**6)` (= 64)
|
# `uint64(2**6)` (= 64)
|
||||||
FIELD_ELEMENTS_PER_CELL: 64
|
FIELD_ELEMENTS_PER_CELL: 64
|
||||||
|
# uint64(floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments'))
|
||||||
|
KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH: 4
|
||||||
|
# `uint64((FIELD_ELEMENTS_PER_BLOB * 2) // FIELD_ELEMENTS_PER_CELL)` (= 128)
|
||||||
|
NUMBER_OF_COLUMNS: 128
|
||||||
|
|||||||
@ -14,7 +14,15 @@ from eth2spec.deneb import {preset_name} as deneb
|
|||||||
'''
|
'''
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:
|
def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
|
||||||
return {
|
return {
|
||||||
'FIELD_ELEMENTS_PER_CELL': spec_object.preset_vars['FIELD_ELEMENTS_PER_CELL'].value,
|
'FIELD_ELEMENTS_PER_CELL': spec_object.preset_vars['FIELD_ELEMENTS_PER_CELL'].value,
|
||||||
|
'NUMBER_OF_COLUMNS': spec_object.preset_vars['NUMBER_OF_COLUMNS'].value,
|
||||||
|
'FIELD_ELEMENTS_PER_CELL': spec_object.preset_vars['FIELD_ELEMENTS_PER_CELL'].value,
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
|
||||||
|
return {
|
||||||
|
'KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH': spec_object.preset_vars['KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH'].value,
|
||||||
}
|
}
|
||||||
|
|||||||
241
specs/_features/eip7594/das-core.md
Normal file
241
specs/_features/eip7594/das-core.md
Normal file
@ -0,0 +1,241 @@
|
|||||||
|
# EIP-7594 -- Data Availability Sampling Core
|
||||||
|
|
||||||
|
**Notice**: This document is a work-in-progress for researchers and implementers.
|
||||||
|
|
||||||
|
## Table of contents
|
||||||
|
|
||||||
|
<!-- TOC -->
|
||||||
|
<!-- START doctoc generated TOC please keep comment here to allow auto update -->
|
||||||
|
<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
|
||||||
|
|
||||||
|
- [Custom types](#custom-types)
|
||||||
|
- [Configuration](#configuration)
|
||||||
|
- [Data size](#data-size)
|
||||||
|
- [Custody setting](#custody-setting)
|
||||||
|
- [Helper functions](#helper-functions)
|
||||||
|
- [`get_custody_lines`](#get_custody_lines)
|
||||||
|
- [`compute_extended_data`](#compute_extended_data)
|
||||||
|
- [`compute_extended_matrix`](#compute_extended_matrix)
|
||||||
|
- [`compute_samples_and_proofs`](#compute_samples_and_proofs)
|
||||||
|
- [`get_data_column_sidecars`](#get_data_column_sidecars)
|
||||||
|
- [Custody](#custody)
|
||||||
|
- [Custody requirement](#custody-requirement)
|
||||||
|
- [Public, deterministic selection](#public-deterministic-selection)
|
||||||
|
- [Peer discovery](#peer-discovery)
|
||||||
|
- [Extended data](#extended-data)
|
||||||
|
- [Column gossip](#column-gossip)
|
||||||
|
- [Parameters](#parameters)
|
||||||
|
- [Reconstruction and cross-seeding](#reconstruction-and-cross-seeding)
|
||||||
|
- [Peer sampling](#peer-sampling)
|
||||||
|
- [Peer scoring](#peer-scoring)
|
||||||
|
- [DAS providers](#das-providers)
|
||||||
|
- [A note on fork choice](#a-note-on-fork-choice)
|
||||||
|
- [FAQs](#faqs)
|
||||||
|
- [Row (blob) custody](#row-blob-custody)
|
||||||
|
- [Subnet stability](#subnet-stability)
|
||||||
|
|
||||||
|
<!-- END doctoc generated TOC please keep comment here to allow auto update -->
|
||||||
|
<!-- /TOC -->
|
||||||
|
|
||||||
|
## Custom types
|
||||||
|
|
||||||
|
We define the following Python custom types for type hinting and readability:
|
||||||
|
|
||||||
|
| Name | SSZ equivalent | Description |
|
||||||
|
| - | - | - |
|
||||||
|
| `DataCell` | `Vector[BLSFieldElement, FIELD_ELEMENTS_PER_CELL]` | The data unit of a cell in the extended data matrix |
|
||||||
|
| `DataColumn` | `List[DataCell, MAX_BLOBS_PER_BLOCK]` | The data of each column in EIP7594 |
|
||||||
|
| `ExtendedMatrix` | `List[DataCell, MAX_BLOBS_PER_BLOCK * NUMBER_OF_COLUMNS]` | The full data with blobs and one-dimensional erasure coding extension |
|
||||||
|
| `FlatExtendedMatrix` | `List[BLSFieldElement, MAX_BLOBS_PER_BLOCK * FIELD_ELEMENTS_PER_BLOB * NUMBER_OF_COLUMNS]` | The flattened format of `ExtendedMatrix` |
|
||||||
|
| `LineIndex` | `uint64` | The index of the rows or columns in `FlatExtendedMatrix` matrix |
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
### Data size
|
||||||
|
|
||||||
|
| Name | Value | Description |
|
||||||
|
| - | - | - |
|
||||||
|
| `FIELD_ELEMENTS_PER_CELL` | `uint64(2**6)` (= 64) | Elements per `DataCell` |
|
||||||
|
| `NUMBER_OF_COLUMNS` | `uint64((FIELD_ELEMENTS_PER_BLOB * 2) // FIELD_ELEMENTS_PER_CELL)` (= 128) | Number of columns in the extended data matrix. |
|
||||||
|
|
||||||
|
### Custody setting
|
||||||
|
|
||||||
|
| Name | Value | Description |
|
||||||
|
| - | - | - |
|
||||||
|
| `SAMPLES_PER_SLOT` | `8` | Number of random samples a node queries per slot |
|
||||||
|
| `CUSTODY_REQUIREMENT` | `2` | Minimum number of columns an honest node custodies and serves samples from |
|
||||||
|
| `TARGET_NUMBER_OF_PEERS` | `70` | Suggested minimum peer count |
|
||||||
|
|
||||||
|
### Helper functions
|
||||||
|
|
||||||
|
#### `get_custody_lines`
|
||||||
|
|
||||||
|
```python
|
||||||
|
def get_custody_lines(node_id: NodeID, custody_size: uint64) -> Sequence[LineIndex]:
|
||||||
|
assert custody_size <= NUMBER_OF_COLUMNS
|
||||||
|
all_items = list(range(NUMBER_OF_COLUMNS))
|
||||||
|
line_index = node_id % NUMBER_OF_COLUMNS
|
||||||
|
return [LineIndex(all_items[(line_index + i) % len(all_items)]) for i in range(custody_size)]
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `compute_extended_data`
|
||||||
|
|
||||||
|
```python
|
||||||
|
def compute_extended_data(data: Sequence[BLSFieldElement]) -> Sequence[BLSFieldElement]:
|
||||||
|
# TODO
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `compute_extended_matrix`
|
||||||
|
|
||||||
|
```python
|
||||||
|
def compute_extended_matrix(blobs: Sequence[Blob]) -> FlatExtendedMatrix:
|
||||||
|
matrix = [compute_extended_data(blob) for blob in blobs]
|
||||||
|
return FlatExtendedMatrix(matrix)
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `compute_samples_and_proofs`
|
||||||
|
|
||||||
|
```python
|
||||||
|
def compute_samples_and_proofs(blob: Blob) -> Tuple[
|
||||||
|
Vector[DataCell, NUMBER_OF_COLUMNS],
|
||||||
|
Vector[KZGProof, NUMBER_OF_COLUMNS]]:
|
||||||
|
"""
|
||||||
|
Defined in polynomial-commitments-sampling.md
|
||||||
|
"""
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `get_data_column_sidecars`
|
||||||
|
|
||||||
|
```python
|
||||||
|
def get_data_column_sidecars(signed_block: SignedBeaconBlock,
|
||||||
|
blobs: Sequence[Blob]) -> Sequence[DataColumnSidecar]:
|
||||||
|
signed_block_header = compute_signed_block_header(signed_block)
|
||||||
|
block = signed_block.message
|
||||||
|
kzg_commitments_inclusion_proof = compute_merkle_proof(
|
||||||
|
block.body,
|
||||||
|
get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments'),
|
||||||
|
)
|
||||||
|
cells_and_proofs = [compute_samples_and_proofs(blob) for blob in blobs]
|
||||||
|
blob_count = len(blobs)
|
||||||
|
cells = [cells_and_proofs[i][0] for i in range(blob_count)]
|
||||||
|
proofs = [cells_and_proofs[i][1] for i in range(blob_count)]
|
||||||
|
sidecars = []
|
||||||
|
for column_index in range(NUMBER_OF_COLUMNS):
|
||||||
|
column = DataColumn([cells[row_index][column_index]
|
||||||
|
for row_index in range(blob_count)])
|
||||||
|
kzg_proof_of_column = [proofs[row_index][column_index]
|
||||||
|
for row_index in range(blob_count)]
|
||||||
|
sidecars.append(DataColumnSidecar(
|
||||||
|
index=column_index,
|
||||||
|
column=column,
|
||||||
|
kzg_commitments=block.body.blob_kzg_commitments,
|
||||||
|
kzg_proofs=kzg_proof_of_column,
|
||||||
|
signed_block_header=signed_block_header,
|
||||||
|
kzg_commitments_inclusion_proof=kzg_commitments_inclusion_proof,
|
||||||
|
))
|
||||||
|
return sidecars
|
||||||
|
```
|
||||||
|
|
||||||
|
## Custody
|
||||||
|
|
||||||
|
### Custody requirement
|
||||||
|
|
||||||
|
Each node downloads and custodies a minimum of `CUSTODY_REQUIREMENT` columns per slot. The particular columns that the node is required to custody are selected pseudo-randomly (more on this below).
|
||||||
|
|
||||||
|
A node *may* choose to custody and serve more than the minimum honesty requirement. Such a node explicitly advertises a number greater than `CUSTODY_REQUIREMENT` via the peer discovery mechanism -- for example, in their ENR (e.g. `custody_lines: 8` if the node custodies `8` columns each slot) -- up to a `NUMBER_OF_COLUMNS` (i.e. a super-full node).
|
||||||
|
|
||||||
|
A node stores the custodied columns for the duration of the pruning period and responds to peer requests for samples on those columns.
|
||||||
|
|
||||||
|
### Public, deterministic selection
|
||||||
|
|
||||||
|
The particular columns that a node custodies are selected pseudo-randomly as a function (`get_custody_lines`) of the node-id and custody size -- importantly this function can be run by any party as the inputs are all public.
|
||||||
|
|
||||||
|
*Note*: increasing the `custody_size` parameter for a given `node_id` extends the returned list (rather than being an entirely new shuffle) such that if `custody_size` is unknown, the default `CUSTODY_REQUIREMENT` will be correct for a subset of the node's custody.
|
||||||
|
|
||||||
|
## Peer discovery
|
||||||
|
|
||||||
|
At each slot, a node needs to be able to readily sample from *any* set of columns. To this end, a node should find and maintain a set of diverse and reliable peers that can regularly satisfy their sampling demands.
|
||||||
|
|
||||||
|
A node runs a background peer discovery process, maintaining at least `TARGET_NUMBER_OF_PEERS` of various custody distributions (both custody_size and column assignments). The combination of advertised `custody_size` size and public node-id make this readily and publicly accessible.
|
||||||
|
|
||||||
|
`TARGET_NUMBER_OF_PEERS` should be tuned upward in the event of failed sampling.
|
||||||
|
|
||||||
|
*Note*: while high-capacity and super-full nodes are high value with respect to satisfying sampling requirements, a node should maintain a distribution across node capacities as to not centralize the p2p graph too much (in the extreme becomes hub/spoke) and to distribute sampling load better across all nodes.
|
||||||
|
|
||||||
|
*Note*: A DHT-based peer discovery mechanism is expected to be utilized in the above. The beacon-chain network currently utilizes discv5 in a similar method as described for finding peers of particular distributions of attestation subnets. Additional peer discovery methods are valuable to integrate (e.g., latent peer discovery via libp2p gossipsub) to add a defense in breadth against one of the discovery methods being attacked.
|
||||||
|
|
||||||
|
## Extended data
|
||||||
|
|
||||||
|
In this construction, we extend the blobs using a one-dimensional erasure coding extension. The matrix comprises maximum `MAX_BLOBS_PER_BLOCK` rows and fixed `NUMBER_OF_COLUMNS` columns, with each row containing a `Blob` and its corresponding extension.
|
||||||
|
|
||||||
|
## Column gossip
|
||||||
|
|
||||||
|
### Parameters
|
||||||
|
|
||||||
|
For each column -- use `data_column_sidecar_{subnet_id}` subnets, where each column index maps to the `subnet_id`. The sidecars can be computed with `get_data_column_sidecars(signed_block: SignedBeaconBlock, blobs: Sequence[Blob])` helper.
|
||||||
|
|
||||||
|
To custody a particular column, a node joins the respective gossip subnet. Verifiable samples from their respective column are gossiped on the assigned subnet.
|
||||||
|
|
||||||
|
### Reconstruction and cross-seeding
|
||||||
|
|
||||||
|
If the node obtains 50%+ of all the columns, they can reconstruct the full data matrix via `recover_samples_impl` helper.
|
||||||
|
|
||||||
|
If a node fails to sample a peer or fails to get a column on the column subnet, a node can utilize the Req/Resp message to query the missing column from other peers.
|
||||||
|
|
||||||
|
Once the node obtain the column, the node should send the missing columns to the column subnets.
|
||||||
|
|
||||||
|
*Note*: A node always maintains a matrix view of the rows and columns they are following, able to cross-reference and cross-seed in either direction.
|
||||||
|
|
||||||
|
*Note*: There are timing considerations to analyze -- at what point does a node consider samples missing and choose to reconstruct and cross-seed.
|
||||||
|
|
||||||
|
*Note*: There may be anti-DoS and quality-of-service considerations around how to send samples and consider samples -- is each individual sample a message or are they sent in aggregate forms.
|
||||||
|
|
||||||
|
## Peer sampling
|
||||||
|
|
||||||
|
At each slot, a node makes (locally randomly determined) `SAMPLES_PER_SLOT` queries for samples from their peers via `DataColumnSidecarByRoot` request. A node utilizes `get_custody_lines` helper to determine which peer(s) to request from. If a node has enough good/honest peers across all rows and columns, this has a high chance of success.
|
||||||
|
|
||||||
|
## Peer scoring
|
||||||
|
|
||||||
|
Due to the deterministic custody functions, a node knows exactly what a peer should be able to respond to. In the event that a peer does not respond to samples of their custodied rows/columns, a node may downscore or disconnect from a peer.
|
||||||
|
|
||||||
|
## DAS providers
|
||||||
|
|
||||||
|
A DAS provider is a consistently-available-for-DAS-queries, super-full (or high capacity) node. To the p2p, these look just like other nodes but with high advertised capacity, and they should generally be able to be latently found via normal discovery.
|
||||||
|
|
||||||
|
DAS providers can also be found out-of-band and configured into a node to connect to directly and prioritize. Nodes can add some set of these to their local configuration for persistent connection to bolster their DAS quality of service.
|
||||||
|
|
||||||
|
Such direct peering utilizes a feature supported out of the box today on all nodes and can complement (and reduce attackability and increase quality-of-service) alternative peer discovery mechanisms.
|
||||||
|
|
||||||
|
## A note on fork choice
|
||||||
|
|
||||||
|
*Fork choice spec TBD, but it will just be a replacement of `is_data_available()` call in Deneb with column sampling instead of full download. Note the `is_data_available(slot_N)` will likely do a `-1` follow distance so that you just need to check the availability of slot `N-1` for slot `N` (starting with the block proposer of `N`).*
|
||||||
|
|
||||||
|
The fork choice rule (essentially a DA filter) is *orthogonal to a given DAS design*, other than the efficiency of a particular design impacting it.
|
||||||
|
|
||||||
|
In any DAS design, there are probably a few degrees of freedom around timing, acceptability of short-term re-orgs, etc.
|
||||||
|
|
||||||
|
For example, the fork choice rule might require validators to do successful DAS on slot N to be able to include block of slot `N` in its fork choice. That's the tightest DA filter. But trailing filters are also probably acceptable, knowing that there might be some failures/short re-orgs but that they don't hurt the aggregate security. For example, the rule could be — DAS must be completed for slot N-1 for a child block in N to be included in the fork choice.
|
||||||
|
|
||||||
|
Such trailing techniques and their analysis will be valuable for any DAS construction. The question is — can you relax how quickly you need to do DA and in the worst case not confirm unavailable data via attestations/finality, and what impact does it have on short-term re-orgs and fast confirmation rules.
|
||||||
|
|
||||||
|
## FAQs
|
||||||
|
|
||||||
|
### Row (blob) custody
|
||||||
|
|
||||||
|
In the one-dimension construction, a node samples the peers by requesting the whole `DataColumn`. In reconstruction, a node can reconstruct all the blobs by 50% of the columns. Note that nodes can still download the row via `blob_sidecar_{subnet_id}` subnets.
|
||||||
|
|
||||||
|
The potential benefits of having row custody could include:
|
||||||
|
|
||||||
|
1. Allow for more "natural" distribution of data to consumers -- e.g., roll-ups -- but honestly, they won't know a priori which row their blob is going to be included in in the block, so they would either need to listen to all rows or download a particular row after seeing the block. The former looks just like listening to column [0, N) and the latter is req/resp instead of gossiping.
|
||||||
|
2. Help with some sort of distributed reconstruction. Those with full rows can compute extensions and seed missing samples to the network. This would either need to be able to send individual points on the gossip or would need some sort of req/resp faculty, potentially similar to an `IHAVEPOINTBITFIELD` and `IWANTSAMPLE`.
|
||||||
|
|
||||||
|
However, for simplicity, we don't assign row custody assignments to nodes in the current design.
|
||||||
|
|
||||||
|
|
||||||
|
### Subnet stability
|
||||||
|
|
||||||
|
To start with a simple, stable backbone, for now, we don't shuffle the subnet assignments via the deterministic custody selection helper `get_custody_lines`. However, staggered rotation likely needs to happen on the order of the pruning period to ensure subnets can be utilized for recovery. For example, introducing an `epoch` argument allows the function to maintain stability over many epochs.
|
||||||
180
specs/_features/eip7594/p2p-interface.md
Normal file
180
specs/_features/eip7594/p2p-interface.md
Normal file
@ -0,0 +1,180 @@
|
|||||||
|
# EIP-7594 -- Networking
|
||||||
|
|
||||||
|
**Notice**: This document is a work-in-progress for researchers and implementers.
|
||||||
|
|
||||||
|
## Table of contents
|
||||||
|
|
||||||
|
<!-- TOC -->
|
||||||
|
<!-- START doctoc generated TOC please keep comment here to allow auto update -->
|
||||||
|
<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
|
||||||
|
|
||||||
|
- [Modifications in EIP-7594](#modifications-in-eip-7594)
|
||||||
|
- [Preset](#preset)
|
||||||
|
- [Configuration](#configuration)
|
||||||
|
- [Containers](#containers)
|
||||||
|
- [`DataColumnSidecar`](#datacolumnsidecar)
|
||||||
|
- [`DataColumnIdentifier`](#datacolumnidentifier)
|
||||||
|
- [Helpers](#helpers)
|
||||||
|
- [`verify_data_column_sidecar_kzg_proof`](#verify_data_column_sidecar_kzg_proof)
|
||||||
|
- [`verify_data_column_sidecar_inclusion_proof`](#verify_data_column_sidecar_inclusion_proof)
|
||||||
|
- [`compute_subnet_for_data_column_sidecar`](#compute_subnet_for_data_column_sidecar)
|
||||||
|
- [The gossip domain: gossipsub](#the-gossip-domain-gossipsub)
|
||||||
|
- [Topics and messages](#topics-and-messages)
|
||||||
|
- [Samples subnets](#samples-subnets)
|
||||||
|
- [`data_column_sidecar_{subnet_id}`](#data_column_sidecar_subnet_id)
|
||||||
|
- [The Req/Resp domain](#the-reqresp-domain)
|
||||||
|
- [Messages](#messages)
|
||||||
|
- [DataColumnSidecarByRoot v1](#datacolumnsidecarbyroot-v1)
|
||||||
|
|
||||||
|
<!-- END doctoc generated TOC please keep comment here to allow auto update -->
|
||||||
|
<!-- /TOC -->
|
||||||
|
|
||||||
|
## Modifications in EIP-7594
|
||||||
|
|
||||||
|
### Preset
|
||||||
|
|
||||||
|
| Name | Value | Description |
|
||||||
|
|------------------------------------------|-----------------------------------|---------------------------------------------------------------------|
|
||||||
|
| `KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH` | `uint64(floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')))` (= 4) | <!-- predefined --> Merkle proof index for `blob_kzg_commitments` |
|
||||||
|
|
||||||
|
### Configuration
|
||||||
|
|
||||||
|
| Name | Value | Description |
|
||||||
|
|------------------------------------------|-----------------------------------|---------------------------------------------------------------------|
|
||||||
|
| `DATA_COLUMN_SIDECAR_SUBNET_COUNT` | `32` | The number of data column sidecar subnets used in the gossipsub protocol. |
|
||||||
|
|
||||||
|
### Containers
|
||||||
|
|
||||||
|
#### `DataColumnSidecar`
|
||||||
|
|
||||||
|
```python
|
||||||
|
class DataColumnSidecar(Container):
|
||||||
|
index: LineIndex # Index of column in extended matrix
|
||||||
|
column: DataColumn
|
||||||
|
kzg_commitments: List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK]
|
||||||
|
kzg_proofs: List[KZGProof, MAX_BLOB_COMMITMENTS_PER_BLOCK]
|
||||||
|
signed_block_header: SignedBeaconBlockHeader
|
||||||
|
kzg_commitments_inclusion_proof: Vector[Bytes32, KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH]
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `DataColumnIdentifier`
|
||||||
|
|
||||||
|
```python
|
||||||
|
class DataColumnIdentifier(Container):
|
||||||
|
block_root: Root
|
||||||
|
index: LineIndex
|
||||||
|
```
|
||||||
|
|
||||||
|
### Helpers
|
||||||
|
|
||||||
|
##### `verify_data_column_sidecar_kzg_proof`
|
||||||
|
|
||||||
|
```python
|
||||||
|
def verify_data_column_sidecar_kzg_proof(sidecar: DataColumnSidecar) -> bool:
|
||||||
|
"""
|
||||||
|
Verify if the proofs are correct
|
||||||
|
"""
|
||||||
|
row_ids = [LineIndex(i) for i in range(len(sidecar.column))]
|
||||||
|
assert len(sidecar.column) == len(sidecar.kzg_commitments) == len(sidecar.kzg_proofs)
|
||||||
|
|
||||||
|
# KZG batch verifies that the cells match the corresponding commitments and proofs
|
||||||
|
return verify_cell_proof_batch(
|
||||||
|
row_commitments=sidecar.kzg_commitments,
|
||||||
|
row_ids=row_ids, # all rows
|
||||||
|
column_ids=[sidecar.index],
|
||||||
|
datas=sidecar.column,
|
||||||
|
proofs=sidecar.kzg_proofs,
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
##### `verify_data_column_sidecar_inclusion_proof`
|
||||||
|
|
||||||
|
```python
|
||||||
|
def verify_data_column_sidecar_inclusion_proof(sidecar: DataColumnSidecar) -> bool:
|
||||||
|
"""
|
||||||
|
Verify if the given KZG commitments included in the given beacon block.
|
||||||
|
"""
|
||||||
|
gindex = get_subtree_index(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments'))
|
||||||
|
return is_valid_merkle_branch(
|
||||||
|
leaf=hash_tree_root(sidecar.kzg_commitments),
|
||||||
|
branch=sidecar.kzg_commitments_inclusion_proof,
|
||||||
|
depth=KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH,
|
||||||
|
index=gindex,
|
||||||
|
root=sidecar.signed_block_header.message.body_root,
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
##### `compute_subnet_for_data_column_sidecar`
|
||||||
|
|
||||||
|
```python
|
||||||
|
def compute_subnet_for_data_column_sidecar(column_index: LineIndex) -> SubnetID:
|
||||||
|
return SubnetID(column_index % DATA_COLUMN_SIDECAR_SUBNET_COUNT)
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
### The gossip domain: gossipsub
|
||||||
|
|
||||||
|
Some gossip meshes are upgraded in the EIP-7594 fork to support upgraded types.
|
||||||
|
|
||||||
|
#### Topics and messages
|
||||||
|
|
||||||
|
##### Samples subnets
|
||||||
|
|
||||||
|
###### `data_column_sidecar_{subnet_id}`
|
||||||
|
|
||||||
|
This topic is used to propagate column sidecars, where each column maps to some `subnet_id`.
|
||||||
|
|
||||||
|
The *type* of the payload of this topic is `DataColumnSidecar`.
|
||||||
|
|
||||||
|
The following validations MUST pass before forwarding the `sidecar: DataColumnSidecar` on the network, assuming the alias `block_header = sidecar.signed_block_header.message`:
|
||||||
|
|
||||||
|
- _[REJECT]_ The sidecar's index is consistent with `NUMBER_OF_COLUMNS` -- i.e. `sidecar.index < NUMBER_OF_COLUMNS`.
|
||||||
|
- _[REJECT]_ The sidecar is for the correct subnet -- i.e. `compute_subnet_for_data_column_sidecar(sidecar.index) == subnet_id`.
|
||||||
|
- _[IGNORE]_ The sidecar is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `block_header.slot <= current_slot` (a client MAY queue future sidecars for processing at the appropriate slot).
|
||||||
|
- _[IGNORE]_ The sidecar is from a slot greater than the latest finalized slot -- i.e. validate that `block_header.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)`
|
||||||
|
- _[REJECT]_ The proposer signature of `sidecar.signed_block_header`, is valid with respect to the `block_header.proposer_index` pubkey.
|
||||||
|
- _[IGNORE]_ The sidecar's block's parent (defined by `block_header.parent_root`) has been seen (via both gossip and non-gossip sources) (a client MAY queue sidecars for processing once the parent block is retrieved).
|
||||||
|
- _[REJECT]_ The sidecar's block's parent (defined by `block_header.parent_root`) passes validation.
|
||||||
|
- _[REJECT]_ The sidecar is from a higher slot than the sidecar's block's parent (defined by `block_header.parent_root`).
|
||||||
|
- _[REJECT]_ The current finalized_checkpoint is an ancestor of the sidecar's block -- i.e. `get_checkpoint_block(store, block_header.parent_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root`.
|
||||||
|
- _[REJECT]_ The sidecar's `kzg_commitments` field inclusion proof is valid as verified by `verify_data_column_sidecar_inclusion_proof(sidecar)`.
|
||||||
|
- _[REJECT]_ The sidecar's column data is valid as verified by `verify_data_column_sidecar_kzg_proof(sidecar)`.
|
||||||
|
- _[IGNORE]_ The sidecar is the first sidecar for the tuple `(block_header.slot, block_header.proposer_index, sidecar.index)` with valid header signature, sidecar inclusion proof, and kzg proof.
|
||||||
|
- _[REJECT]_ The sidecar is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `block_header.parent_root`/`block_header.slot`).
|
||||||
|
If the `proposer_index` cannot immediately be verified against the expected shuffling, the sidecar MAY be queued for later processing while proposers for the block's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message.
|
||||||
|
|
||||||
|
*Note:* In the `verify_data_column_sidecar_inclusion_proof(sidecar)` check, for all the sidecars of the same block, it verifies against the same set of `kzg_commitments` of the given beacon beacon. Client can choose to cache the result of the arguments tuple `(sidecar.kzg_commitments, sidecar.kzg_commitments_inclusion_proof, sidecar.signed_block_header)`.
|
||||||
|
|
||||||
|
### The Req/Resp domain
|
||||||
|
|
||||||
|
#### Messages
|
||||||
|
|
||||||
|
##### DataColumnSidecarByRoot v1
|
||||||
|
|
||||||
|
**Protocol ID:** `/eth2/beacon_chain/req/data_column_sidecar_by_root/1/`
|
||||||
|
|
||||||
|
*[New in Deneb:EIP4844]*
|
||||||
|
|
||||||
|
The `<context-bytes>` field is calculated as `context = compute_fork_digest(fork_version, genesis_validators_root)`:
|
||||||
|
|
||||||
|
[1]: # (eth2spec: skip)
|
||||||
|
|
||||||
|
| `fork_version` | Chunk SSZ type |
|
||||||
|
|--------------------------|-------------------------------|
|
||||||
|
| `EIP7594_FORK_VERSION` | `eip7594.DataColumnSidecar` |
|
||||||
|
|
||||||
|
Request Content:
|
||||||
|
|
||||||
|
```
|
||||||
|
(
|
||||||
|
DataColumnIdentifier
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Response Content:
|
||||||
|
|
||||||
|
```
|
||||||
|
(
|
||||||
|
DataColumnSidecar
|
||||||
|
)
|
||||||
|
```
|
||||||
@ -1,4 +1,4 @@
|
|||||||
# Deneb -- Polynomial Commitments
|
# EIP-7594 -- Polynomial Commitments
|
||||||
|
|
||||||
## Table of contents
|
## Table of contents
|
||||||
|
|
||||||
|
|||||||
@ -63,6 +63,19 @@ class GetPayloadResponse(object):
|
|||||||
blobs_bundle: BlobsBundle # [New in Deneb:EIP4844]
|
blobs_bundle: BlobsBundle # [New in Deneb:EIP4844]
|
||||||
```
|
```
|
||||||
|
|
||||||
|
```python
|
||||||
|
def compute_signed_block_header(signed_block: SignedBeaconBlock) -> SignedBeaconBlockHeader:
|
||||||
|
block = signed_block.message
|
||||||
|
block_header = BeaconBlockHeader(
|
||||||
|
slot=block.slot,
|
||||||
|
proposer_index=block.proposer_index,
|
||||||
|
parent_root=block.parent_root,
|
||||||
|
state_root=block.state_root,
|
||||||
|
body_root=hash_tree_root(block.body),
|
||||||
|
)
|
||||||
|
return SignedBeaconBlockHeader(message=block_header, signature=signed_block.signature)
|
||||||
|
```
|
||||||
|
|
||||||
## Protocol
|
## Protocol
|
||||||
|
|
||||||
### `ExecutionEngine`
|
### `ExecutionEngine`
|
||||||
@ -147,14 +160,7 @@ def get_blob_sidecars(signed_block: SignedBeaconBlock,
|
|||||||
blobs: Sequence[Blob],
|
blobs: Sequence[Blob],
|
||||||
blob_kzg_proofs: Sequence[KZGProof]) -> Sequence[BlobSidecar]:
|
blob_kzg_proofs: Sequence[KZGProof]) -> Sequence[BlobSidecar]:
|
||||||
block = signed_block.message
|
block = signed_block.message
|
||||||
block_header = BeaconBlockHeader(
|
signed_block_header = compute_signed_block_header(signed_block)
|
||||||
slot=block.slot,
|
|
||||||
proposer_index=block.proposer_index,
|
|
||||||
parent_root=block.parent_root,
|
|
||||||
state_root=block.state_root,
|
|
||||||
body_root=hash_tree_root(block.body),
|
|
||||||
)
|
|
||||||
signed_block_header = SignedBeaconBlockHeader(message=block_header, signature=signed_block.signature)
|
|
||||||
return [
|
return [
|
||||||
BlobSidecar(
|
BlobSidecar(
|
||||||
index=index,
|
index=index,
|
||||||
|
|||||||
@ -1,174 +0,0 @@
|
|||||||
# Peer Data Availability Sampling -- Core
|
|
||||||
|
|
||||||
**Notice**: This document is a work-in-progress for researchers and implementers.
|
|
||||||
|
|
||||||
## Table of contents
|
|
||||||
|
|
||||||
<!-- TOC -->
|
|
||||||
<!-- START doctoc generated TOC please keep comment here to allow auto update -->
|
|
||||||
<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
|
|
||||||
|
|
||||||
- [Custom types](#custom-types)
|
|
||||||
- [Configuration](#configuration)
|
|
||||||
- [Data size](#data-size)
|
|
||||||
- [Custody setting](#custody-setting)
|
|
||||||
- [Helper functions](#helper-functions)
|
|
||||||
- [`cycle`](#cycle)
|
|
||||||
- [`get_custody_lines`](#get_custody_lines)
|
|
||||||
- [Honest peer guide](#honest-peer-guide)
|
|
||||||
- [Custody](#custody)
|
|
||||||
- [1. Custody](#1-custody)
|
|
||||||
- [`CUSTODY_REQUIREMENT`](#custody_requirement)
|
|
||||||
- [Public, deterministic selection](#public-deterministic-selection)
|
|
||||||
- [2. Peer discovery](#2-peer-discovery)
|
|
||||||
- [3. Row/Column gossip](#3-rowcolumn-gossip)
|
|
||||||
- [Parameters](#parameters)
|
|
||||||
- [Reconstruction and cross-seeding](#reconstruction-and-cross-seeding)
|
|
||||||
- [4. Peer sampling](#4-peer-sampling)
|
|
||||||
- [5. Peer scoring](#5-peer-scoring)
|
|
||||||
- [6. DAS providers](#6-das-providers)
|
|
||||||
- [7. A note on fork choice](#7-a-note-on-fork-choice)
|
|
||||||
|
|
||||||
<!-- END doctoc generated TOC please keep comment here to allow auto update -->
|
|
||||||
<!-- /TOC -->
|
|
||||||
|
|
||||||
## Custom types
|
|
||||||
|
|
||||||
| Name | SSZ equivalent | Description |
|
|
||||||
| - | - | - |
|
|
||||||
| `SampleIndex` | `uint64` | A sample index, corresponding to chunk of extended data |
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
### Data size
|
|
||||||
|
|
||||||
| Name | Value | Description |
|
|
||||||
| - | - | - |
|
|
||||||
| `NUMBER_OF_ROWS` | `uint64(2**4)` (= 32) | Number of rows in the 2D data array |
|
|
||||||
| `NUMBER_OF_COLUMNS` | `uint64(2**4)` (= 32) | Number of columns in the 2D data array |
|
|
||||||
| `DATA_PER_BLOB` | `FIELD_ELEMENTS_PER_BLOB * BYTES_PER_FIELD_ELEMENT` | Bytes |
|
|
||||||
| `DATA_PER_SLOT` | `MAX_BLOBS_PER_BLOCK * BLOB_SIZE * 4` | Bytes. Including the extension. |
|
|
||||||
| `DATA_PER_ROW` | `DATA_PER_SLOT / NUMBER_OF_ROWS` | |
|
|
||||||
| `DATA_PER_COLUMN` | `DATA_PER_SLOT / NUMBER_OF_COLUMNS` | |
|
|
||||||
| `DATA_PER_SAMPLE` | `DATA_PER_SLOT / (NUMBER_OF_COLUMNS * NUMBER_OF_ROWS)` |
|
|
||||||
|
|
||||||
### Custody setting
|
|
||||||
|
|
||||||
| Name | Value | Description |
|
|
||||||
| - | - | - |
|
|
||||||
| `SAMPLES_PER_SLOT` | `70` |
|
|
||||||
| `CUSTODY_REQUIREMENT` | `2` | |
|
|
||||||
|
|
||||||
### Helper functions
|
|
||||||
|
|
||||||
#### `cycle`
|
|
||||||
```python
|
|
||||||
def cycle(seq: Sequence[Any], start: int) -> Any:
|
|
||||||
while True:
|
|
||||||
yield seq[start]
|
|
||||||
start = (start + 1) % len(seq)
|
|
||||||
```
|
|
||||||
|
|
||||||
#### `get_custody_lines`
|
|
||||||
|
|
||||||
```python
|
|
||||||
def get_custody_lines(node_id: int, epoch: int, custody_size: int, line_type: LineType) -> list[int]:
|
|
||||||
bound = NUMBER_OF_ROWS if line_type else NUMBER_OF_COLUMNS
|
|
||||||
all_items = list(range(bound))
|
|
||||||
line_index = (node_id + epoch) % bound
|
|
||||||
iterator = cycle(all_items, line_index)
|
|
||||||
return [next(iterator) for _ in range(custody_size)]
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Honest peer guide
|
|
||||||
|
|
||||||
## Custody
|
|
||||||
|
|
||||||
|
|
||||||
#### 1. Custody
|
|
||||||
|
|
||||||
##### `CUSTODY_REQUIREMENT`
|
|
||||||
|
|
||||||
Each node downloads and custodies a minimum of `CUSTODY_REQUIREMENT` rows and `CUSTODY_REQUIREMENT` columns per slot. The particular rows and columns that the node is required to custody are selected pseudo-randomly (more on this below).
|
|
||||||
|
|
||||||
A node *may* choose to custody and serve more than the minimum honesty requirement. Such a node explicitly advertises a number greater than `CUSTODY_REQUIREMENT` via the peer discovery mechanism -- for example, in their ENR (e.g. `custody_lines: 8` if the node custodies `8` rows and `8` columns each slot) -- up to a maximum of `max(NUMBER_OF_ROWS, NUMBER_OF_COLUMNS)` (i.e. a super-full node).
|
|
||||||
|
|
||||||
A node stores the custodied rows/columns for the duration of the pruning period and responds to peer requests for samples on those rows/columns.
|
|
||||||
|
|
||||||
##### Public, deterministic selection
|
|
||||||
|
|
||||||
The particular rows and columns that a node custodies are selected pseudo-randomly as a function of the node-id, epoch, and custody size (sample function interface: `get_custody_lines(config: Config, node_id: int, epoch: int, custody_size: int, line_type: LineType) -> list[int]` and column variant) -- importantly this function can be run by any party as the inputs are all public.
|
|
||||||
|
|
||||||
*Note*: `line_type` could be `LineType.ROW` or `LineType.COLUMN`.
|
|
||||||
|
|
||||||
*Note*: increasing the `custody_size` parameter for a given `node_id` and `epoch` extends the returned list (rather than being an entirely new shuffle) such that if `custody_size` is unknown, the default `CUSTODY_REQUIREMENT` will be correct for a subset of the node's custody.
|
|
||||||
|
|
||||||
*Note*: Even though this function accepts `epoch` as an input, the function can be tuned to remain stable for many epochs depending on network/subnet stability requirements. There is a trade-off between rigidity of the network and the depth to which a subnet can be utilized for recovery. To ensure subnets can be utilized for recovery, staggered rotation needs to happen likely on the order of the prune period.
|
|
||||||
|
|
||||||
#### 2. Peer discovery
|
|
||||||
|
|
||||||
At each slot, a node needs to be able to readily sample from *any* set of rows and columns. To this end, a node should find and maintain a set of diverse and reliable peers that can regularly satisfy their sampling demands.
|
|
||||||
|
|
||||||
A node runs a background peer discovery process, maintaining at least `NUMBER_OF_PEERS` of various custody distributions (both custody_size and row/column assignments). The combination of advertised `custody_size` size and public node-id make this readily, publicly accessible.
|
|
||||||
|
|
||||||
`NUMBER_OF_PEERS` should be tuned upward in the event of failed sampling.
|
|
||||||
|
|
||||||
*Note*: while high-capacity and super-full nodes are high value with respect to satisfying sampling requirements, a node should maintain a distribution across node capacities as to not centralize the p2p graph too much (in the extreme becomes hub/spoke) and to distribute sampling load better across all nodes.
|
|
||||||
|
|
||||||
*Note*: A DHT-based peer discovery mechanism is expected to be utilized in the above. The beacon-chain network currently utilizes discv5 in a similar method as described for finding peers of particular distributions of attestation subnets. Additional peer discovery methods are valuable to integrate (e.g. latent peer discovery via libp2p gossipsub) to add a defense in breadth against one of the discovery methods being attacked.
|
|
||||||
|
|
||||||
#### 3. Row/Column gossip
|
|
||||||
|
|
||||||
##### Parameters
|
|
||||||
|
|
||||||
There are both `NUMBER_OF_ROWS` row and `NUMBER_OF_COLUMNS` column gossip topics.
|
|
||||||
|
|
||||||
1. For each column -- `row_x` for `x` from `0` to `NUMBER_OF_COLUMNS` (non-inclusive).
|
|
||||||
2. For each row -- `column_y` for `y` from `0` to `NUMBER_OF_ROWS` (non-inclusive).
|
|
||||||
|
|
||||||
To custody a particular row or column, a node joins the respective gossip subnet. Verifiable samples from their respective row/column are gossiped on the assigned subnet.
|
|
||||||
|
|
||||||
|
|
||||||
##### Reconstruction and cross-seeding
|
|
||||||
|
|
||||||
In the event a node does *not* receive all samples for a given row/column but does receive enough to reconstruct (e.g. 50%+, a function of coding rate), the node should reconstruct locally and send the reconstructed samples on the subnet.
|
|
||||||
|
|
||||||
Additionally, the node should send (cross-seed) any samples missing from a given row/column they are assigned to that they have obtained via an alternative method (ancillary gossip or reconstruction). E.g., if node reconstructs `row_x` and is also participating in the `column_y` subnet in which the `(x, y)` sample was missing, send the reconstructed sample to `column_y`.
|
|
||||||
|
|
||||||
*Note*: A node is always maintaining a matrix view of the rows and columns they are following, able to cross-reference and cross-seed in either direction.
|
|
||||||
|
|
||||||
*Note*: There are timing considerations to analyze -- at what point does a node consider samples missing and chooses to reconstruct and cross-seed.
|
|
||||||
|
|
||||||
*Note*: There may be anti-DoS and quality-of-service considerations around how to send samples and consider samples -- is each individual sample a message or are they sent in aggregate forms.
|
|
||||||
|
|
||||||
#### 4. Peer sampling
|
|
||||||
|
|
||||||
At each slot, a node makes (locally randomly determined) `SAMPLES_PER_SLOT` queries for samples from their peers. A node utilizes `get_custody_lines(..., line_type=LineType.ROW)`/`get_custody_lines(..., line_type=LineType.COLUMN)` to determine which peer(s) to request from. If a node has enough good/honest peers across all rows and columns, this has a high chance of success.
|
|
||||||
|
|
||||||
Upon sampling, the node sends an `DO_YOU_HAVE` packet for all samples to all peers who are determined to custody this sample according to their `get_custody_lines` results. All peers answer first with a bitfield of the samples that they have.
|
|
||||||
|
|
||||||
Upon receiving a sample, a node will pass on the sample to any node which did not previously have this sample, known by `DO_YOU_HAVE` response (but was supposed to have it according to its `get_custody_lines` results).
|
|
||||||
|
|
||||||
#### 5. Peer scoring
|
|
||||||
|
|
||||||
Due to the deterministic custody functions, a node knows exactly what a peer should be able to respond to. In the event that a peer does not respond to samples of their custodied rows/columns, a node may downscore or disconnect from a peer.
|
|
||||||
|
|
||||||
*Note*: a peer might not respond to requests either because they are dishonest (don't actually custody the data), because of bandwidth saturation (local throttling), or because they were, themselves, not able to get all the samples. In the first two cases, the peer is not of consistent DAS value and a node can/should seek to optimize for better peers. In the latter, the node can make local determinations based on repeated `DO_YOU_HAVE` queries to that peer and other peers to assess the value/honesty of the peer.
|
|
||||||
|
|
||||||
#### 6. DAS providers
|
|
||||||
|
|
||||||
A DAS provider is a consistently-available-for-DAS-queries, super-full (or high capacity) node. To the p2p, these look just like other nodes but with high advertised capacity, and they should generally be able to be latently found via normal discovery.
|
|
||||||
|
|
||||||
They can also be found out-of-band and configured into a node to connect to directly and prioritize. E.g., some L2 DAO might support 10 super-full nodes as a public good, and nodes could choose to add some set of these to their local configuration to bolster their DAS quality of service.
|
|
||||||
|
|
||||||
Such direct peering utilizes a feature supported out of the box today on all nodes and can complement (and reduce attackability) of alternative peer discovery mechanisms.
|
|
||||||
|
|
||||||
#### 7. A note on fork choice
|
|
||||||
|
|
||||||
The fork choice rule (essentially a DA filter) is *orthogonal to a given DAS design*, other than the efficiency of particular design impacting it.
|
|
||||||
|
|
||||||
In any DAS design, there are probably a few degrees of freedom around timing, acceptability of short-term re-orgs, etc.
|
|
||||||
|
|
||||||
For example, the fork choice rule might require validators to do successful DAS on slot N to be able to include block of slot N in it's fork choice. That's the tightest DA filter. But trailing filters are also probably acceptable, knowing that there might be some failures/short re-orgs but that it doesn't hurt the aggregate security. E.g. The rule could be -- DAS must be completed for slot N-1 for a child block in N to be included in the fork choice.
|
|
||||||
|
|
||||||
Such trailing techniques and their analyiss will be valuable for any DAS construction. The question is — can you relax how quickly you need to do DA and in the worst case not confirm unavailable data via attestations/finality, and what impact does it have on short-term re-orgs and fast confirmation rules.
|
|
||||||
@ -0,0 +1,74 @@
|
|||||||
|
import random
|
||||||
|
|
||||||
|
from eth2spec.test.context import (
|
||||||
|
spec_state_test,
|
||||||
|
with_eip7594_and_later,
|
||||||
|
with_test_suite_name,
|
||||||
|
)
|
||||||
|
from eth2spec.test.helpers.block import (
|
||||||
|
build_empty_block_for_next_slot,
|
||||||
|
sign_block,
|
||||||
|
)
|
||||||
|
from eth2spec.test.helpers.execution_payload import (
|
||||||
|
compute_el_block_hash,
|
||||||
|
)
|
||||||
|
from eth2spec.test.helpers.sharding import (
|
||||||
|
get_sample_opaque_tx,
|
||||||
|
)
|
||||||
|
from eth2spec.debug.random_value import (
|
||||||
|
RandomizationMode,
|
||||||
|
get_random_ssz_object,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_blob_kzg_commitments_merkle_proof_test(spec, state, rng=None):
|
||||||
|
opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_opaque_tx(spec, blob_count=1)
|
||||||
|
if rng is None:
|
||||||
|
block = build_empty_block_for_next_slot(spec, state)
|
||||||
|
else:
|
||||||
|
block = get_random_ssz_object(
|
||||||
|
rng,
|
||||||
|
spec.BeaconBlock,
|
||||||
|
max_bytes_length=2000,
|
||||||
|
max_list_length=2000,
|
||||||
|
mode=RandomizationMode,
|
||||||
|
chaos=True,
|
||||||
|
)
|
||||||
|
block.body.blob_kzg_commitments = blob_kzg_commitments
|
||||||
|
block.body.execution_payload.transactions = [opaque_tx]
|
||||||
|
block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload)
|
||||||
|
signed_block = sign_block(spec, state, block, proposer_index=0)
|
||||||
|
column_sidcars = spec.get_data_column_sidecars(signed_block, blobs)
|
||||||
|
column_sidcar = column_sidcars[0]
|
||||||
|
|
||||||
|
yield "object", block.body
|
||||||
|
kzg_commitments_inclusion_proof = column_sidcar.kzg_commitments_inclusion_proof
|
||||||
|
gindex = spec.get_generalized_index(spec.BeaconBlockBody, 'blob_kzg_commitments')
|
||||||
|
yield "proof", {
|
||||||
|
"leaf": "0x" + column_sidcar.kzg_commitments.hash_tree_root().hex(),
|
||||||
|
"leaf_index": gindex,
|
||||||
|
"branch": ['0x' + root.hex() for root in kzg_commitments_inclusion_proof]
|
||||||
|
}
|
||||||
|
assert spec.is_valid_merkle_branch(
|
||||||
|
leaf=column_sidcar.kzg_commitments.hash_tree_root(),
|
||||||
|
branch=column_sidcar.kzg_commitments_inclusion_proof,
|
||||||
|
depth=spec.floorlog2(gindex),
|
||||||
|
index=spec.get_subtree_index(gindex),
|
||||||
|
root=column_sidcar.signed_block_header.message.body_root,
|
||||||
|
)
|
||||||
|
assert spec.verify_data_column_sidecar_inclusion_proof(column_sidcar)
|
||||||
|
|
||||||
|
|
||||||
|
@with_test_suite_name("BeaconBlockBody")
|
||||||
|
@with_eip7594_and_later
|
||||||
|
@spec_state_test
|
||||||
|
def test_blob_kzg_commitments_merkle_proof__basic(spec, state):
|
||||||
|
yield from _run_blob_kzg_commitments_merkle_proof_test(spec, state)
|
||||||
|
|
||||||
|
|
||||||
|
@with_test_suite_name("BeaconBlockBody")
|
||||||
|
@with_eip7594_and_later
|
||||||
|
@spec_state_test
|
||||||
|
def test_blob_kzg_commitments_merkle_proof__random_block_1(spec, state):
|
||||||
|
rng = random.Random(1111)
|
||||||
|
yield from _run_blob_kzg_commitments_merkle_proof_test(spec, state, rng=rng)
|
||||||
@ -0,0 +1,17 @@
|
|||||||
|
from eth2spec.test.context import (
|
||||||
|
spec_test,
|
||||||
|
single_phase,
|
||||||
|
with_eip7594_and_later,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@with_eip7594_and_later
|
||||||
|
@spec_test
|
||||||
|
@single_phase
|
||||||
|
def test_invariants(spec):
|
||||||
|
assert spec.FIELD_ELEMENTS_PER_BLOB % spec.FIELD_ELEMENTS_PER_CELL == 0
|
||||||
|
assert spec.FIELD_ELEMENTS_PER_BLOB * 2 % spec.NUMBER_OF_COLUMNS == 0
|
||||||
|
assert spec.SAMPLES_PER_SLOT <= spec.NUMBER_OF_COLUMNS
|
||||||
|
assert spec.CUSTODY_REQUIREMENT <= spec.NUMBER_OF_COLUMNS
|
||||||
|
assert spec.DATA_COLUMN_SIDECAR_SUBNET_COUNT <= spec.NUMBER_OF_COLUMNS
|
||||||
|
assert spec.NUMBER_OF_COLUMNS % spec.DATA_COLUMN_SIDECAR_SUBNET_COUNT == 0
|
||||||
@ -0,0 +1,41 @@
|
|||||||
|
from eth2spec.test.context import (
|
||||||
|
expect_assertion_error,
|
||||||
|
spec_test,
|
||||||
|
single_phase,
|
||||||
|
with_eip7594_and_later,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@with_eip7594_and_later
|
||||||
|
@spec_test
|
||||||
|
@single_phase
|
||||||
|
def test_get_custody_lines_peers_within_number_of_columns(spec):
|
||||||
|
peer_count = 10
|
||||||
|
custody_size = spec.CUSTODY_REQUIREMENT
|
||||||
|
assert spec.NUMBER_OF_COLUMNS > peer_count
|
||||||
|
assignments = [spec.get_custody_lines(node_id, custody_size) for node_id in range(peer_count)]
|
||||||
|
|
||||||
|
for assignment in assignments:
|
||||||
|
assert len(assignment) == custody_size
|
||||||
|
|
||||||
|
|
||||||
|
@with_eip7594_and_later
|
||||||
|
@spec_test
|
||||||
|
@single_phase
|
||||||
|
def test_get_custody_lines_peers_more_than_number_of_columns(spec):
|
||||||
|
peer_count = 200
|
||||||
|
custody_size = spec.CUSTODY_REQUIREMENT
|
||||||
|
assert spec.NUMBER_OF_COLUMNS < peer_count
|
||||||
|
assignments = [spec.get_custody_lines(node_id, custody_size) for node_id in range(peer_count)]
|
||||||
|
|
||||||
|
for assignment in assignments:
|
||||||
|
assert len(assignment) == custody_size
|
||||||
|
|
||||||
|
|
||||||
|
@with_eip7594_and_later
|
||||||
|
@spec_test
|
||||||
|
@single_phase
|
||||||
|
def test_get_custody_lines_custody_size_more_than_number_of_columns(spec):
|
||||||
|
node_id = 1
|
||||||
|
custody_size = spec.NUMBER_OF_COLUMNS + 1
|
||||||
|
expect_assertion_error(lambda: spec.get_custody_lines(node_id, custody_size))
|
||||||
@ -0,0 +1,19 @@
|
|||||||
|
from eth2spec.test.context import (
|
||||||
|
spec_test,
|
||||||
|
single_phase,
|
||||||
|
with_eip7594_and_later,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@with_eip7594_and_later
|
||||||
|
@spec_test
|
||||||
|
@single_phase
|
||||||
|
def test_compute_subnet_for_data_column_sidecar(spec):
|
||||||
|
subnet_results = []
|
||||||
|
for column_index in range(spec.DATA_COLUMN_SIDECAR_SUBNET_COUNT):
|
||||||
|
subnet_results.append(spec.compute_subnet_for_data_column_sidecar(column_index))
|
||||||
|
# no duplicates
|
||||||
|
assert len(subnet_results) == len(set(subnet_results))
|
||||||
|
# next one should be duplicate
|
||||||
|
next_subnet = spec.compute_subnet_for_data_column_sidecar(spec.DATA_COLUMN_SIDECAR_SUBNET_COUNT)
|
||||||
|
assert next_subnet == subnet_results[0]
|
||||||
@ -0,0 +1,24 @@
|
|||||||
|
from eth2spec.test.context import (
|
||||||
|
MAINNET,
|
||||||
|
spec_test,
|
||||||
|
single_phase,
|
||||||
|
with_eip7594_and_later,
|
||||||
|
with_phases,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@with_eip7594_and_later
|
||||||
|
@spec_test
|
||||||
|
@single_phase
|
||||||
|
@with_phases([MAINNET])
|
||||||
|
def test_sampling_config(spec):
|
||||||
|
probability_of_unavailable = 2 ** (-int(spec.SAMPLES_PER_SLOT))
|
||||||
|
# TODO: What is the security requirement?
|
||||||
|
security_requirement = 0.01
|
||||||
|
assert probability_of_unavailable <= security_requirement
|
||||||
|
|
||||||
|
column_size_in_bytes = spec.FIELD_ELEMENTS_PER_CELL * spec.BYTES_PER_FIELD_ELEMENT * spec.MAX_BLOBS_PER_BLOCK
|
||||||
|
bytes_per_slot = column_size_in_bytes * spec.SAMPLES_PER_SLOT
|
||||||
|
# TODO: What is the bandwidth requirement?
|
||||||
|
bandwidth_requirement = 10000 # bytes/s
|
||||||
|
assert bytes_per_slot // spec.config.SECONDS_PER_SLOT < bandwidth_requirement
|
||||||
0
tests/core/pyspec/eth2spec/test/helpers/das.py
Normal file
0
tests/core/pyspec/eth2spec/test/helpers/das.py
Normal file
Loading…
x
Reference in New Issue
Block a user