From 65be5b055631c1ce566d1293c3d7e354f3525367 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 20 Jan 2024 00:24:53 +0800 Subject: [PATCH 1/3] Make `CUSTODY_REQUIREMENT` unit be subnets; move some depended helpers to `das-core.md` --- configs/mainnet.yaml | 3 ++ configs/minimal.yaml | 4 +- specs/_features/eip7594/das-core.md | 52 ++++++++++++++----- specs/_features/eip7594/p2p-interface.md | 19 ------- .../unittests/test_config_invariants.py | 6 +-- .../test/eip7594/unittests/test_custody.py | 41 +++++++++------ .../test/eip7594/unittests/test_networking.py | 4 +- 7 files changed, 77 insertions(+), 52 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index fa664e122..3b1c14e4b 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -157,3 +157,6 @@ BLOB_SIDECAR_SUBNET_COUNT: 6 WHISK_EPOCHS_PER_SHUFFLING_PHASE: 256 # `Epoch(2)` WHISK_PROPOSER_SELECTION_GAP: 2 + +# EIP7594 +DATA_COLUMN_SIDECAR_SUBNET_COUNT: 32 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 9ac4249a5..d6d4a0942 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -62,7 +62,6 @@ WHISK_FORK_EPOCH: 18446744073709551615 EIP7594_FORK_VERSION: 0x06000001 EIP7594_FORK_EPOCH: 18446744073709551615 - # Time parameters # --------------------------------------------------------------- # [customized] Faster for testing purposes @@ -156,3 +155,6 @@ BLOB_SIDECAR_SUBNET_COUNT: 6 # Whisk WHISK_EPOCHS_PER_SHUFFLING_PHASE: 4 WHISK_PROPOSER_SELECTION_GAP: 1 + +# EIP7594 +DATA_COLUMN_SIDECAR_SUBNET_COUNT: 32 diff --git a/specs/_features/eip7594/das-core.md b/specs/_features/eip7594/das-core.md index 109ffb3a0..a7aa745ee 100644 --- a/specs/_features/eip7594/das-core.md +++ b/specs/_features/eip7594/das-core.md @@ -13,7 +13,7 @@ - [Data size](#data-size) - [Custody setting](#custody-setting) - [Helper functions](#helper-functions) - - [`get_custody_lines`](#get_custody_lines) + - [`get_custody_columns`](#get_custody_columns) - [`compute_extended_data`](#compute_extended_data) - [`compute_extended_matrix`](#compute_extended_matrix) - [`get_data_column_sidecars`](#get_data_column_sidecars) @@ -54,23 +54,51 @@ We define the following Python custom types for type hinting and readability: | - | - | - | | `NUMBER_OF_COLUMNS` | `uint64((FIELD_ELEMENTS_PER_BLOB * 2) // FIELD_ELEMENTS_PER_CELL)` (= 128) | Number of columns in the extended data matrix. | +### Networking + +| Name | Value | Description | +|------------------------------------|------------------|---------------------------------------------------------------------| +| `DATA_COLUMN_SIDECAR_SUBNET_COUNT` | `32` | The number of data column sidecar subnets used in the gossipsub protocol. | + ### Custody setting | Name | Value | Description | | - | - | - | | `SAMPLES_PER_SLOT` | `8` | Number of `DataColumn` random samples a node queries per slot | -| `CUSTODY_REQUIREMENT` | `2` | Minimum number of columns an honest node custodies and serves samples from | +| `CUSTODY_REQUIREMENT` | `1` | Minimum number of subnets an honest node custodies and serves samples from | | `TARGET_NUMBER_OF_PEERS` | `70` | Suggested minimum peer count | +### Containers + +#### `DataColumnSidecar` + +```python +class DataColumnSidecar(Container): + index: ColumnIndex # Index of column in extended matrix + column: DataColumn + kzg_commitments: List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK] + kzg_proofs: List[KZGProof, MAX_BLOB_COMMITMENTS_PER_BLOCK] + signed_block_header: SignedBeaconBlockHeader + kzg_commitments_inclusion_proof: Vector[Bytes32, KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH] +``` + ### Helper functions -#### `get_custody_lines` +#### `get_custody_columns` ```python -def get_custody_lines(node_id: NodeID, custody_size: uint64) -> Sequence[ColumnIndex]: - assert custody_size <= NUMBER_OF_COLUMNS - column_index = node_id % NUMBER_OF_COLUMNS - return [ColumnIndex((column_index + i) % NUMBER_OF_COLUMNS) for i in range(custody_size)] +def get_custody_columns(node_id: NodeID, custody_subnet_count: uint64) -> Sequence[ColumnIndex]: + assert custody_subnet_count <= DATA_COLUMN_SIDECAR_SUBNET_COUNT + subnet_ids = [ + bytes_to_uint64(hash(uint_to_bytes(uint64(node_id + i)))[0:8]) % DATA_COLUMN_SIDECAR_SUBNET_COUNT + for i in range(custody_subnet_count) + ] + columns_per_subnet = NUMBER_OF_COLUMNS // DATA_COLUMN_SIDECAR_SUBNET_COUNT + return [ + ColumnIndex(subnet_id + (i * columns_per_subnet)) + for i in range(columns_per_subnet) + for subnet_id in subnet_ids + ] ``` #### `compute_extended_data` @@ -126,15 +154,15 @@ def get_data_column_sidecars(signed_block: SignedBeaconBlock, ### Custody requirement -Each node downloads and custodies a minimum of `CUSTODY_REQUIREMENT` columns per slot. The particular columns that the node is required to custody are selected pseudo-randomly (more on this below). +Each node downloads and custodies a minimum of `CUSTODY_REQUIREMENT` subnets per slot. The particular columns that the node is required to custody are selected pseudo-randomly (more on this below). -A node *may* choose to custody and serve more than the minimum honesty requirement. Such a node explicitly advertises a number greater than `CUSTODY_REQUIREMENT` via the peer discovery mechanism -- for example, in their ENR (e.g. `custody_lines: 8` if the node custodies `8` columns each slot) -- up to a `NUMBER_OF_COLUMNS` (i.e. a super-full node). +A node *may* choose to custody and serve more than the minimum honesty requirement. Such a node explicitly advertises a number greater than `CUSTODY_REQUIREMENT` via the peer discovery mechanism -- for example, in their ENR (e.g. `custody_lines: 4` if the node custodies `4` subnets each slot) -- up to a `DATA_COLUMN_SIDECAR_SUBNET_COUNT` (i.e. a super-full node). A node stores the custodied columns for the duration of the pruning period and responds to peer requests for samples on those columns. ### Public, deterministic selection -The particular columns that a node custodies are selected pseudo-randomly as a function (`get_custody_lines`) of the node-id and custody size -- importantly this function can be run by any party as the inputs are all public. +The particular columns that a node custodies are selected pseudo-randomly as a function (`get_custody_columns`) of the node-id and custody size -- importantly this function can be run by any party as the inputs are all public. *Note*: increasing the `custody_size` parameter for a given `node_id` extends the returned list (rather than being an entirely new shuffle) such that if `custody_size` is unknown, the default `CUSTODY_REQUIREMENT` will be correct for a subset of the node's custody. @@ -178,7 +206,7 @@ Once the node obtain the column, the node should send the missing columns to the ## Peer sampling -At each slot, a node makes (locally randomly determined) `SAMPLES_PER_SLOT` queries for samples from their peers via `DataColumnSidecarByRoot` request. A node utilizes `get_custody_lines` helper to determine which peer(s) to request from. If a node has enough good/honest peers across all rows and columns, this has a high chance of success. +At each slot, a node makes (locally randomly determined) `SAMPLES_PER_SLOT` queries for samples from their peers via `DataColumnSidecarByRoot` request. A node utilizes `get_custody_columns` helper to determine which peer(s) to request from. If a node has enough good/honest peers across all rows and columns, this has a high chance of success. ## Peer scoring @@ -220,4 +248,4 @@ However, for simplicity, we don't assign row custody assignments to nodes in the ### Subnet stability -To start with a simple, stable backbone, for now, we don't shuffle the subnet assignments via the deterministic custody selection helper `get_custody_lines`. However, staggered rotation likely needs to happen on the order of the pruning period to ensure subnets can be utilized for recovery. For example, introducing an `epoch` argument allows the function to maintain stability over many epochs. +To start with a simple, stable backbone, for now, we don't shuffle the subnet assignments via the deterministic custody selection helper `get_custody_columns`. However, staggered rotation likely needs to happen on the order of the pruning period to ensure subnets can be utilized for recovery. For example, introducing an `epoch` argument allows the function to maintain stability over many epochs. diff --git a/specs/_features/eip7594/p2p-interface.md b/specs/_features/eip7594/p2p-interface.md index ad9a7eb54..b50e4d5a0 100644 --- a/specs/_features/eip7594/p2p-interface.md +++ b/specs/_features/eip7594/p2p-interface.md @@ -37,26 +37,8 @@ |------------------------------------------|-----------------------------------|---------------------------------------------------------------------| | `KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH` | `uint64(floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')))` (= 4) | Merkle proof index for `blob_kzg_commitments` | -### Configuration - -| Name | Value | Description | -|------------------------------------------|-----------------------------------|---------------------------------------------------------------------| -| `DATA_COLUMN_SIDECAR_SUBNET_COUNT` | `32` | The number of data column sidecar subnets used in the gossipsub protocol. | - ### Containers -#### `DataColumnSidecar` - -```python -class DataColumnSidecar(Container): - index: ColumnIndex # Index of column in extended matrix - column: DataColumn - kzg_commitments: List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK] - kzg_proofs: List[KZGProof, MAX_BLOB_COMMITMENTS_PER_BLOCK] - signed_block_header: SignedBeaconBlockHeader - kzg_commitments_inclusion_proof: Vector[Bytes32, KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH] -``` - #### `DataColumnIdentifier` ```python @@ -111,7 +93,6 @@ def compute_subnet_for_data_column_sidecar(column_index: ColumnIndex) -> SubnetI return SubnetID(column_index % DATA_COLUMN_SIDECAR_SUBNET_COUNT) ``` - ### The gossip domain: gossipsub Some gossip meshes are upgraded in the EIP-7594 fork to support upgraded types. diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py index 712e9892c..5f709a22a 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py @@ -12,6 +12,6 @@ def test_invariants(spec): assert spec.FIELD_ELEMENTS_PER_BLOB % spec.FIELD_ELEMENTS_PER_CELL == 0 assert spec.FIELD_ELEMENTS_PER_BLOB * 2 % spec.NUMBER_OF_COLUMNS == 0 assert spec.SAMPLES_PER_SLOT <= spec.NUMBER_OF_COLUMNS - assert spec.CUSTODY_REQUIREMENT <= spec.NUMBER_OF_COLUMNS - assert spec.DATA_COLUMN_SIDECAR_SUBNET_COUNT <= spec.NUMBER_OF_COLUMNS - assert spec.NUMBER_OF_COLUMNS % spec.DATA_COLUMN_SIDECAR_SUBNET_COUNT == 0 + assert spec.CUSTODY_REQUIREMENT <= spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT + assert spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT <= spec.NUMBER_OF_COLUMNS + assert spec.NUMBER_OF_COLUMNS % spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT == 0 diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_custody.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_custody.py index d629b3a06..9c9bcb2a1 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_custody.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_custody.py @@ -6,36 +6,47 @@ from eth2spec.test.context import ( ) +def run_get_custody_columns(spec, peer_count, custody_subnet_count): + assignments = [spec.get_custody_columns(node_id, custody_subnet_count) for node_id in range(peer_count)] + + subnet_per_column = spec.NUMBER_OF_COLUMNS // spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT + for assignment in assignments: + assert len(assignment) == custody_subnet_count * subnet_per_column + + @with_eip7594_and_later @spec_test @single_phase -def test_get_custody_lines_peers_within_number_of_columns(spec): +def test_get_custody_columns_peers_within_number_of_columns(spec): peer_count = 10 - custody_size = spec.CUSTODY_REQUIREMENT + custody_subnet_count = spec.CUSTODY_REQUIREMENT assert spec.NUMBER_OF_COLUMNS > peer_count - assignments = [spec.get_custody_lines(node_id, custody_size) for node_id in range(peer_count)] - - for assignment in assignments: - assert len(assignment) == custody_size + run_get_custody_columns(spec, peer_count, custody_subnet_count) @with_eip7594_and_later @spec_test @single_phase -def test_get_custody_lines_peers_more_than_number_of_columns(spec): +def test_get_custody_columns_peers_more_than_number_of_columns(spec): peer_count = 200 - custody_size = spec.CUSTODY_REQUIREMENT + custody_subnet_count = spec.CUSTODY_REQUIREMENT assert spec.NUMBER_OF_COLUMNS < peer_count - assignments = [spec.get_custody_lines(node_id, custody_size) for node_id in range(peer_count)] - - for assignment in assignments: - assert len(assignment) == custody_size + run_get_custody_columns(spec, peer_count, custody_subnet_count) @with_eip7594_and_later @spec_test @single_phase -def test_get_custody_lines_custody_size_more_than_number_of_columns(spec): +def test_get_custody_columns_maximum_subnets(spec): + peer_count = 10 + custody_subnet_count = spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT + run_get_custody_columns(spec, peer_count, custody_subnet_count) + + +@with_eip7594_and_later +@spec_test +@single_phase +def test_get_custody_columns_custody_size_more_than_number_of_columns(spec): node_id = 1 - custody_size = spec.NUMBER_OF_COLUMNS + 1 - expect_assertion_error(lambda: spec.get_custody_lines(node_id, custody_size)) + custody_subnet_count = spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT + 1 + expect_assertion_error(lambda: spec.get_custody_columns(node_id, custody_subnet_count)) diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py index 972539f5c..2ab52be6c 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_networking.py @@ -10,10 +10,10 @@ from eth2spec.test.context import ( @single_phase def test_compute_subnet_for_data_column_sidecar(spec): subnet_results = [] - for column_index in range(spec.DATA_COLUMN_SIDECAR_SUBNET_COUNT): + for column_index in range(spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT): subnet_results.append(spec.compute_subnet_for_data_column_sidecar(column_index)) # no duplicates assert len(subnet_results) == len(set(subnet_results)) # next one should be duplicate - next_subnet = spec.compute_subnet_for_data_column_sidecar(spec.DATA_COLUMN_SIDECAR_SUBNET_COUNT) + next_subnet = spec.compute_subnet_for_data_column_sidecar(spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT) assert next_subnet == subnet_results[0] From 4477cc695263e3b691abb41416cbda875ee7eab7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 20 Jan 2024 13:51:18 +0800 Subject: [PATCH 2/3] Fix column computation --- specs/_features/eip7594/das-core.md | 19 ++++++++++++++----- .../test/eip7594/unittests/test_custody.py | 7 +++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/specs/_features/eip7594/das-core.md b/specs/_features/eip7594/das-core.md index a7aa745ee..4ebb35248 100644 --- a/specs/_features/eip7594/das-core.md +++ b/specs/_features/eip7594/das-core.md @@ -89,13 +89,22 @@ class DataColumnSidecar(Container): ```python def get_custody_columns(node_id: NodeID, custody_subnet_count: uint64) -> Sequence[ColumnIndex]: assert custody_subnet_count <= DATA_COLUMN_SIDECAR_SUBNET_COUNT - subnet_ids = [ - bytes_to_uint64(hash(uint_to_bytes(uint64(node_id + i)))[0:8]) % DATA_COLUMN_SIDECAR_SUBNET_COUNT - for i in range(custody_subnet_count) - ] + + subnet_ids = [] + i = 0 + while len(subnet_ids) < custody_subnet_count: + subnet_id = ( + bytes_to_uint64(hash(uint_to_bytes(uint64(node_id + i)))[0:8]) + % DATA_COLUMN_SIDECAR_SUBNET_COUNT + ) + if subnet_id not in subnet_ids: + subnet_ids.append(subnet_id) + i += 1 + assert len(subnet_ids) == len(set(subnet_ids)) + columns_per_subnet = NUMBER_OF_COLUMNS // DATA_COLUMN_SIDECAR_SUBNET_COUNT return [ - ColumnIndex(subnet_id + (i * columns_per_subnet)) + ColumnIndex(DATA_COLUMN_SIDECAR_SUBNET_COUNT * i + subnet_id) for i in range(columns_per_subnet) for subnet_id in subnet_ids ] diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_custody.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_custody.py index 9c9bcb2a1..9c8168b33 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_custody.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_custody.py @@ -9,9 +9,12 @@ from eth2spec.test.context import ( def run_get_custody_columns(spec, peer_count, custody_subnet_count): assignments = [spec.get_custody_columns(node_id, custody_subnet_count) for node_id in range(peer_count)] - subnet_per_column = spec.NUMBER_OF_COLUMNS // spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT + columns_per_subnet = spec.NUMBER_OF_COLUMNS // spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT for assignment in assignments: - assert len(assignment) == custody_subnet_count * subnet_per_column + assert len(assignment) == custody_subnet_count * columns_per_subnet + print('assignment', assignment) + print('set(assignment)', set(assignment)) + assert len(assignment) == len(set(assignment)) @with_eip7594_and_later From edeef070d81db0c0ab1b1691ce829f0b87cad06f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 28 Jan 2024 22:47:49 +0800 Subject: [PATCH 3/3] toc --- specs/_features/eip7594/das-core.md | 3 +++ specs/_features/eip7594/p2p-interface.md | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/_features/eip7594/das-core.md b/specs/_features/eip7594/das-core.md index 4ebb35248..e3d4cb7a5 100644 --- a/specs/_features/eip7594/das-core.md +++ b/specs/_features/eip7594/das-core.md @@ -11,7 +11,10 @@ - [Custom types](#custom-types) - [Configuration](#configuration) - [Data size](#data-size) + - [Networking](#networking) - [Custody setting](#custody-setting) + - [Containers](#containers) + - [`DataColumnSidecar`](#datacolumnsidecar) - [Helper functions](#helper-functions) - [`get_custody_columns`](#get_custody_columns) - [`compute_extended_data`](#compute_extended_data) diff --git a/specs/_features/eip7594/p2p-interface.md b/specs/_features/eip7594/p2p-interface.md index b50e4d5a0..e53374e6d 100644 --- a/specs/_features/eip7594/p2p-interface.md +++ b/specs/_features/eip7594/p2p-interface.md @@ -10,9 +10,7 @@ - [Modifications in EIP-7594](#modifications-in-eip-7594) - [Preset](#preset) - - [Configuration](#configuration) - [Containers](#containers) - - [`DataColumnSidecar`](#datacolumnsidecar) - [`DataColumnIdentifier`](#datacolumnidentifier) - [Helpers](#helpers) - [`verify_data_column_sidecar_kzg_proof`](#verify_data_column_sidecar_kzg_proof)