From 444d1dd0935f531d51dcff5831a7b22a1ae4f625 Mon Sep 17 00:00:00 2001 From: Agnish Ghosh <80243668+agnxsh@users.noreply.github.com> Date: Wed, 4 Sep 2024 19:35:18 +0530 Subject: [PATCH] add: `get_custody_columns` for das_core (#6532) * add: get_custody_columnns in das core specs with tests * apply review changes * review changes 2 * review 3 --- AllTests-mainnet.md | 15 ++- beacon_chain/spec/datatypes/eip7594.nim | 110 ++++++++++++++++++ beacon_chain/spec/eip7594_helpers.nim | 105 +++++++++++++++++ tests/consensus_spec/all_tests.nim | 1 + .../test_fixture_networking.nim | 51 ++++++++ 5 files changed, 281 insertions(+), 1 deletion(-) create mode 100644 beacon_chain/spec/datatypes/eip7594.nim create mode 100644 beacon_chain/spec/eip7594_helpers.nim create mode 100644 tests/consensus_spec/test_fixture_networking.nim diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md index 03cacba7e..a327e1580 100644 --- a/AllTests-mainnet.md +++ b/AllTests-mainnet.md @@ -175,6 +175,19 @@ OK: 1/1 Fail: 0/1 Skip: 0/1 + Tail block only in common OK ``` OK: 2/2 Fail: 0/2 Skip: 0/2 +## EF - EIP7594 - Networking [Preset: mainnet] +```diff ++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK ++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK ++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK ++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK ++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK ++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK ++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK ++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK ++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK +``` +OK: 9/9 Fail: 0/9 Skip: 0/9 ## EF - KZG ```diff + KZG - Blob to KZG commitment - blob_to_kzg_commitment_case_invalid_blob_59d64ff6b4648fad OK @@ -1100,4 +1113,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 OK: 9/9 Fail: 0/9 Skip: 0/9 ---TOTAL--- -OK: 749/754 Fail: 0/754 Skip: 5/754 +OK: 758/763 Fail: 0/763 Skip: 5/763 diff --git a/beacon_chain/spec/datatypes/eip7594.nim b/beacon_chain/spec/datatypes/eip7594.nim new file mode 100644 index 000000000..2ec8d4672 --- /dev/null +++ b/beacon_chain/spec/datatypes/eip7594.nim @@ -0,0 +1,110 @@ +# beacon_chain +# Copyright (c) 2022-2024 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import + std/[sequtils], + "."/[altair, base, deneb], + kzg4844/[kzg, kzg_abi] + +from std/strutils import join + +export base + +const + # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/polynomial-commitments-sampling.md#cells + FIELD_ELEMENTS_PER_EXT_BLOB* = 2 * kzg_abi.FIELD_ELEMENTS_PER_BLOB + # Number of field elements in a Reed-Solomon extended blob | + FIELD_ELEMENTS_PER_CELL* = 64 # Number of field elements in a cell | + BYTES_PER_CELL* = FIELD_ELEMENTS_PER_CELL * kzg_abi.BYTES_PER_FIELD_ELEMENT + # The number of bytes in a cell | + CELLS_PER_EXT_BLOB* = FIELD_ELEMENTS_PER_EXT_BLOB div FIELD_ELEMENTS_PER_CELL + # The number of cells in an extended blob | + + # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/p2p-interface.md#preset + KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH* = 4 + +type + # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/polynomial-commitments-sampling.md#custom-types + BLSFieldElement* = KzgBytes32 + G2Point* = array[96, byte] + PolynomialCoeff* = List[BLSFieldElement, FIELD_ELEMENTS_PER_EXT_BLOB] + Coset* = array[FIELD_ELEMENTS_PER_CELL, BLSFieldElement] + CosetEvals* = array[FIELD_ELEMENTS_PER_CELL, BLSFieldElement] + Cell* = KzgCell + Cells* = KzgCells + CellsAndProofs* = KzgCellsAndKzgProofs + + # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#custom-types + RowIndex* = uint64 + ColumnIndex* = uint64 + CellIndex* = uint64 + +const + # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#data-size + NUMBER_OF_COLUMNS* = 128 + + # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#networking + DATA_COLUMN_SIDECAR_SUBNET_COUNT* = 128 + + # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#custody-setting + SAMPLES_PER_SLOT* = 8 + CUSTODY_REQUIREMENT* = 4 + +type + DataColumn* = List[KzgCell, Limit(MAX_BLOB_COMMITMENTS_PER_BLOCK)] + + # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#datacolumnsidecar + DataColumnSidecar* = object + index*: ColumnIndex # Index of column in extended matrix + column*: DataColumn + kzg_commitments*: KzgCommitments + kzg_proofs*: KzgProofs + signed_block_header*: SignedBeaconBlockHeader + kzg_commitments_inclusion_proof*: + array[KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH, Eth2Digest] + + DataColumnSidecars* = seq[ref DataColumnSidecar] + + # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/p2p-interface.md#datacolumnidentifier + DataColumnIdentifier* = object + block_root*: Eth2Digest + index*: ColumnIndex + + # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#matrixentry + MatrixEntry* = object + cell*: Cell + kzg_proof*: KzgProof + column_index*: ColumnIndex + row_index*: RowIndex + + # Not in spec, defined in order to compute custody subnets + CscBits* = BitArray[DATA_COLUMN_SIDECAR_SUBNET_COUNT] + + CscCount* = uint8 + + # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/p2p-interface.md#metadata + MetaData* = object + seq_number*: uint64 + attnets*: AttnetBits + syncnets*: SyncnetBits + custody_subnet_count*: CscCount + +func shortLog*(v: DataColumnSidecar): auto = + ( + index: v.index, + kzg_commitments: v.kzg_commitments.len, + kzg_proofs: v.kzg_proofs.len, + block_header: shortLog(v.signed_block_header.message), + ) + +func shortLog*(v: seq[DataColumnSidecar]): auto = + "[" & v.mapIt(shortLog(it)).join(", ") & "]" + +func shortLog*(x: seq[DataColumnIdentifier]): string = + "[" & x.mapIt(shortLog(it.block_root) & "/" & $it.index).join(", ") & "]" diff --git a/beacon_chain/spec/eip7594_helpers.nim b/beacon_chain/spec/eip7594_helpers.nim new file mode 100644 index 000000000..e55c7e82f --- /dev/null +++ b/beacon_chain/spec/eip7594_helpers.nim @@ -0,0 +1,105 @@ +# beacon_chain +# Copyright (c) 2018-2024 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +# Uncategorized helper functions from the spec +import + std/[algorithm, hashes], + results, + eth/p2p/discoveryv5/[node], + ./[helpers, digest], + ./datatypes/[eip7594] + +proc sortedColumnIndices*(columnsPerSubnet: ColumnIndex, + subnetIds: HashSet[uint64]): + seq[ColumnIndex] = + var res: seq[ColumnIndex] = @[] + for i in 0'u64 ..< columnsPerSubnet: + for subnetId in subnetIds: + let index = DATA_COLUMN_SIDECAR_SUBNET_COUNT * i + subnetId + res.add(ColumnIndex(index)) + res.sort + res + +proc sortedColumnIndexList*(columnsPerSubnet: ColumnIndex, + subnetIds: HashSet[uint64]): + List[ColumnIndex, NUMBER_OF_COLUMNS] = + var + res: seq[ColumnIndex] + for i in 0'u64 ..< columnsPerSubnet: + for subnetId in subnetIds: + let index = DATA_COLUMN_SIDECAR_SUBNET_COUNT * i + subnetId + res.add(ColumnIndex(index)) + res.sort() + let list = List[ColumnIndex, NUMBER_OF_COLUMNS].init(res) + list + +proc get_custody_column_subnets*(node_id: NodeId, + custody_subnet_count: uint64): + Result[HashSet[uint64], cstring] = + + # Decouples the custody subnet computation part from + # `get_custody_columns`, in order to later use this subnet list + # in order to maintain subscription to specific column subnets. + + if not (custody_subnet_count <= DATA_COLUMN_SIDECAR_SUBNET_COUNT): + return err("Eip7594: Custody subnet count exceeds the DATA_COLUMN_SIDECAR_SUBNET_COUNT") + + var + subnet_ids: HashSet[uint64] + current_id = node_id + + while subnet_ids.lenu64 < custody_subnet_count: + var + hashed_bytes: array[8, byte] + + let + current_id_bytes = current_id.toBytesLE() + hashed_current_id = eth2digest(current_id_bytes) + + hashed_bytes[0..7] = hashed_current_id.data.toOpenArray(0,7) + let subnet_id = bytes_to_uint64(hashed_bytes) mod + DATA_COLUMN_SIDECAR_SUBNET_COUNT + + subnet_ids.incl(subnet_id) + + if current_id == UInt256.high.NodeId: + # Overflow prevention + current_id = NodeId(StUint[256].zero) + current_id += NodeId(StUint[256].one) + + ok(subnet_ids) + +# https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#get_custody_columns +proc get_custody_columns*(node_id: NodeId, + custody_subnet_count: uint64): + seq[ColumnIndex] = + let + subnet_ids = + get_custody_column_subnets(node_id, custody_subnet_count).get + const + columns_per_subnet = + NUMBER_OF_COLUMNS div DATA_COLUMN_SIDECAR_SUBNET_COUNT + + sortedColumnIndices(ColumnIndex(columns_per_subnet), subnet_ids) + + +proc get_custody_column_list*(node_id: NodeId, + custody_subnet_count: uint64): + List[ColumnIndex, NUMBER_OF_COLUMNS] = + + # Not in spec in the exact format, but it is useful in sorting custody columns + # before sending, data_column_sidecars_by_range requests + let + subnet_ids = + get_custody_column_subnets(node_id, custody_subnet_count).get + const + columns_per_subnet = + NUMBER_OF_COLUMNS div DATA_COLUMN_SIDECAR_SUBNET_COUNT + + sortedColumnIndexList(ColumnIndex(columns_per_subnet), subnet_ids) diff --git a/tests/consensus_spec/all_tests.nim b/tests/consensus_spec/all_tests.nim index a0f3af220..d17bf5f54 100644 --- a/tests/consensus_spec/all_tests.nim +++ b/tests/consensus_spec/all_tests.nim @@ -15,4 +15,5 @@ import ./test_fixture_kzg, + ./test_fixture_networking, ./test_fixture_ssz_generic_types diff --git a/tests/consensus_spec/test_fixture_networking.nim b/tests/consensus_spec/test_fixture_networking.nim new file mode 100644 index 000000000..9a8c11b77 --- /dev/null +++ b/tests/consensus_spec/test_fixture_networking.nim @@ -0,0 +1,51 @@ +# beacon_chain +# Copyright (c) 2024 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} +{.used.} + +import + std/[json, streams], + yaml, + kzg4844/[kzg, kzg_abi], + stint, + eth/p2p/discoveryv5/[node], + ../../beacon_chain/spec/eip7594_helpers, + ../testutil, + ./fixtures_utils, ./os_ops + +from std/sequtils import mapIt + +proc runGetCustodyColumns(suiteName, path: string) = + let relativePathComponent = path.relativeTestPathComponent() + test "Networking - Get Custody Columns - " & relativePathComponent: + type TestMetaYaml = object + node_id: string + custody_subnet_count: uint64 + result: seq[uint64] + let + meta = block: + var s = openFileStream(path/"meta.yaml") + defer: close(s) + var res: TestMetaYaml + yaml.load(s, res) + res + node_id = UInt256.fromDecimal(meta.node_id) + custody_subnet_count = meta.custody_subnet_count + reslt = (meta.result).mapIt(it) + + let columns = get_custody_columns(node_id, custody_subnet_count) + + for i in 0..