status-im
/
nimbus-eth2
mirror of https://github.com/status-im/nimbus-eth2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

introduced custody groups, and renamed csc to cgc (#6789) 

* introduced custody groups, and renamed csc to cgc

* accomodate tests

* revert to naming columns

* applied review changes

* updated all tests file

* addressed review 2

* merged in typo fixes by airdop farmers/other spam PRs

* handle lint ci

* shift to iterators, avoid redundant copying
This commit is contained in:
Agnish Ghosh 2024-12-29 02:37:12 +05:30 committed by GitHub
parent 50ab4cf392
commit c0108c2f2a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 148 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
AllTests-mainnet.md
View File

@ -196,15 +196,15 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
OK: 2/2 Fail: 0/2 Skip: 0/2 OK: 2/2 Fail: 0/2 Skip: 0/2
## EF - EIP7594 - Networking [Preset: mainnet] ## EF - EIP7594 - Networking [Preset: mainnet]
```diff ```diff
+ Networking - Get Custody Columns - mainnet/fulu/networking/get_custody_columns/pyspec_test OK + Networking - Get Custody Groups - mainnet/fulu/networking/get_custody_columns/pyspec_tests OK
+ Networking - Get Custody Columns - mainnet/fulu/networking/get_custody_columns/pyspec_test OK + Networking - Get Custody Groups - mainnet/fulu/networking/get_custody_columns/pyspec_tests OK
+ Networking - Get Custody Columns - mainnet/fulu/networking/get_custody_columns/pyspec_test OK + Networking - Get Custody Groups - mainnet/fulu/networking/get_custody_columns/pyspec_tests OK
+ Networking - Get Custody Columns - mainnet/fulu/networking/get_custody_columns/pyspec_test OK + Networking - Get Custody Groups - mainnet/fulu/networking/get_custody_columns/pyspec_tests OK
+ Networking - Get Custody Columns - mainnet/fulu/networking/get_custody_columns/pyspec_test OK + Networking - Get Custody Groups - mainnet/fulu/networking/get_custody_columns/pyspec_tests OK
+ Networking - Get Custody Columns - mainnet/fulu/networking/get_custody_columns/pyspec_test OK + Networking - Get Custody Groups - mainnet/fulu/networking/get_custody_columns/pyspec_tests OK
+ Networking - Get Custody Columns - mainnet/fulu/networking/get_custody_columns/pyspec_test OK + Networking - Get Custody Groups - mainnet/fulu/networking/get_custody_columns/pyspec_tests OK
+ Networking - Get Custody Columns - mainnet/fulu/networking/get_custody_columns/pyspec_test OK + Networking - Get Custody Groups - mainnet/fulu/networking/get_custody_columns/pyspec_tests OK
+ Networking - Get Custody Columns - mainnet/fulu/networking/get_custody_columns/pyspec_test OK + Networking - Get Custody Groups - mainnet/fulu/networking/get_custody_columns/pyspec_tests OK
``` ```
OK: 9/9 Fail: 0/9 Skip: 0/9 OK: 9/9 Fail: 0/9 Skip: 0/9
## EF - KZG ## EF - KZG

2
CHANGELOG.md
View File

@ -2407,7 +2407,7 @@ It also brings further performance optimizations.
* A new `slashingdb` sub-command with `import` and `export` options. This allows for * A new `slashingdb` sub-command with `import` and `export` options. This allows for
safely migrating to Nimbus from another client (as per the [EIP-3076](https://eips.ethereum.org/EIPS/eip-3076) safely migrating to Nimbus from another client (as per the [EIP-3076](https://eips.ethereum.org/EIPS/eip-3076)
slashing protection interchange format). slashing protection interchange format).
Please see the the newly prepared [migration guides](https://nimbus.guide/migration.html) for the details. Please see the newly prepared [migration guides](https://nimbus.guide/migration.html) for the details.
* A new `ncli_db validatorPerf` command. This can be used to perform a textual * A new `ncli_db validatorPerf` command. This can be used to perform a textual
report for the attestation performance of a particular validator report for the attestation performance of a particular validator

2
beacon_chain/gossip_processing/optimistic_processor.nim
View File

@ -94,7 +94,7 @@ proc processSignedBeaconBlock*(
# Block validation is delegated to the sync committee and is done with delay. # Block validation is delegated to the sync committee and is done with delay.
# If we forward invalid spam blocks, we may be disconnected + IP banned, # If we forward invalid spam blocks, we may be disconnected + IP banned,
# so we avoid accepting any blocks. Since we don't meaningfully contribute # so we avoid accepting any blocks. Since we don't meaningfully contribute
# to the blocks gossip, we may also accummulate negative peer score over time. # to the blocks gossip, we may also accumulate negative peer score over time.
# However, we are actively contributing to other topics, so some of the # However, we are actively contributing to other topics, so some of the
# negative peer score may be offset through those different topics. # negative peer score may be offset through those different topics.
# The practical impact depends on the actually deployed scoring heuristics. # The practical impact depends on the actually deployed scoring heuristics.

14
beacon_chain/networking/eth2_discovery.nim
View File

@ -127,7 +127,7 @@ proc queryRandom*(
forkId: ENRForkID, forkId: ENRForkID,
wantedAttnets: AttnetBits, wantedAttnets: AttnetBits,
wantedSyncnets: SyncnetBits, wantedSyncnets: SyncnetBits,
wantedCscnets: CscBits, wantedCgcnets: CgcBits,
minScore: int): Future[seq[Node]] {.async: (raises: [CancelledError]).} = minScore: int): Future[seq[Node]] {.async: (raises: [CancelledError]).} =
## Perform a discovery query for a random target ## Perform a discovery query for a random target
## (forkId) and matching at least one of the attestation subnets. ## (forkId) and matching at least one of the attestation subnets.
@ -152,17 +152,17 @@ proc queryRandom*(
if not forkId.isCompatibleForkId(peerForkId): if not forkId.isCompatibleForkId(peerForkId):
continue continue
let cscCountBytes = n.record.get(enrCustodySubnetCountField, seq[byte]) let cgcCountBytes = n.record.get(enrCustodySubnetCountField, seq[byte])
if cscCountBytes.isOk(): if cgcCountBytes.isOk():
let cscCountNode = let cgcCountNode =
try: try:
SSZ.decode(cscCountBytes.get(), uint8) SSZ.decode(cgcCountBytes.get(), uint8)
except SerializationError as e: except SerializationError as e:
debug "Could not decode the csc ENR field of peer", debug "Could not decode the cgc ENR field of peer",
peer = n.record.toURI(), exception = e.name, msg = e.msg peer = n.record.toURI(), exception = e.name, msg = e.msg
continue continue
if wantedCscnets.countOnes().uint8 == cscCountNode: if wantedCgcnets.countOnes().uint8 == cgcCountNode:
score += 1 score += 1
let attnetsBytes = n.record.get(enrAttestationSubnetsField, seq[byte]) let attnetsBytes = n.record.get(enrAttestationSubnetsField, seq[byte])

36
beacon_chain/networking/eth2_network.nim
View File

@ -1506,7 +1506,7 @@ proc trimConnections(node: Eth2Node, count: int) =
if toKick <= 0: return if toKick <= 0: return
proc getLowSubnets(node: Eth2Node, epoch: Epoch): proc getLowSubnets(node: Eth2Node, epoch: Epoch):
(AttnetBits, SyncnetBits, CscBits) = (AttnetBits, SyncnetBits, CgcBits) =
# Returns the subnets required to have a healthy mesh # Returns the subnets required to have a healthy mesh
# The subnets are computed, to, in order: # The subnets are computed, to, in order:
# - Have 0 subnet with < `dLow` peers from topic subscription # - Have 0 subnet with < `dLow` peers from topic subscription
@ -1575,7 +1575,7 @@ proc getLowSubnets(node: Eth2Node, epoch: Epoch):
if epoch >= node.cfg.FULU_FORK_EPOCH: if epoch >= node.cfg.FULU_FORK_EPOCH:
findLowSubnets(getDataColumnSidecarTopic, uint64, (DATA_COLUMN_SIDECAR_SUBNET_COUNT).int) findLowSubnets(getDataColumnSidecarTopic, uint64, (DATA_COLUMN_SIDECAR_SUBNET_COUNT).int)
else: else:
default(CscBits) default(CgcBits)
) )
proc runDiscoveryLoop(node: Eth2Node) {.async: (raises: [CancelledError]).} = proc runDiscoveryLoop(node: Eth2Node) {.async: (raises: [CancelledError]).} =
@ -1584,20 +1584,20 @@ proc runDiscoveryLoop(node: Eth2Node) {.async: (raises: [CancelledError]).} =
while true: while true:
let let
currentEpoch = node.getBeaconTime().slotOrZero.epoch currentEpoch = node.getBeaconTime().slotOrZero.epoch
(wantedAttnets, wantedSyncnets, wantedCscnets) = node.getLowSubnets(currentEpoch) (wantedAttnets, wantedSyncnets, wantedCgcnets) = node.getLowSubnets(currentEpoch)
wantedAttnetsCount = wantedAttnets.countOnes() wantedAttnetsCount = wantedAttnets.countOnes()
wantedSyncnetsCount = wantedSyncnets.countOnes() wantedSyncnetsCount = wantedSyncnets.countOnes()
wantedCscnetsCount = wantedCscnets.countOnes() wantedCgcnetsCount = wantedCgcnets.countOnes()
outgoingPeers = node.peerPool.lenCurrent({PeerType.Outgoing}) outgoingPeers = node.peerPool.lenCurrent({PeerType.Outgoing})
targetOutgoingPeers = max(node.wantedPeers div 10, 3) targetOutgoingPeers = max(node.wantedPeers div 10, 3)
if wantedAttnetsCount > 0 or wantedSyncnetsCount > 0 or if wantedAttnetsCount > 0 or wantedSyncnetsCount > 0 or
wantedCscnetsCount > 0 or outgoingPeers < targetOutgoingPeers: wantedCgcnetsCount > 0 or outgoingPeers < targetOutgoingPeers:
let let
minScore = minScore =
if wantedAttnetsCount > 0 or wantedSyncnetsCount > 0 or if wantedAttnetsCount > 0 or wantedSyncnetsCount > 0 or
wantedCscnetsCount > 0: wantedCgcnetsCount > 0:
1 1
else: else:
0 0
@ -1605,7 +1605,7 @@ proc runDiscoveryLoop(node: Eth2Node) {.async: (raises: [CancelledError]).} =
node.discoveryForkId, node.discoveryForkId,
wantedAttnets, wantedAttnets,
wantedSyncnets, wantedSyncnets,
wantedCscnets, wantedCgcnets,
minScore) minScore)
let newPeers = block: let newPeers = block:
@ -2435,18 +2435,18 @@ func announcedENR*(node: Eth2Node): enr.Record =
doAssert node.discovery != nil, "The Eth2Node must be initialized" doAssert node.discovery != nil, "The Eth2Node must be initialized"
node.discovery.localNode.record node.discovery.localNode.record
proc lookupCscFromPeer*(peer: Peer): uint64 = proc lookupCgcFromPeer*(peer: Peer): uint64 =
# Fetches the custody column count from a remote peer. # Fetches the custody column count from a remote peer.
# If the peer advertises their custody column count via the `csc` ENR field, # If the peer advertises their custody column count via the `cgc` ENR field,
# that value is returned. Otherwise, the default value `CUSTODY_REQUIREMENT` # that value is returned. Otherwise, the default value `CUSTODY_REQUIREMENT`
# is assumed. # is assumed.
let metadata = peer.metadata let metadata = peer.metadata
if metadata.isOk: if metadata.isOk:
return metadata.get.custody_subnet_count return metadata.get.custody_group_count
# Try getting the custody count from ENR if metadata fetch fails. # Try getting the custody count from ENR if metadata fetch fails.
debug "Could not get csc from metadata, trying from ENR", debug "Could not get cgc from metadata, trying from ENR",
peer_id = peer.peerId peer_id = peer.peerId
let enrOpt = peer.enr let enrOpt = peer.enr
if not enrOpt.isNone: if not enrOpt.isNone:
@ -2454,8 +2454,8 @@ proc lookupCscFromPeer*(peer: Peer): uint64 =
let enrFieldOpt = enr.get(enrCustodySubnetCountField, seq[byte]) let enrFieldOpt = enr.get(enrCustodySubnetCountField, seq[byte])
if enrFieldOpt.isOk: if enrFieldOpt.isOk:
try: try:
let csc = SSZ.decode(enrFieldOpt.get, uint8) let cgc = SSZ.decode(enrFieldOpt.get, uint8)
return csc.uint64 return cgc.uint64
except SszError, SerializationError: except SszError, SerializationError:
discard # Ignore decoding errors and fallback to default discard # Ignore decoding errors and fallback to default
@ -2623,19 +2623,19 @@ proc updateStabilitySubnetMetadata*(node: Eth2Node, attnets: AttnetBits) =
else: else:
debug "Stability subnets changed; updated ENR attnets", attnets debug "Stability subnets changed; updated ENR attnets", attnets
proc loadCscnetMetadataAndEnr*(node: Eth2Node, cscnets: CscCount) = proc loadCgcnetMetadataAndEnr*(node: Eth2Node, cgcnets: CgcCount) =
node.metadata.custody_subnet_count = cscnets.uint64 node.metadata.custody_group_count = cgcnets.uint64
let res = let res =
node.discovery.updateRecord({ node.discovery.updateRecord({
enrCustodySubnetCountField: SSZ.encode(cscnets) enrCustodySubnetCountField: SSZ.encode(cgcnets)
}) })
if res.isErr: if res.isErr:
# This should not occur in this scenario as the private key would always # This should not occur in this scenario as the private key would always
# be the correct one and the ENR will not increase in size # be the correct one and the ENR will not increase in size
warn "Failed to update the ENR csc field", error = res.error warn "Failed to update the ENR cgc field", error = res.error
else: else:
debug "Updated ENR csc", cscnets debug "Updated ENR cgc", cgcnets
proc updateSyncnetsMetadata*(node: Eth2Node, syncnets: SyncnetBits) = proc updateSyncnetsMetadata*(node: Eth2Node, syncnets: SyncnetBits) =
# https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/altair/validator.md#sync-committee-subnet-stability # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/altair/validator.md#sync-committee-subnet-stability

4
beacon_chain/networking/network_metadata_downloads.nim
View File

@ -5,6 +5,8 @@
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms. # at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import import
std/uri, std/uri,
stew/io2, chronos, chronos/apps/http/httpclient, snappy, stew/io2, chronos, chronos/apps/http/httpclient, snappy,
@ -41,7 +43,7 @@ proc fetchGenesisBytes*(
result = await downloadFile(genesisStateUrlOverride.get(parseUri metadata.genesis.url)) result = await downloadFile(genesisStateUrlOverride.get(parseUri metadata.genesis.url))
# Under the built-in default URL, we serve a snappy-encoded BeaconState in order # Under the built-in default URL, we serve a snappy-encoded BeaconState in order
# to reduce the size of the downloaded file with roughly 50% (this precise ratio # to reduce the size of the downloaded file with roughly 50% (this precise ratio
# depends on the number of validator recors). The user is still free to provide # depends on the number of validator records). The user is still free to provide
# any URL which may serve an uncompressed state (e.g. a Beacon API endpoint) # any URL which may serve an uncompressed state (e.g. a Beacon API endpoint)
# #
# Since a SSZ-encoded BeaconState will start with a LittleEndian genesis time # Since a SSZ-encoded BeaconState will start with a LittleEndian genesis time

24
beacon_chain/nimbus_beacon_node.nim
View File

@ -417,14 +417,15 @@ proc initFullNode(
blobQuarantine = newClone(BlobQuarantine.init(onBlobSidecarAdded)) blobQuarantine = newClone(BlobQuarantine.init(onBlobSidecarAdded))
dataColumnQuarantine = newClone(DataColumnQuarantine.init()) dataColumnQuarantine = newClone(DataColumnQuarantine.init())
supernode = node.config.peerdasSupernode supernode = node.config.peerdasSupernode
localCustodySubnets = localCustodyGroups =
if supernode: if supernode:
DATA_COLUMN_SIDECAR_SUBNET_COUNT.uint64 NUMBER_OF_CUSTODY_GROUPS.uint64
else: else:
CUSTODY_REQUIREMENT.uint64 CUSTODY_REQUIREMENT.uint64
custody_columns_set = custody_columns_set =
node.network.nodeId.get_custody_columns_set(max(SAMPLES_PER_SLOT.uint64, node.network.nodeId.resolve_column_sets_from_custody_groups(
localCustodySubnets)) max(SAMPLES_PER_SLOT.uint64,
localCustodyGroups))
consensusManager = ConsensusManager.new( consensusManager = ConsensusManager.new(
dag, attestationPool, quarantine, node.elManager, dag, attestationPool, quarantine, node.elManager,
ActionTracker.init(node.network.nodeId, config.subscribeAllSubnets), ActionTracker.init(node.network.nodeId, config.subscribeAllSubnets),
@ -548,26 +549,27 @@ proc initFullNode(
# really variable. Whenever the BN is a supernode, column quarantine # really variable. Whenever the BN is a supernode, column quarantine
# essentially means all the NUMBER_OF_COLUMNS, as per mentioned in the # essentially means all the NUMBER_OF_COLUMNS, as per mentioned in the
# spec. However, in terms of fullnode, quarantine is really dependent # spec. However, in terms of fullnode, quarantine is really dependent
# on the randomly assigned columns, by `get_custody_columns`. # on the randomly assigned columns, by `resolve_columns_from_custody_groups`.
# Hence, in order to keep column quarantine accurate and error proof # Hence, in order to keep column quarantine accurate and error proof
# the custody columns are computed once as the BN boots. Then the values # the custody columns are computed once as the BN boots. Then the values
# are used globally around the codebase. # are used globally around the codebase.
# `get_custody_columns` is not a very expensive function, but there # `resolve_columns_from_custody_groups` is not a very expensive function,
# are multiple instances of computing custody columns, especially # but there are multiple instances of computing custody columns, especially
# during peer selection, sync with columns, and so on. That is why, # during peer selection, sync with columns, and so on. That is why,
# the rationale of populating it at boot and using it gloabally. # the rationale of populating it at boot and using it gloabally.
dataColumnQuarantine[].supernode = supernode dataColumnQuarantine[].supernode = supernode
dataColumnQuarantine[].custody_columns = dataColumnQuarantine[].custody_columns =
node.network.nodeId.get_custody_columns(max(SAMPLES_PER_SLOT.uint64, node.network.nodeId.resolve_columns_from_custody_groups(
localCustodySubnets)) max(SAMPLES_PER_SLOT.uint64,
localCustodyGroups))
if node.config.peerdasSupernode: if node.config.peerdasSupernode:
node.network.loadCscnetMetadataAndEnr(DATA_COLUMN_SIDECAR_SUBNET_COUNT.uint8) node.network.loadCgcnetMetadataAndEnr(NUMBER_OF_CUSTODY_GROUPS.uint8)
else: else:
node.network.loadCscnetMetadataAndEnr(CUSTODY_REQUIREMENT.uint8) node.network.loadCgcnetMetadataAndEnr(CUSTODY_REQUIREMENT.uint8)
if node.config.lightClientDataServe: if node.config.lightClientDataServe:
proc scheduleSendingLightClientUpdates(slot: Slot) = proc scheduleSendingLightClientUpdates(slot: Slot) =

2
beacon_chain/rpc/rest_event_api.nim
View File

@ -177,7 +177,7 @@ proc installEventApiHandlers*(router: var RestRouter, node: BeaconNode) =
discard await race(handlers) discard await race(handlers)
except ValueError: except ValueError:
raiseAssert "There should be more than one event handler at this point!" raiseAssert "There should be more than one event handler at this point!"
# One of the handlers finished, it means that connection has been droped, so # One of the handlers finished, it means that connection has been dropped, so
# we cancelling all other handlers. # we cancelling all other handlers.
let pending = let pending =
handlers.filterIt(not(it.finished())).mapIt(it.cancelAndWait()) handlers.filterIt(not(it.finished())).mapIt(it.cancelAndWait())

15
beacon_chain/spec/datatypes/fulu.nim
View File

@ -59,9 +59,13 @@ const
# https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#networking # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#networking
DATA_COLUMN_SIDECAR_SUBNET_COUNT* = 128 DATA_COLUMN_SIDECAR_SUBNET_COUNT* = 128
# https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#custody-setting # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/fulu/das-core.md#custody-setting
SAMPLES_PER_SLOT* = 8 SAMPLES_PER_SLOT* = 8
CUSTODY_REQUIREMENT* = 4 CUSTODY_REQUIREMENT* = 4
NUMBER_OF_CUSTODY_GROUPS* = 128
# Number of columns in the network per custody group
COLUMNS_PER_GROUP* = NUMBER_OF_COLUMNS div NUMBER_OF_CUSTODY_GROUPS
type type
# https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.6/specs/_features/eip7594/polynomial-commitments-sampling.md#custom-types # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.6/specs/_features/eip7594/polynomial-commitments-sampling.md#custom-types
@ -74,10 +78,11 @@ type
Cells* = KzgCells Cells* = KzgCells
CellsAndProofs* = KzgCellsAndKzgProofs CellsAndProofs* = KzgCellsAndKzgProofs
# https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#custom-types # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/fulu/das-core.md#custom-types
RowIndex* = uint64 RowIndex* = uint64
ColumnIndex* = uint64 ColumnIndex* = uint64
CellIndex* = uint64 CellIndex* = uint64
CustodyIndex* = uint64
type type
@ -108,16 +113,16 @@ type
row_index*: RowIndex row_index*: RowIndex
# Not in spec, defined in order to compute custody subnets # Not in spec, defined in order to compute custody subnets
CscBits* = BitArray[DATA_COLUMN_SIDECAR_SUBNET_COUNT] CgcBits* = BitArray[DATA_COLUMN_SIDECAR_SUBNET_COUNT]
CscCount* = uint8 CgcCount* = uint8
# https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/p2p-interface.md#metadata # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/p2p-interface.md#metadata
MetaData* = object MetaData* = object
seq_number*: uint64 seq_number*: uint64
attnets*: AttnetBits attnets*: AttnetBits
syncnets*: SyncnetBits syncnets*: SyncnetBits
custody_subnet_count*: uint64 custody_group_count*: uint64
# https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.8/specs/deneb/beacon-chain.md#executionpayload # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.8/specs/deneb/beacon-chain.md#executionpayload
ExecutionPayload* = object ExecutionPayload* = object

2
beacon_chain/spec/network.nim
View File

@ -47,7 +47,7 @@ const
enrAttestationSubnetsField* = "attnets" enrAttestationSubnetsField* = "attnets"
enrSyncSubnetsField* = "syncnets" enrSyncSubnetsField* = "syncnets"
enrCustodySubnetCountField* = "csc" enrCustodySubnetCountField* = "cgc"
enrForkIdField* = "eth2" enrForkIdField* = "eth2"
template eth2Prefix(forkDigest: ForkDigest): string = template eth2Prefix(forkDigest: ForkDigest): string =

103
beacon_chain/spec/peerdas_helpers.nim
View File

@ -9,7 +9,7 @@
# Uncategorized helper functions from the spec # Uncategorized helper functions from the spec
import import
std/algorithm, std/[algorithm, sequtils],
results, results,
eth/p2p/discoveryv5/[node], eth/p2p/discoveryv5/[node],
kzg4844/[kzg], kzg4844/[kzg],
@ -24,6 +24,7 @@ type
CellBytes = array[fulu.CELLS_PER_EXT_BLOB, Cell] CellBytes = array[fulu.CELLS_PER_EXT_BLOB, Cell]
ProofBytes = array[fulu.CELLS_PER_EXT_BLOB, KzgProof] ProofBytes = array[fulu.CELLS_PER_EXT_BLOB, KzgProof]
# Shall be deprecated once alpha 11 tests are released
func sortedColumnIndices(columnsPerSubnet: ColumnIndex, func sortedColumnIndices(columnsPerSubnet: ColumnIndex,
subnetIds: HashSet[uint64]): subnetIds: HashSet[uint64]):
seq[ColumnIndex] = seq[ColumnIndex] =
@ -35,18 +36,7 @@ func sortedColumnIndices(columnsPerSubnet: ColumnIndex,
res.sort res.sort
res res
func sortedColumnIndexList(columnsPerSubnet: ColumnIndex, # Shall be deprecated once alpha 11 tests are released
subnetIds: HashSet[uint64]):
List[ColumnIndex, NUMBER_OF_COLUMNS] =
var
res: seq[ColumnIndex]
for i in 0'u64 ..< columnsPerSubnet:
for subnetId in subnetIds:
let index = DATA_COLUMN_SIDECAR_SUBNET_COUNT * i + subnetId
res.add(ColumnIndex(index))
res.sort()
List[ColumnIndex, NUMBER_OF_COLUMNS].init(res)
func get_custody_column_subnets*(node_id: NodeId, func get_custody_column_subnets*(node_id: NodeId,
custody_subnet_count: uint64): custody_subnet_count: uint64):
HashSet[uint64] = HashSet[uint64] =
@ -81,6 +71,7 @@ func get_custody_column_subnets*(node_id: NodeId,
subnet_ids subnet_ids
# https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#get_custody_columns # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#get_custody_columns
# Shall be deprecated once alpha 11 tests are released
func get_custody_columns*(node_id: NodeId, func get_custody_columns*(node_id: NodeId,
custody_subnet_count: uint64): custody_subnet_count: uint64):
seq[ColumnIndex] = seq[ColumnIndex] =
@ -93,34 +84,70 @@ func get_custody_columns*(node_id: NodeId,
sortedColumnIndices(ColumnIndex(columns_per_subnet), subnet_ids) sortedColumnIndices(ColumnIndex(columns_per_subnet), subnet_ids)
func get_custody_columns_set*(node_id: NodeId, iterator compute_columns_for_custody_group(custody_group: CustodyIndex):
custody_subnet_count: uint64): ColumnIndex =
HashSet[ColumnIndex] = for i in 0'u64 ..< COLUMNS_PER_GROUP:
# This method returns a HashSet of column indices, yield ColumnIndex(NUMBER_OF_CUSTODY_GROUPS * i + custody_group)
# the method is specifically relevant while peer filtering
func handle_custody_groups(node_id: NodeId,
custody_group_count: CustodyIndex):
HashSet[CustodyIndex] =
# Decouples the custody group computation from
# `get_custody_groups`, in order to later use this custody
# group list across various types of output types
var
custody_groups: HashSet[CustodyIndex]
current_id = node_id
while custody_groups.lenu64 < custody_group_count:
var hashed_bytes: array[8, byte]
let
current_id_bytes = current_id.toBytesLE()
hashed_current_id = eth2digest(current_id_bytes)
hashed_bytes[0..7] = hashed_current_id.data.toOpenArray(0,7)
let custody_group = bytes_to_uint64(hashed_bytes) mod
NUMBER_OF_CUSTODY_GROUPS
custody_groups.incl custody_group
inc current_id
custody_groups
# https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/fulu/das-core.md#get_custody_groups
func get_custody_groups*(node_id: NodeId,
custody_group_count: CustodyIndex):
seq[CustodyIndex] =
let custody_groups =
node_id.handle_custody_groups(custody_group_count)
var groups = custody_groups.toSeq()
groups.sort()
groups
func resolve_columns_from_custody_groups*(node_id: NodeId,
custody_group_count: CustodyIndex):
seq[ColumnIndex] =
let let
subnet_ids = custody_groups = node_id.get_custody_groups(custody_group_count)
get_custody_column_subnets(node_id, custody_subnet_count)
const
columns_per_subnet =
NUMBER_OF_COLUMNS div DATA_COLUMN_SIDECAR_SUBNET_COUNT
sortedColumnIndices(ColumnIndex(columns_per_subnet), subnet_ids).toHashSet() var flattened =
newSeqOfCap[ColumnIndex](COLUMNS_PER_GROUP * custody_groups.len)
for group in custody_groups:
for index in compute_columns_for_custody_group(group):
flattened.add index
flattened
func get_custody_column_list*(node_id: NodeId, func resolve_column_sets_from_custody_groups*(node_id: NodeId,
custody_subnet_count: uint64): custody_group_count: CustodyIndex):
List[ColumnIndex, NUMBER_OF_COLUMNS] = HashSet[ColumnIndex] =
# Not in spec in the exact format, but it is useful in sorting custody columns node_id.resolve_columns_from_custody_groups(custody_group_count).toHashSet()
# before sending, data_column_sidecars_by_range requests
let
subnet_ids =
get_custody_column_subnets(node_id, custody_subnet_count)
const
columns_per_subnet =
NUMBER_OF_COLUMNS div DATA_COLUMN_SIDECAR_SUBNET_COUNT
sortedColumnIndexList(ColumnIndex(columns_per_subnet), subnet_ids)
# https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#compute_matrix # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#compute_matrix
proc compute_matrix*(blobs: seq[KzgBlob]): Result[seq[MatrixEntry], cstring] = proc compute_matrix*(blobs: seq[KzgBlob]): Result[seq[MatrixEntry], cstring] =
@ -235,7 +262,7 @@ proc get_data_column_sidecars*(signed_beacon_block: electra.TrustedSignedBeaconB
# blobs from blob bundles # blobs from blob bundles
proc get_data_column_sidecars*(signed_beacon_block: electra.SignedBeaconBlock, proc get_data_column_sidecars*(signed_beacon_block: electra.SignedBeaconBlock,
blobs: seq[KzgBlob]): blobs: seq[KzgBlob]):
Result[seq[DataColumnSidecar], string] = Result[seq[DataColumnSidecar], cstring] =
## Given a signed beacon block and the blobs corresponding to the block, ## Given a signed beacon block and the blobs corresponding to the block,
## this function assembles the sidecars which can be distributed to ## this function assembles the sidecars which can be distributed to
## the peers post data column reconstruction at every slot start. ## the peers post data column reconstruction at every slot start.

19
beacon_chain/sync/request_manager.nim
View File

@ -306,29 +306,30 @@ proc checkPeerCustody*(rman: RequestManager,
# to filter other supernodes, rather than filter # to filter other supernodes, rather than filter
# too many full nodes that have a subset of the custody # too many full nodes that have a subset of the custody
# columns # columns
if peer.lookupCscFromPeer() == if peer.lookupCgcFromPeer() ==
DATA_COLUMN_SIDECAR_SUBNET_COUNT.uint64: NUMBER_OF_CUSTODY_GROUPS.uint64:
return true return true
else: else:
if peer.lookupCscFromPeer() == if peer.lookupCgcFromPeer() ==
DATA_COLUMN_SIDECAR_SUBNET_COUNT.uint64: NUMBER_OF_CUSTODY_GROUPS.uint64:
return true return true
elif peer.lookupCscFromPeer() == elif peer.lookupCgcFromPeer() ==
CUSTODY_REQUIREMENT.uint64: CUSTODY_REQUIREMENT.uint64:
# Fetch the remote custody count # Fetch the remote custody count
let remoteCustodySubnetCount = let remoteCustodyGroupCount =
peer.lookupCscFromPeer() peer.lookupCgcFromPeer()
# Extract remote peer's nodeID from peerID # Extract remote peer's nodeID from peerID
# Fetch custody columns from remote peer # Fetch custody columns from remote peer
let let
remoteNodeId = fetchNodeIdFromPeerId(peer) remoteNodeId = fetchNodeIdFromPeerId(peer)
remoteCustodyColumns = remoteCustodyColumns =
remoteNodeId.get_custody_columns_set(max(SAMPLES_PER_SLOT.uint64, remoteNodeId.resolve_column_sets_from_custody_groups(
remoteCustodySubnetCount)) max(SAMPLES_PER_SLOT.uint64,
remoteCustodyGroupCount))
for local_column in rman.custody_columns_set: for local_column in rman.custody_columns_set:
if local_column notin remoteCustodyColumns: if local_column notin remoteCustodyColumns:

2
docs/the_nimbus_book/src/keep-an-eye.md
View File

@ -53,4 +53,4 @@ The string of letters -- what we call the `sync worker map` (in the above case r
``` ```
!!! tip !!! tip
You can also use you calls outlined in the [REST API page](./rest-api.md) to retrieve similar information. You can also use the calls outlined in the [REST API page](./rest-api.md) to retrieve similar information.

5
tests/consensus_spec/test_fixture_networking.nim
View File

@ -22,7 +22,7 @@ from std/sequtils import mapIt
proc runGetCustodyColumns(suiteName, path: string) = proc runGetCustodyColumns(suiteName, path: string) =
let relativePathComponent = path.relativeTestPathComponent() let relativePathComponent = path.relativeTestPathComponent()
test "Networking - Get Custody Columns - " & relativePathComponent: test "Networking - Get Custody Groups - " & relativePathComponent:
type TestMetaYaml = object type TestMetaYaml = object
node_id: string node_id: string
custody_group_count: uint64 custody_group_count: uint64
@ -38,13 +38,14 @@ proc runGetCustodyColumns(suiteName, path: string) =
custody_group_count = meta.custody_group_count custody_group_count = meta.custody_group_count
reslt = (meta.result).mapIt(it) reslt = (meta.result).mapIt(it)
let columns = get_custody_columns(node_id, custody_group_count) let columns = get_custody_groups(node_id, custody_group_count)
for i in 0..<columns.lenu64: for i in 0..<columns.lenu64:
check columns[i] == reslt[i] check columns[i] == reslt[i]
suite "EF - EIP7594 - Networking" & preset(): suite "EF - EIP7594 - Networking" & preset():
const presetPath = SszTestsDir/const_preset const presetPath = SszTestsDir/const_preset
# foldering to be resolved in alpha 11 release of consensus spec tests
let basePath = let basePath =
presetPath/"fulu"/"networking"/"get_custody_columns"/"pyspec_tests" presetPath/"fulu"/"networking"/"get_custody_columns"/"pyspec_tests"
for kind, path in walkDir(basePath, relative = true, checkDir = true): for kind, path in walkDir(basePath, relative = true, checkDir = true):

10
tests/test_discovery.nim
View File

@ -36,7 +36,7 @@ proc generateNode(rng: ref HmacDrbgContext, port: Port,
# TODO: Add tests with a syncnets preference # TODO: Add tests with a syncnets preference
const noSyncnetsPreference = SyncnetBits() const noSyncnetsPreference = SyncnetBits()
const noCscnetsPreference = CscBits() const noCgcnetsPreference = CgcBits()
procSuite "Eth2 specific discovery tests": procSuite "Eth2 specific discovery tests":
let let
@ -69,7 +69,7 @@ procSuite "Eth2 specific discovery tests":
let discovered = await node1.queryRandom( let discovered = await node1.queryRandom(
enrForkId, attnetsSelected, noSyncnetsPreference, enrForkId, attnetsSelected, noSyncnetsPreference,
noCscnetsPreference, 1) noCgcnetsPreference, 1)
check discovered.len == 1 check discovered.len == 1
await node1.closeWait() await node1.closeWait()
@ -108,7 +108,7 @@ procSuite "Eth2 specific discovery tests":
let discovered = await node1.queryRandom( let discovered = await node1.queryRandom(
enrForkId, attnetsSelected, noSyncnetsPreference, enrForkId, attnetsSelected, noSyncnetsPreference,
noCscnetsPreference, 1) noCgcnetsPreference, 1)
check discovered.len == 1 check discovered.len == 1
await node1.closeWait() await node1.closeWait()
@ -137,7 +137,7 @@ procSuite "Eth2 specific discovery tests":
block: block:
let discovered = await node1.queryRandom( let discovered = await node1.queryRandom(
enrForkId, attnetsSelected, noSyncnetsPreference, enrForkId, attnetsSelected, noSyncnetsPreference,
noCscnetsPreference, 1) noCgcnetsPreference, 1)
check discovered.len == 0 check discovered.len == 0
block: block:
@ -153,7 +153,7 @@ procSuite "Eth2 specific discovery tests":
let discovered = await node1.queryRandom( let discovered = await node1.queryRandom(
enrForkId, attnetsSelected, noSyncnetsPreference, enrForkId, attnetsSelected, noSyncnetsPreference,
noCscnetsPreference, 1) noCgcnetsPreference, 1)
check discovered.len == 1 check discovered.len == 1
await node1.closeWait() await node1.closeWait()