status-im
/
nimbus-eth2
mirror of https://github.com/status-im/nimbus-eth2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

in exit pool, bundle receive messages into beaconblocks (#1812) 

* in exit pool, filter out already-packaged messages; bundle remaining messages into beaconblocks

* filter messages at block construction time

* allow adding up to intended capacity of buffers, beyond per-block limits

* document rationale/design for filtering mechanism
This commit is contained in:
tersec 2020-10-07 16:57:21 +00:00 committed by GitHub
parent 2ea5385aa0
commit f08f44b9a2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 139 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
AllTests-mainnet.md
View File

@ -98,15 +98,19 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
+ Mocked start private key OK
```
OK: 3/3 Fail: 0/3 Skip: 0/3
## Keystore
## KeyStorage testing suite
```diff
+ Pbkdf2 decryption OK
+ Pbkdf2 encryption OK
+ Pbkdf2 errors OK
+ Scrypt decryption OK
+ Scrypt encryption OK
+ [PBKDF2] Keystore decryption OK
+ [PBKDF2] Keystore encryption OK
+ [PBKDF2] Network Keystore decryption OK
+ [PBKDF2] Network Keystore encryption OK
+ [SCRYPT] Keystore decryption OK
+ [SCRYPT] Keystore encryption OK
+ [SCRYPT] Network Keystore decryption OK
+ [SCRYPT] Network Keystore encryption OK
```
OK: 5/5 Fail: 0/5 Skip: 0/5
OK: 9/9 Fail: 0/9 Skip: 0/9
## Mocking utilities
```diff
+ merkle_minimal OK
@ -281,4 +285,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
OK: 1/1 Fail: 0/1 Skip: 0/1
---TOTAL---
OK: 151/159 Fail: 0/159 Skip: 8/159
OK: 155/163 Fail: 0/163 Skip: 8/163

6
beacon_chain/beacon_node_types.nim
View File

@ -84,13 +84,13 @@ type
voluntary_exits*: Deque[SignedVoluntaryExit] ## \
## Not a function of chain DAG branch; just used as a FIFO queue for blocks
prior_seen_attester_slashed_indices*: HashSet[ValidatorIndex] ##\
prior_seen_attester_slashed_indices*: HashSet[uint64] ##\
## Records attester-slashed indices seen.
prior_seen_proposer_slashed_indices*: HashSet[ValidatorIndex] ##\
prior_seen_proposer_slashed_indices*: HashSet[uint64] ##\
## Records proposer-slashed indices seen.
prior_seen_voluntary_exit_indices*: HashSet[ValidatorIndex] ##\
prior_seen_voluntary_exit_indices*: HashSet[uint64] ##\
## Records voluntary exit indices seen.
chainDag*: ChainDAGRef

128
beacon_chain/exit_pool.nim
View File

@ -9,7 +9,7 @@
import
# Standard libraries
std/[deques, options, sequtils, sets],
std/[deques, options, sequtils, sets, tables],
# Status libraries
chronicles, json_serialization/std/sets as jsonSets,
# Internal
@ -19,55 +19,125 @@ import
export beacon_node_types, sets
logScope: topics = "slashpool"
logScope: topics = "exitpool"
const
ATTESTER_SLASHINGS_BOUND = MAX_ATTESTER_SLASHINGS * 2
PROPOSER_SLASHINGS_BOUND = MAX_PROPOSER_SLASHINGS * 2
VOLUNTARY_EXITS_BOUND = MAX_VOLUNTARY_EXITS * 2
proc init*(
T: type ExitPool, chainDag: ChainDAGRef, quarantine: QuarantineRef): T =
## Initialize an ExitPool from the chainDag `headState`
T(
# Allow for filtering out some exit messages during block production
attester_slashings:
initDeque[AttesterSlashing](initialSize = MAX_ATTESTER_SLASHINGS.int),
initDeque[AttesterSlashing](initialSize = ATTESTER_SLASHINGS_BOUND.int),
proposer_slashings:
initDeque[ProposerSlashing](initialSize = MAX_PROPOSER_SLASHINGS.int),
initDeque[ProposerSlashing](initialSize = PROPOSER_SLASHINGS_BOUND.int),
voluntary_exits:
initDeque[SignedVoluntaryExit](initialSize = MAX_VOLUNTARY_EXITS.int),
initDeque[SignedVoluntaryExit](initialSize = VOLUNTARY_EXITS_BOUND.int),
chainDag: chainDag,
quarantine: quarantine
)
func addExitMessage*(subpool: var auto, exitMessage, bound: auto) =
# Prefer newer to older exit message
# Prefer newer to older exit messages
while subpool.lenu64 >= bound:
discard subpool.popFirst()
subpool.addLast(exitMessage)
doAssert subpool.lenu64 <= bound
func getExitMessagesForBlock[T](subpool: var Deque[T], bound: uint64): seq[T] =
for i in 0 ..< bound:
if subpool.len == 0:
break
result.add subpool.popFirst()
iterator getValidatorIndices(attester_slashing: AttesterSlashing): uint64 =
# TODO rely on sortedness and do this sans memory allocations, but it's only
# when producing a beacon block, which is rare bottlenecked elsewhere.
let
attestation_1_indices =
attester_slashing.attestation_1.attesting_indices.asSeq
attestation_2_indices =
attester_slashing.attestation_2.attesting_indices.asSeq
attester_slashed_indices =
toHashSet(attestation_1_indices) * toHashSet(attestation_2_indices)
for validator_index in attester_slashed_indices:
yield validator_index
iterator getValidatorIndices(proposer_slashing: ProposerSlashing): uint64 =
yield proposer_slashing.signed_header_1.message.proposer_index
iterator getValidatorIndices(voluntary_exit: SignedVoluntaryExit): uint64 =
yield voluntary_exit.message.validator_index
# TODO stew/sequtils2
template allIt(s, pred: untyped): bool =
# https://github.com/nim-lang/Nim/blob/version-1-2/lib/pure/collections/sequtils.nim#L640-L662
# without the items(...)
var result = true
for it {.inject.} in s:
if not pred:
result = false
break
result
func getExitMessagesForBlock[T](
subpool: var Deque[T], pool: var ExitPool, bound: uint64): seq[T] =
# Approach taken here is to simply collect messages, effectively, a circular
# buffer and only re-validate that they haven't already found themselves out
# of the network eventually via some exit message at block construction time
# at which point we use exit_epoch. It doesn't matter which of these message
# types has triggered that exit, as the validation on incoming messages will
# find it to either be IGNORE (if it's the same type of exit message) or, if
# it's a different type, REJECT. Neither is worth packaging into BeaconBlock
# messages we broadcast.
#
# Beyond that, no other criterion of the exit messages' validity changes from
# when they were created, so given that we validated them to start with, they
# otherwise remain as valid as when we received them. There's no need to thus
# re-validate them on their way out.
#
# This overall approach handles a scenario wherein we receive an exit message
# over gossip and put it in the pool; receive a block X, with that message in
# it, and select it as head; then orphan block X and build instead on X-1. If
# this occurs, only validating after the fact ensures that we still broadcast
# out those exit messages that were in orphaned block X by not having eagerly
# removed them, if we have the chance.
while true:
if subpool.len == 0 or result.lenu64 >= bound:
break
# Prefer recent messages
let exit_message = subpool.popLast()
if allIt(
getValidatorIndices(exit_message),
pool.chainDag.headState.data.data.validators[it].exit_epoch !=
FAR_FUTURE_EPOCH):
# A beacon block exit message already targeted all these validators
continue
result.add exit_message
subpool.clear()
doAssert result.lenu64 <= bound
func getAttesterSlashingsForBlock*(pool: var ExitPool):
seq[AttesterSlashing] =
## Retrieve attester slashings that may be added to a new block
getExitMessagesForBlock[AttesterSlashing](
pool.attester_slashings, MAX_ATTESTER_SLASHINGS)
pool.attester_slashings, pool, MAX_ATTESTER_SLASHINGS)
func getProposerSlashingsForBlock*(pool: var ExitPool):
seq[ProposerSlashing] =
## Retrieve proposer slashings that may be added to a new block
getExitMessagesForBlock[ProposerSlashing](
pool.proposer_slashings, MAX_PROPOSER_SLASHINGS)
pool.proposer_slashings, pool, MAX_PROPOSER_SLASHINGS)
func getVoluntaryExitsForBlock*(pool: var ExitPool):
seq[SignedVoluntaryExit] =
## Retrieve voluntary exits that may be added to a new block
getExitMessagesForBlock[SignedVoluntaryExit](
pool.voluntary_exits, MAX_VOLUNTARY_EXITS)
pool.voluntary_exits, pool, MAX_VOLUNTARY_EXITS)
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.3/specs/phase0/p2p-interface.md#attester_slashing
proc validateAttesterSlashing*(
@ -78,14 +148,14 @@ proc validateAttesterSlashing*(
# attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices),
# verify if any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))).
# TODO sequtils2 should be able to make this more reasonable, from asSeq on
# down
let attester_slashed_indices =
toHashSet(mapIt(
attester_slashing.attestation_1.attesting_indices.asSeq,
it.ValidatorIndex)) *
toHashSet(mapIt(
attester_slashing.attestation_2.attesting_indices.asSeq,
it.ValidatorIndex))
# down, and can sort and just find intersection that way
let
attestation_1_indices =
attester_slashing.attestation_1.attesting_indices.asSeq
attestation_2_indices =
attester_slashing.attestation_2.attesting_indices.asSeq
attester_slashed_indices =
toHashSet(attestation_1_indices) * toHashSet(attestation_2_indices)
if not disjoint(
attester_slashed_indices, pool.prior_seen_attester_slashed_indices):
@ -106,7 +176,7 @@ proc validateAttesterSlashing*(
pool.prior_seen_attester_slashed_indices.incl attester_slashed_indices
pool.attester_slashings.addExitMessage(
attester_slashing, MAX_ATTESTER_SLASHINGS)
attester_slashing, ATTESTER_SLASHINGS_BOUND)
ok(true)
@ -117,7 +187,7 @@ proc validateProposerSlashing*(
# [IGNORE] The proposer slashing is the first valid proposer slashing
# received for the proposer with index
# proposer_slashing.signed_header_1.message.proposer_index.
if proposer_slashing.signed_header_1.message.proposer_index.ValidatorIndex in
if proposer_slashing.signed_header_1.message.proposer_index in
pool.prior_seen_proposer_slashed_indices:
const err_str: cstring =
"validateProposerSlashing: proposer-slashed index already proposer-slashed"
@ -134,9 +204,9 @@ proc validateProposerSlashing*(
return err((EVRESULT_REJECT, proposer_slashing_validity.error))
pool.prior_seen_proposer_slashed_indices.incl(
proposer_slashing.signed_header_1.message.proposer_index.ValidatorIndex)
proposer_slashing.signed_header_1.message.proposer_index)
pool.proposer_slashings.addExitMessage(
proposer_slashing, MAX_PROPOSER_SLASHINGS)
proposer_slashing, PROPOSER_SLASHINGS_BOUND)
ok(true)
@ -150,7 +220,7 @@ proc validateVoluntaryExit*(
pool.chainDag.headState.data.data.validators.lenu64:
const err_str: cstring = "validateVoluntaryExit: validator index too high"
return err((EVRESULT_IGNORE, err_str))
if signed_voluntary_exit.message.validator_index.ValidatorIndex in
if signed_voluntary_exit.message.validator_index in
pool.prior_seen_voluntary_exit_indices:
const err_str: cstring = "validateVoluntaryExit: validator index already voluntarily exited"
return err((EVRESULT_IGNORE, err_str))
@ -167,8 +237,8 @@ proc validateVoluntaryExit*(
return err((EVRESULT_REJECT, voluntary_exit_validity.error))
pool.prior_seen_voluntary_exit_indices.incl(
signed_voluntary_exit.message.validator_index.ValidatorIndex)
signed_voluntary_exit.message.validator_index)
pool.voluntary_exits.addExitMessage(
signed_voluntary_exit, MAX_VOLUNTARY_EXITS)
signed_voluntary_exit, VOLUNTARY_EXITS_BOUND)
ok(true)

11
beacon_chain/spec/state_transition.nim
View File

@ -263,6 +263,9 @@ proc makeBeaconBlock*(
graffiti: GraffitiBytes,
attestations: seq[Attestation],
deposits: seq[Deposit],
proposerSlashings: seq[ProposerSlashing],
attesterSlashings: seq[AttesterSlashing],
voluntaryExits: seq[SignedVoluntaryExit],
rollback: RollbackHashedProc,
cache: var StateCache): Option[BeaconBlock] =
## Create a block for the given state. The last block applied to it must be
@ -280,8 +283,14 @@ proc makeBeaconBlock*(
randao_reveal: randao_reveal,
eth1_data: eth1data,
graffiti: graffiti,
proposer_slashings: List[ProposerSlashing, Limit MAX_PROPOSER_SLASHINGS](
proposerSlashings),
attester_slashings: List[AttesterSlashing, Limit MAX_ATTESTER_SLASHINGS](
attesterSlashings),
attestations: List[Attestation, Limit MAX_ATTESTATIONS](attestations),
deposits: List[Deposit, Limit MAX_DEPOSITS](deposits)))
deposits: List[Deposit, Limit MAX_DEPOSITS](deposits),
voluntary_exits:
List[SignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS](voluntaryExits)))
let ok = process_block(preset, state.data, blck, {skipBlsValidation}, cache)

5
beacon_chain/validator_duties.nim
View File

@ -21,7 +21,7 @@ import
spec/[datatypes, digest, crypto, helpers, validator, network, signatures],
spec/state_transition,
conf, time, validator_pool,
attestation_pool, block_pools/[spec_cache, chain_dag, clearance],
attestation_pool, exit_pool, block_pools/[spec_cache, chain_dag, clearance],
eth2_network, keystore_management, beacon_node_common, beacon_node_types,
nimbus_binary_common, mainchain_monitor, version, ssz/merkleization, interop,
attestation_aggregation, sync_manager, sszdump,
@ -237,6 +237,9 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode,
graffiti,
node.attestationPool[].getAttestationsForBlock(state, cache),
deposits,
node.exitPool[].getProposerSlashingsForBlock(),
node.exitPool[].getAttesterSlashingsForBlock(),
node.exitPool[].getVoluntaryExitsForBlock(),
restore,
cache)

3
research/block_sim.nim
View File

@ -119,6 +119,9 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
default(GraffitiBytes),
attPool.getAttestationsForBlock(state, cache),
@[],
@[],
@[],
@[],
noRollback,
cache)

7
tests/test_exit_pool.nim
View File

@ -35,7 +35,12 @@ suiteReport "Exit pool testing suite":
for i in 0'u64 .. MAX_ATTESTER_SLASHINGS + 5:
for j in 0'u64 .. i:
pool.attester_slashings.addExitMessage(
AttesterSlashing(), MAX_ATTESTER_SLASHINGS)
AttesterSlashing(
attestation_1: IndexedAttestation(attesting_indices:
List[uint64, Limit MAX_VALIDATORS_PER_COMMITTEE](@[0'u64])),
attestation_2: IndexedAttestation(attesting_indices:
List[uint64, Limit MAX_VALIDATORS_PER_COMMITTEE](@[0'u64]))),
MAX_ATTESTER_SLASHINGS)
check:
pool[].getAttesterSlashingsForBlock().lenu64 ==
min(i + 1, MAX_ATTESTER_SLASHINGS)

3
tests/testblockutil.nim
View File

@ -122,6 +122,9 @@ proc addTestBlock*(
graffiti,
attestations,
deposits,
@[],
@[],
@[],
noRollback,
cache)