diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md index 53ebeb730..fd4cb0586 100644 --- a/AllTests-mainnet.md +++ b/AllTests-mainnet.md @@ -98,15 +98,19 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 + Mocked start private key OK ``` OK: 3/3 Fail: 0/3 Skip: 0/3 -## Keystore +## KeyStorage testing suite ```diff -+ Pbkdf2 decryption OK -+ Pbkdf2 encryption OK + Pbkdf2 errors OK -+ Scrypt decryption OK -+ Scrypt encryption OK ++ [PBKDF2] Keystore decryption OK ++ [PBKDF2] Keystore encryption OK ++ [PBKDF2] Network Keystore decryption OK ++ [PBKDF2] Network Keystore encryption OK ++ [SCRYPT] Keystore decryption OK ++ [SCRYPT] Keystore encryption OK ++ [SCRYPT] Network Keystore decryption OK ++ [SCRYPT] Network Keystore encryption OK ``` -OK: 5/5 Fail: 0/5 Skip: 0/5 +OK: 9/9 Fail: 0/9 Skip: 0/9 ## Mocking utilities ```diff + merkle_minimal OK @@ -281,4 +285,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1 OK: 1/1 Fail: 0/1 Skip: 0/1 ---TOTAL--- -OK: 151/159 Fail: 0/159 Skip: 8/159 +OK: 155/163 Fail: 0/163 Skip: 8/163 diff --git a/beacon_chain/beacon_node_types.nim b/beacon_chain/beacon_node_types.nim index 65a5c3878..185f81508 100644 --- a/beacon_chain/beacon_node_types.nim +++ b/beacon_chain/beacon_node_types.nim @@ -84,13 +84,13 @@ type voluntary_exits*: Deque[SignedVoluntaryExit] ## \ ## Not a function of chain DAG branch; just used as a FIFO queue for blocks - prior_seen_attester_slashed_indices*: HashSet[ValidatorIndex] ##\ + prior_seen_attester_slashed_indices*: HashSet[uint64] ##\ ## Records attester-slashed indices seen. - prior_seen_proposer_slashed_indices*: HashSet[ValidatorIndex] ##\ + prior_seen_proposer_slashed_indices*: HashSet[uint64] ##\ ## Records proposer-slashed indices seen. - prior_seen_voluntary_exit_indices*: HashSet[ValidatorIndex] ##\ + prior_seen_voluntary_exit_indices*: HashSet[uint64] ##\ ## Records voluntary exit indices seen. chainDag*: ChainDAGRef diff --git a/beacon_chain/exit_pool.nim b/beacon_chain/exit_pool.nim index fe83aa0e5..b3289e0d6 100644 --- a/beacon_chain/exit_pool.nim +++ b/beacon_chain/exit_pool.nim @@ -9,7 +9,7 @@ import # Standard libraries - std/[deques, options, sequtils, sets], + std/[deques, options, sequtils, sets, tables], # Status libraries chronicles, json_serialization/std/sets as jsonSets, # Internal @@ -19,55 +19,125 @@ import export beacon_node_types, sets -logScope: topics = "slashpool" +logScope: topics = "exitpool" + +const + ATTESTER_SLASHINGS_BOUND = MAX_ATTESTER_SLASHINGS * 2 + PROPOSER_SLASHINGS_BOUND = MAX_PROPOSER_SLASHINGS * 2 + VOLUNTARY_EXITS_BOUND = MAX_VOLUNTARY_EXITS * 2 proc init*( T: type ExitPool, chainDag: ChainDAGRef, quarantine: QuarantineRef): T = ## Initialize an ExitPool from the chainDag `headState` T( + # Allow for filtering out some exit messages during block production attester_slashings: - initDeque[AttesterSlashing](initialSize = MAX_ATTESTER_SLASHINGS.int), + initDeque[AttesterSlashing](initialSize = ATTESTER_SLASHINGS_BOUND.int), proposer_slashings: - initDeque[ProposerSlashing](initialSize = MAX_PROPOSER_SLASHINGS.int), + initDeque[ProposerSlashing](initialSize = PROPOSER_SLASHINGS_BOUND.int), voluntary_exits: - initDeque[SignedVoluntaryExit](initialSize = MAX_VOLUNTARY_EXITS.int), + initDeque[SignedVoluntaryExit](initialSize = VOLUNTARY_EXITS_BOUND.int), chainDag: chainDag, quarantine: quarantine ) func addExitMessage*(subpool: var auto, exitMessage, bound: auto) = - # Prefer newer to older exit message + # Prefer newer to older exit messages while subpool.lenu64 >= bound: discard subpool.popFirst() subpool.addLast(exitMessage) doAssert subpool.lenu64 <= bound -func getExitMessagesForBlock[T](subpool: var Deque[T], bound: uint64): seq[T] = - for i in 0 ..< bound: - if subpool.len == 0: - break - result.add subpool.popFirst() +iterator getValidatorIndices(attester_slashing: AttesterSlashing): uint64 = + # TODO rely on sortedness and do this sans memory allocations, but it's only + # when producing a beacon block, which is rare bottlenecked elsewhere. + let + attestation_1_indices = + attester_slashing.attestation_1.attesting_indices.asSeq + attestation_2_indices = + attester_slashing.attestation_2.attesting_indices.asSeq + attester_slashed_indices = + toHashSet(attestation_1_indices) * toHashSet(attestation_2_indices) + for validator_index in attester_slashed_indices: + yield validator_index + +iterator getValidatorIndices(proposer_slashing: ProposerSlashing): uint64 = + yield proposer_slashing.signed_header_1.message.proposer_index + +iterator getValidatorIndices(voluntary_exit: SignedVoluntaryExit): uint64 = + yield voluntary_exit.message.validator_index + +# TODO stew/sequtils2 +template allIt(s, pred: untyped): bool = + # https://github.com/nim-lang/Nim/blob/version-1-2/lib/pure/collections/sequtils.nim#L640-L662 + # without the items(...) + var result = true + for it {.inject.} in s: + if not pred: + result = false + break + result + +func getExitMessagesForBlock[T]( + subpool: var Deque[T], pool: var ExitPool, bound: uint64): seq[T] = + # Approach taken here is to simply collect messages, effectively, a circular + # buffer and only re-validate that they haven't already found themselves out + # of the network eventually via some exit message at block construction time + # at which point we use exit_epoch. It doesn't matter which of these message + # types has triggered that exit, as the validation on incoming messages will + # find it to either be IGNORE (if it's the same type of exit message) or, if + # it's a different type, REJECT. Neither is worth packaging into BeaconBlock + # messages we broadcast. + # + # Beyond that, no other criterion of the exit messages' validity changes from + # when they were created, so given that we validated them to start with, they + # otherwise remain as valid as when we received them. There's no need to thus + # re-validate them on their way out. + # + # This overall approach handles a scenario wherein we receive an exit message + # over gossip and put it in the pool; receive a block X, with that message in + # it, and select it as head; then orphan block X and build instead on X-1. If + # this occurs, only validating after the fact ensures that we still broadcast + # out those exit messages that were in orphaned block X by not having eagerly + # removed them, if we have the chance. + while true: + if subpool.len == 0 or result.lenu64 >= bound: + break + + # Prefer recent messages + let exit_message = subpool.popLast() + + if allIt( + getValidatorIndices(exit_message), + pool.chainDag.headState.data.data.validators[it].exit_epoch != + FAR_FUTURE_EPOCH): + # A beacon block exit message already targeted all these validators + continue + + result.add exit_message + + subpool.clear() doAssert result.lenu64 <= bound func getAttesterSlashingsForBlock*(pool: var ExitPool): seq[AttesterSlashing] = ## Retrieve attester slashings that may be added to a new block getExitMessagesForBlock[AttesterSlashing]( - pool.attester_slashings, MAX_ATTESTER_SLASHINGS) + pool.attester_slashings, pool, MAX_ATTESTER_SLASHINGS) func getProposerSlashingsForBlock*(pool: var ExitPool): seq[ProposerSlashing] = ## Retrieve proposer slashings that may be added to a new block getExitMessagesForBlock[ProposerSlashing]( - pool.proposer_slashings, MAX_PROPOSER_SLASHINGS) + pool.proposer_slashings, pool, MAX_PROPOSER_SLASHINGS) func getVoluntaryExitsForBlock*(pool: var ExitPool): seq[SignedVoluntaryExit] = ## Retrieve voluntary exits that may be added to a new block getExitMessagesForBlock[SignedVoluntaryExit]( - pool.voluntary_exits, MAX_VOLUNTARY_EXITS) + pool.voluntary_exits, pool, MAX_VOLUNTARY_EXITS) # https://github.com/ethereum/eth2.0-specs/blob/v0.12.3/specs/phase0/p2p-interface.md#attester_slashing proc validateAttesterSlashing*( @@ -78,14 +148,14 @@ proc validateAttesterSlashing*( # attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices), # verify if any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))). # TODO sequtils2 should be able to make this more reasonable, from asSeq on - # down - let attester_slashed_indices = - toHashSet(mapIt( - attester_slashing.attestation_1.attesting_indices.asSeq, - it.ValidatorIndex)) * - toHashSet(mapIt( - attester_slashing.attestation_2.attesting_indices.asSeq, - it.ValidatorIndex)) + # down, and can sort and just find intersection that way + let + attestation_1_indices = + attester_slashing.attestation_1.attesting_indices.asSeq + attestation_2_indices = + attester_slashing.attestation_2.attesting_indices.asSeq + attester_slashed_indices = + toHashSet(attestation_1_indices) * toHashSet(attestation_2_indices) if not disjoint( attester_slashed_indices, pool.prior_seen_attester_slashed_indices): @@ -106,7 +176,7 @@ proc validateAttesterSlashing*( pool.prior_seen_attester_slashed_indices.incl attester_slashed_indices pool.attester_slashings.addExitMessage( - attester_slashing, MAX_ATTESTER_SLASHINGS) + attester_slashing, ATTESTER_SLASHINGS_BOUND) ok(true) @@ -117,7 +187,7 @@ proc validateProposerSlashing*( # [IGNORE] The proposer slashing is the first valid proposer slashing # received for the proposer with index # proposer_slashing.signed_header_1.message.proposer_index. - if proposer_slashing.signed_header_1.message.proposer_index.ValidatorIndex in + if proposer_slashing.signed_header_1.message.proposer_index in pool.prior_seen_proposer_slashed_indices: const err_str: cstring = "validateProposerSlashing: proposer-slashed index already proposer-slashed" @@ -134,9 +204,9 @@ proc validateProposerSlashing*( return err((EVRESULT_REJECT, proposer_slashing_validity.error)) pool.prior_seen_proposer_slashed_indices.incl( - proposer_slashing.signed_header_1.message.proposer_index.ValidatorIndex) + proposer_slashing.signed_header_1.message.proposer_index) pool.proposer_slashings.addExitMessage( - proposer_slashing, MAX_PROPOSER_SLASHINGS) + proposer_slashing, PROPOSER_SLASHINGS_BOUND) ok(true) @@ -150,7 +220,7 @@ proc validateVoluntaryExit*( pool.chainDag.headState.data.data.validators.lenu64: const err_str: cstring = "validateVoluntaryExit: validator index too high" return err((EVRESULT_IGNORE, err_str)) - if signed_voluntary_exit.message.validator_index.ValidatorIndex in + if signed_voluntary_exit.message.validator_index in pool.prior_seen_voluntary_exit_indices: const err_str: cstring = "validateVoluntaryExit: validator index already voluntarily exited" return err((EVRESULT_IGNORE, err_str)) @@ -167,8 +237,8 @@ proc validateVoluntaryExit*( return err((EVRESULT_REJECT, voluntary_exit_validity.error)) pool.prior_seen_voluntary_exit_indices.incl( - signed_voluntary_exit.message.validator_index.ValidatorIndex) + signed_voluntary_exit.message.validator_index) pool.voluntary_exits.addExitMessage( - signed_voluntary_exit, MAX_VOLUNTARY_EXITS) + signed_voluntary_exit, VOLUNTARY_EXITS_BOUND) ok(true) diff --git a/beacon_chain/spec/state_transition.nim b/beacon_chain/spec/state_transition.nim index 1785c2d64..8f59f3a57 100644 --- a/beacon_chain/spec/state_transition.nim +++ b/beacon_chain/spec/state_transition.nim @@ -263,6 +263,9 @@ proc makeBeaconBlock*( graffiti: GraffitiBytes, attestations: seq[Attestation], deposits: seq[Deposit], + proposerSlashings: seq[ProposerSlashing], + attesterSlashings: seq[AttesterSlashing], + voluntaryExits: seq[SignedVoluntaryExit], rollback: RollbackHashedProc, cache: var StateCache): Option[BeaconBlock] = ## Create a block for the given state. The last block applied to it must be @@ -280,8 +283,14 @@ proc makeBeaconBlock*( randao_reveal: randao_reveal, eth1_data: eth1data, graffiti: graffiti, + proposer_slashings: List[ProposerSlashing, Limit MAX_PROPOSER_SLASHINGS]( + proposerSlashings), + attester_slashings: List[AttesterSlashing, Limit MAX_ATTESTER_SLASHINGS]( + attesterSlashings), attestations: List[Attestation, Limit MAX_ATTESTATIONS](attestations), - deposits: List[Deposit, Limit MAX_DEPOSITS](deposits))) + deposits: List[Deposit, Limit MAX_DEPOSITS](deposits), + voluntary_exits: + List[SignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS](voluntaryExits))) let ok = process_block(preset, state.data, blck, {skipBlsValidation}, cache) diff --git a/beacon_chain/validator_duties.nim b/beacon_chain/validator_duties.nim index 3045de534..412b61d1f 100644 --- a/beacon_chain/validator_duties.nim +++ b/beacon_chain/validator_duties.nim @@ -21,7 +21,7 @@ import spec/[datatypes, digest, crypto, helpers, validator, network, signatures], spec/state_transition, conf, time, validator_pool, - attestation_pool, block_pools/[spec_cache, chain_dag, clearance], + attestation_pool, exit_pool, block_pools/[spec_cache, chain_dag, clearance], eth2_network, keystore_management, beacon_node_common, beacon_node_types, nimbus_binary_common, mainchain_monitor, version, ssz/merkleization, interop, attestation_aggregation, sync_manager, sszdump, @@ -237,6 +237,9 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode, graffiti, node.attestationPool[].getAttestationsForBlock(state, cache), deposits, + node.exitPool[].getProposerSlashingsForBlock(), + node.exitPool[].getAttesterSlashingsForBlock(), + node.exitPool[].getVoluntaryExitsForBlock(), restore, cache) diff --git a/research/block_sim.nim b/research/block_sim.nim index cfde435c4..4e7561527 100644 --- a/research/block_sim.nim +++ b/research/block_sim.nim @@ -119,6 +119,9 @@ cli do(slots = SLOTS_PER_EPOCH * 6, default(GraffitiBytes), attPool.getAttestationsForBlock(state, cache), @[], + @[], + @[], + @[], noRollback, cache) diff --git a/tests/test_exit_pool.nim b/tests/test_exit_pool.nim index 7cc61e63f..f540ba238 100644 --- a/tests/test_exit_pool.nim +++ b/tests/test_exit_pool.nim @@ -35,7 +35,12 @@ suiteReport "Exit pool testing suite": for i in 0'u64 .. MAX_ATTESTER_SLASHINGS + 5: for j in 0'u64 .. i: pool.attester_slashings.addExitMessage( - AttesterSlashing(), MAX_ATTESTER_SLASHINGS) + AttesterSlashing( + attestation_1: IndexedAttestation(attesting_indices: + List[uint64, Limit MAX_VALIDATORS_PER_COMMITTEE](@[0'u64])), + attestation_2: IndexedAttestation(attesting_indices: + List[uint64, Limit MAX_VALIDATORS_PER_COMMITTEE](@[0'u64]))), + MAX_ATTESTER_SLASHINGS) check: pool[].getAttesterSlashingsForBlock().lenu64 == min(i + 1, MAX_ATTESTER_SLASHINGS) diff --git a/tests/testblockutil.nim b/tests/testblockutil.nim index e45fc01f2..acd001eb8 100644 --- a/tests/testblockutil.nim +++ b/tests/testblockutil.nim @@ -122,6 +122,9 @@ proc addTestBlock*( graffiti, attestations, deposits, + @[], + @[], + @[], noRollback, cache)