Merge pull request #1837 from status-im/devel

Merge devel to fix links in the README

AllTests-mainnet.md

@@ -98,15 +98,19 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 + Mocked start private key                                                                   OK
 ```
 OK: 3/3 Fail: 0/3 Skip: 0/3
-## Keystore
+## KeyStorage testing suite
 ```diff
-+ Pbkdf2 decryption                                                                          OK
-+ Pbkdf2 encryption                                                                          OK
 + Pbkdf2 errors                                                                              OK
-+ Scrypt decryption                                                                          OK
++ [PBKDF2] Keystore decryption                                                               OK
-+ Scrypt encryption                                                                          OK
++ [PBKDF2] Keystore encryption                                                               OK
++ [PBKDF2] Network Keystore decryption                                                       OK
++ [PBKDF2] Network Keystore encryption                                                       OK
++ [SCRYPT] Keystore decryption                                                               OK
++ [SCRYPT] Keystore encryption                                                               OK
++ [SCRYPT] Network Keystore decryption                                                       OK
++ [SCRYPT] Network Keystore encryption                                                       OK
 ```
-OK: 5/5 Fail: 0/5 Skip: 0/5
+OK: 9/9 Fail: 0/9 Skip: 0/9
 ## Mocking utilities
 ```diff
 + merkle_minimal                                                                             OK
@@ -281,4 +285,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 OK: 1/1 Fail: 0/1 Skip: 0/1
 
 ---TOTAL---
-OK: 151/159 Fail: 0/159 Skip: 8/159
+OK: 155/163 Fail: 0/163 Skip: 8/163

README.md

@@ -47,9 +47,9 @@ Nimbus beacon chain is a research implementation of the beacon chain component o
 
 ## Documentation
 
-You can find the information you need to run a beacon node and operate as a validator in [The Book](https://status-im.github.io/nim-beacon-chain/).
+You can find the information you need to run a beacon node and operate as a validator in [The Book](https://status-im.github.io/nimbus-eth2/).
 
-See [this page](https://status-im.github.io/nim-beacon-chain/medalla.html) for how to become a validator on Medalla.
+See [this page](https://status-im.github.io/nimbus-eth2/medalla.html) for how to become a validator on Medalla.
 
 ## Related projects
 
@@ -175,7 +175,7 @@ Unfortunately, this simple method only offers one snapshot in time (you'll need
 
 In order to settle on a better solution, we'll need the help of two external projects -- [Prometheus](https://prometheus.io/) and [Grafana](https://grafana.com/).
 
-See [this page](https://status-im.github.io/nim-beacon-chain/metrics-pretty-pictures.html#prometheus-and-grafana) from our Nimbus book for a step-by-step guide on how to use Prometheus and Grafana to spin up a beautiful and useful monitoring dashboard for your validator.
+See [this page](https://status-im.github.io/nimbus-eth2/metrics-pretty-pictures.html#prometheus-and-grafana) from our Nimbus book for a step-by-step guide on how to use Prometheus and Grafana to spin up a beautiful and useful monitoring dashboard for your validator.
 
 ------------------
 *For those of you who are already familiar with Prometheus and Grafana*

beacon_chain/beacon_node_types.nim

@@ -84,13 +84,13 @@ type
     voluntary_exits*: Deque[SignedVoluntaryExit]  ## \
     ## Not a function of chain DAG branch; just used as a FIFO queue for blocks
 
-    prior_seen_attester_slashed_indices*: HashSet[ValidatorIndex] ##\
+    prior_seen_attester_slashed_indices*: HashSet[uint64] ##\
     ## Records attester-slashed indices seen.
 
-    prior_seen_proposer_slashed_indices*: HashSet[ValidatorIndex] ##\
+    prior_seen_proposer_slashed_indices*: HashSet[uint64] ##\
     ## Records proposer-slashed indices seen.
 
-    prior_seen_voluntary_exit_indices*: HashSet[ValidatorIndex] ##\
+    prior_seen_voluntary_exit_indices*: HashSet[uint64] ##\
     ## Records voluntary exit indices seen.
 
     chainDag*: ChainDAGRef

beacon_chain/exit_pool.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard libraries
-  std/[deques, options, sequtils, sets],
+  std/[deques, options, sequtils, sets, tables],
   # Status libraries
   chronicles, json_serialization/std/sets as jsonSets,
   # Internal
@@ -19,55 +19,125 @@ import
 
 export beacon_node_types, sets
 
-logScope: topics = "slashpool"
+logScope: topics = "exitpool"
+
+const
+  ATTESTER_SLASHINGS_BOUND = MAX_ATTESTER_SLASHINGS * 2
+  PROPOSER_SLASHINGS_BOUND = MAX_PROPOSER_SLASHINGS * 2
+  VOLUNTARY_EXITS_BOUND = MAX_VOLUNTARY_EXITS * 2
 
 proc init*(
     T: type ExitPool, chainDag: ChainDAGRef, quarantine: QuarantineRef): T =
   ## Initialize an ExitPool from the chainDag `headState`
   T(
+    # Allow for filtering out some exit messages during block production
     attester_slashings:
-      initDeque[AttesterSlashing](initialSize = MAX_ATTESTER_SLASHINGS.int),
+      initDeque[AttesterSlashing](initialSize = ATTESTER_SLASHINGS_BOUND.int),
     proposer_slashings:
-      initDeque[ProposerSlashing](initialSize = MAX_PROPOSER_SLASHINGS.int),
+      initDeque[ProposerSlashing](initialSize = PROPOSER_SLASHINGS_BOUND.int),
     voluntary_exits:
-      initDeque[SignedVoluntaryExit](initialSize = MAX_VOLUNTARY_EXITS.int),
+      initDeque[SignedVoluntaryExit](initialSize = VOLUNTARY_EXITS_BOUND.int),
     chainDag: chainDag,
     quarantine: quarantine
    )
 
 func addExitMessage*(subpool: var auto, exitMessage, bound: auto) =
-  # Prefer newer to older exit message
+  # Prefer newer to older exit messages
   while subpool.lenu64 >= bound:
     discard subpool.popFirst()
 
   subpool.addLast(exitMessage)
   doAssert subpool.lenu64 <= bound
 
-func getExitMessagesForBlock[T](subpool: var Deque[T], bound: uint64): seq[T] =
+iterator getValidatorIndices(attester_slashing: AttesterSlashing): uint64 =
-  for i in 0 ..< bound:
+  # TODO rely on sortedness and do this sans memory allocations, but it's only
-    if subpool.len == 0:
+  # when producing a beacon block, which is rare bottlenecked elsewhere.
-      break
+  let
-    result.add subpool.popFirst()
+    attestation_1_indices =
+      attester_slashing.attestation_1.attesting_indices.asSeq
+    attestation_2_indices =
+      attester_slashing.attestation_2.attesting_indices.asSeq
+    attester_slashed_indices =
+      toHashSet(attestation_1_indices) * toHashSet(attestation_2_indices)
 
+  for validator_index in attester_slashed_indices:
+    yield validator_index
+
+iterator getValidatorIndices(proposer_slashing: ProposerSlashing): uint64 =
+  yield proposer_slashing.signed_header_1.message.proposer_index
+
+iterator getValidatorIndices(voluntary_exit: SignedVoluntaryExit): uint64 =
+  yield voluntary_exit.message.validator_index
+
+# TODO stew/sequtils2
+template allIt(s, pred: untyped): bool =
+  # https://github.com/nim-lang/Nim/blob/version-1-2/lib/pure/collections/sequtils.nim#L640-L662
+  # without the items(...)
+  var result = true
+  for it {.inject.} in s:
+    if not pred:
+      result = false
+      break
+  result
+
+func getExitMessagesForBlock[T](
+    subpool: var Deque[T], pool: var ExitPool, bound: uint64): seq[T] =
+  # Approach taken here is to simply collect messages, effectively, a circular
+  # buffer and only re-validate that they haven't already found themselves out
+  # of the network eventually via some exit message at block construction time
+  # at which point we use exit_epoch. It doesn't matter which of these message
+  # types has triggered that exit, as the validation on incoming messages will
+  # find it to either be IGNORE (if it's the same type of exit message) or, if
+  # it's a different type, REJECT. Neither is worth packaging into BeaconBlock
+  # messages we broadcast.
+  #
+  # Beyond that, no other criterion of the exit messages' validity changes from
+  # when they were created, so given that we validated them to start with, they
+  # otherwise remain as valid as when we received them. There's no need to thus
+  # re-validate them on their way out.
+  #
+  # This overall approach handles a scenario wherein we receive an exit message
+  # over gossip and put it in the pool; receive a block X, with that message in
+  # it, and select it as head; then orphan block X and build instead on X-1. If
+  # this occurs, only validating after the fact ensures that we still broadcast
+  # out those exit messages that were in orphaned block X by not having eagerly
+  # removed them, if we have the chance.
+  while true:
+    if subpool.len == 0 or result.lenu64 >= bound:
+      break
+
+    # Prefer recent messages
+    let exit_message = subpool.popLast()
+
+    if allIt(
+        getValidatorIndices(exit_message),
+        pool.chainDag.headState.data.data.validators[it].exit_epoch !=
+          FAR_FUTURE_EPOCH):
+      # A beacon block exit message already targeted all these validators
+      continue
+
+    result.add exit_message
+
+  subpool.clear()
   doAssert result.lenu64 <= bound
 
 func getAttesterSlashingsForBlock*(pool: var ExitPool):
                                    seq[AttesterSlashing] =
   ## Retrieve attester slashings that may be added to a new block
   getExitMessagesForBlock[AttesterSlashing](
-    pool.attester_slashings, MAX_ATTESTER_SLASHINGS)
+    pool.attester_slashings, pool, MAX_ATTESTER_SLASHINGS)
 
 func getProposerSlashingsForBlock*(pool: var ExitPool):
                                    seq[ProposerSlashing] =
   ## Retrieve proposer slashings that may be added to a new block
   getExitMessagesForBlock[ProposerSlashing](
-    pool.proposer_slashings, MAX_PROPOSER_SLASHINGS)
+    pool.proposer_slashings, pool, MAX_PROPOSER_SLASHINGS)
 
 func getVoluntaryExitsForBlock*(pool: var ExitPool):
                                 seq[SignedVoluntaryExit] =
   ## Retrieve voluntary exits that may be added to a new block
   getExitMessagesForBlock[SignedVoluntaryExit](
-    pool.voluntary_exits, MAX_VOLUNTARY_EXITS)
+    pool.voluntary_exits, pool, MAX_VOLUNTARY_EXITS)
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.3/specs/phase0/p2p-interface.md#attester_slashing
 proc validateAttesterSlashing*(
@@ -78,14 +148,14 @@ proc validateAttesterSlashing*(
   # attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices),
   # verify if any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))).
   # TODO sequtils2 should be able to make this more reasonable, from asSeq on
-  # down
+  # down, and can sort and just find intersection that way
-  let attester_slashed_indices =
+  let
-    toHashSet(mapIt(
+    attestation_1_indices =
-      attester_slashing.attestation_1.attesting_indices.asSeq,
+      attester_slashing.attestation_1.attesting_indices.asSeq
-      it.ValidatorIndex)) *
+    attestation_2_indices =
-    toHashSet(mapIt(
+      attester_slashing.attestation_2.attesting_indices.asSeq
-      attester_slashing.attestation_2.attesting_indices.asSeq,
+    attester_slashed_indices =
-      it.ValidatorIndex))
+      toHashSet(attestation_1_indices) * toHashSet(attestation_2_indices)
 
   if not disjoint(
       attester_slashed_indices, pool.prior_seen_attester_slashed_indices):
@@ -106,7 +176,7 @@ proc validateAttesterSlashing*(
 
   pool.prior_seen_attester_slashed_indices.incl attester_slashed_indices
   pool.attester_slashings.addExitMessage(
-    attester_slashing, MAX_ATTESTER_SLASHINGS)
+    attester_slashing, ATTESTER_SLASHINGS_BOUND)
 
   ok(true)
 
@@ -117,7 +187,7 @@ proc validateProposerSlashing*(
   # [IGNORE] The proposer slashing is the first valid proposer slashing
   # received for the proposer with index
   # proposer_slashing.signed_header_1.message.proposer_index.
-  if proposer_slashing.signed_header_1.message.proposer_index.ValidatorIndex in
+  if proposer_slashing.signed_header_1.message.proposer_index in
       pool.prior_seen_proposer_slashed_indices:
     const err_str: cstring =
       "validateProposerSlashing: proposer-slashed index already proposer-slashed"
@@ -134,9 +204,9 @@ proc validateProposerSlashing*(
     return err((EVRESULT_REJECT, proposer_slashing_validity.error))
 
   pool.prior_seen_proposer_slashed_indices.incl(
-    proposer_slashing.signed_header_1.message.proposer_index.ValidatorIndex)
+    proposer_slashing.signed_header_1.message.proposer_index)
   pool.proposer_slashings.addExitMessage(
-    proposer_slashing, MAX_PROPOSER_SLASHINGS)
+    proposer_slashing, PROPOSER_SLASHINGS_BOUND)
 
   ok(true)
 
@@ -150,7 +220,7 @@ proc validateVoluntaryExit*(
       pool.chainDag.headState.data.data.validators.lenu64:
     const err_str: cstring = "validateVoluntaryExit: validator index too high"
     return err((EVRESULT_IGNORE, err_str))
-  if signed_voluntary_exit.message.validator_index.ValidatorIndex in
+  if signed_voluntary_exit.message.validator_index in
       pool.prior_seen_voluntary_exit_indices:
     const err_str: cstring = "validateVoluntaryExit: validator index already voluntarily exited"
     return err((EVRESULT_IGNORE, err_str))
@@ -167,8 +237,8 @@ proc validateVoluntaryExit*(
     return err((EVRESULT_REJECT, voluntary_exit_validity.error))
 
   pool.prior_seen_voluntary_exit_indices.incl(
-    signed_voluntary_exit.message.validator_index.ValidatorIndex)
+    signed_voluntary_exit.message.validator_index)
   pool.voluntary_exits.addExitMessage(
-    signed_voluntary_exit, MAX_VOLUNTARY_EXITS)
+    signed_voluntary_exit, VOLUNTARY_EXITS_BOUND)
 
   ok(true)

beacon_chain/spec/state_transition.nim

@@ -263,6 +263,9 @@ proc makeBeaconBlock*(
     graffiti: GraffitiBytes,
     attestations: seq[Attestation],
     deposits: seq[Deposit],
+    proposerSlashings: seq[ProposerSlashing],
+    attesterSlashings: seq[AttesterSlashing],
+    voluntaryExits: seq[SignedVoluntaryExit],
     rollback: RollbackHashedProc,
     cache: var StateCache): Option[BeaconBlock] =
   ## Create a block for the given state. The last block applied to it must be
@@ -280,8 +283,14 @@ proc makeBeaconBlock*(
       randao_reveal: randao_reveal,
       eth1_data: eth1data,
       graffiti: graffiti,
+      proposer_slashings: List[ProposerSlashing, Limit MAX_PROPOSER_SLASHINGS](
+        proposerSlashings),
+      attester_slashings: List[AttesterSlashing, Limit MAX_ATTESTER_SLASHINGS](
+        attesterSlashings),
       attestations: List[Attestation, Limit MAX_ATTESTATIONS](attestations),
-      deposits: List[Deposit, Limit MAX_DEPOSITS](deposits)))
+      deposits: List[Deposit, Limit MAX_DEPOSITS](deposits),
+      voluntary_exits:
+        List[SignedVoluntaryExit, Limit MAX_VOLUNTARY_EXITS](voluntaryExits)))
 
   let ok = process_block(preset, state.data, blck, {skipBlsValidation}, cache)
 

beacon_chain/validator_duties.nim

@@ -21,7 +21,7 @@ import
   spec/[datatypes, digest, crypto, helpers, validator, network, signatures],
   spec/state_transition,
   conf, time, validator_pool,
-  attestation_pool, block_pools/[spec_cache, chain_dag, clearance],
+  attestation_pool, exit_pool, block_pools/[spec_cache, chain_dag, clearance],
   eth2_network, keystore_management, beacon_node_common, beacon_node_types,
   nimbus_binary_common, mainchain_monitor, version, ssz/merkleization, interop,
   attestation_aggregation, sync_manager, sszdump,
@@ -237,6 +237,9 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode,
       graffiti,
       node.attestationPool[].getAttestationsForBlock(state, cache),
       deposits,
+      node.exitPool[].getProposerSlashingsForBlock(),
+      node.exitPool[].getAttesterSlashingsForBlock(),
+      node.exitPool[].getVoluntaryExitsForBlock(),
       restore,
       cache)
 

research/block_sim.nim

@@ -119,6 +119,9 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
           default(GraffitiBytes),
           attPool.getAttestationsForBlock(state, cache),
           @[],
+          @[],
+          @[],
+          @[],
           noRollback,
           cache)
 

tests/test_exit_pool.nim

@@ -35,7 +35,12 @@ suiteReport "Exit pool testing suite":
     for i in 0'u64 .. MAX_ATTESTER_SLASHINGS + 5:
       for j in 0'u64 .. i:
         pool.attester_slashings.addExitMessage(
-          AttesterSlashing(), MAX_ATTESTER_SLASHINGS)
+          AttesterSlashing(
+            attestation_1: IndexedAttestation(attesting_indices:
+              List[uint64, Limit MAX_VALIDATORS_PER_COMMITTEE](@[0'u64])),
+            attestation_2: IndexedAttestation(attesting_indices:
+              List[uint64, Limit MAX_VALIDATORS_PER_COMMITTEE](@[0'u64]))),
+          MAX_ATTESTER_SLASHINGS)
       check:
         pool[].getAttesterSlashingsForBlock().lenu64 ==
           min(i + 1, MAX_ATTESTER_SLASHINGS)

tests/testblockutil.nim

@@ -122,6 +122,9 @@ proc addTestBlock*(
       graffiti,
       attestations,
       deposits,
+      @[],
+      @[],
+      @[],
       noRollback,
       cache)