block pool simulator (#956)

* block pool simulator

like state_sim, but more

Makefile

@@ -23,7 +23,9 @@ TOOLS := \
 	ncli_pretty \
 	ncli_transition \
 	process_dashboard \
-	stackSizes
+	stackSizes \
+	state_sim \
+	block_sim
 	# bench_bls_sig_agggregation TODO reenable after bls v0.10.1 changes
 TOOLS_DIRS := \
 	beacon_chain \
@@ -134,7 +136,7 @@ schlesi-dev: | build deps
 		$(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims $(SCRIPT_PARAMS) --dev-build shared/schlesi
 
 clean: | clean-common
-	rm -rf build/{$(TOOLS_CSV),all_tests,*_node,*ssz*,beacon_node_testnet*,state_sim,transition*}
+	rm -rf build/{$(TOOLS_CSV),all_tests,*_node,*ssz*,beacon_node_testnet*,block_sim,state_sim,transition*}
 ifneq ($(USE_LIBBACKTRACE), 0)
 	+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
 endif

beacon_chain/block_pool.nim

@@ -230,7 +230,6 @@ proc init*(T: type BlockPool, db: BeaconChainDB): BlockPool =
     justifiedHead = headRef.atSlot(justifiedSlot)
     head = Head(blck: headRef, justified: justifiedHead)
     justifiedBlock = db.getBlock(justifiedHead.blck.root).get()
-    justifiedStateRoot = justifiedBlock.message.state_root
 
   doAssert justifiedHead.slot >= finalizedHead.slot,
     "justified head comes before finalized head - database corrupt?"

research/block_sim.nim

@@ -0,0 +1,155 @@
+# beacon_chain
+# Copyright (c) 2019-2020 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+# `block_sim` is a block and attestation simulator similar to `state_sim` whose
+# task is to run the beacon chain without considering the network or the
+# wall clock. Functionally, it achieves the same as the distributed beacon chain
+# by producing blocks and attestations as if they were created by separate
+# nodes, just like a set of `beacon_node` instances would.
+#
+# Similar to `state_sim`, but uses the block and attestation pools along with
+# a database, as if a real node was running.
+
+import
+  confutils, chronicles, stats, times,
+  strformat,
+  options, random, tables,
+  ../tests/[testblockutil],
+  ../beacon_chain/spec/[
+    beaconstate, crypto, datatypes, digest, helpers, validator,
+    state_transition_block],
+  ../beacon_chain/[
+    attestation_pool, block_pool, beacon_node_types, beacon_chain_db,
+    interop, ssz, validator_pool],
+  eth/db/[kvstore, kvstore_sqlite3],
+  ./simutils
+
+type Timers = enum
+  tBlock = "Process non-epoch slot with block"
+  tEpoch = "Process epoch slot with block"
+  tHashBlock = "Tree-hash block"
+  tSignBlock = "Sign block"
+  tShuffle = "Retrieve committee once using get_beacon_committee"
+  tAttest = "Combine committee attestations"
+
+# TODO confutils is an impenetrable black box. how can a help text be added here?
+cli do(slots = SLOTS_PER_EPOCH * 6,
+       validators = SLOTS_PER_EPOCH * 100, # One per shard is minimum
+       attesterRatio {.desc: "ratio of validators that attest in each round"} = 0.73):
+  let
+    state = loadGenesis(validators, true)
+    genesisBlock = get_initial_beacon_block(state[])
+
+  echo "Starting simulation..."
+
+  let
+    db = BeaconChainDB.init(kvStore SqStoreRef.init(".", "block_sim").tryGet())
+
+  BlockPool.preInit(db, state[], genesisBlock)
+
+  var
+    blockPool = BlockPool.init(db)
+    attPool = AttestationPool.init(blockPool)
+    timers: array[Timers, RunningStat]
+    attesters: RunningStat
+    r: Rand
+
+  proc handleAttestations() =
+    let
+      slot = blockPool.head.blck.slot
+      attestationHead = blockPool.head.blck.atSlot(slot)
+
+    blockPool.withState(blockPool.tmpState, attestationHead):
+      var cache = get_empty_per_epoch_cache()
+      let committees_per_slot = get_committee_count_at_slot(state, slot)
+
+      for committee_index in 0'u64..<committees_per_slot:
+        let committee = get_beacon_committee(
+          state, slot, committee_index.CommitteeIndex, cache)
+
+        for index_in_committee, validatorIdx in committee:
+          if (rand(r, high(int)).float * attesterRatio).int <= high(int):
+            let
+              data = makeAttestationData(state, slot, committee_index, blck.root)
+              sig =
+                get_attestation_signature(state.fork,
+                  state.genesis_validators_root,
+                  data, hackPrivKey(state.validators[validatorIdx]))
+            var aggregation_bits = CommitteeValidatorsBits.init(committee.len)
+            aggregation_bits.setBit index_in_committee
+
+            attPool.add(
+              Attestation(
+                data: data,
+                aggregation_bits: aggregation_bits,
+                signature: sig
+              ))
+
+  proc proposeBlock() =
+    let
+      head = blockPool.head.blck
+      slot = blockPool.head.blck.slot + 1
+
+    blockPool.withState(blockPool.tmpState, head.atSlot(slot)):
+      var cache = get_empty_per_epoch_cache()
+
+      let
+        proposerIdx = get_beacon_proposer_index(state, cache).get()
+        privKey = hackPrivKey(state.validators[proposerIdx])
+        eth1data = get_eth1data_stub(
+          state.eth1_deposit_index, slot.compute_epoch_at_slot())
+        message = makeBeaconBlock(
+          state,
+          head.root,
+          privKey.genRandaoReveal(state.fork, state.genesis_validators_root, slot),
+          eth1data,
+          Eth2Digest(),
+          attPool.getAttestationsForBlock(state),
+          @[])
+
+      var
+        newBlock = SignedBeaconBlock(
+          message: message.get()
+        )
+
+      let blockRoot = withTimerRet(timers[tHashBlock]):
+        hash_tree_root(newBlock.message)
+
+      # Careful, state no longer valid after here because of the await..
+      newBlock.signature = withTimerRet(timers[tSignBlock]):
+        get_block_signature(
+          state.fork, state.genesis_validators_root, newBlock.message.slot,
+          blockRoot, privKey)
+
+      let added = blockPool.add(blockRoot, newBlock)
+      blockPool.updateHead(added)
+
+  for i in 0..<slots:
+    let
+      slot = blockPool.headState.data.data.slot + 1
+      t =
+        if slot mod SLOTS_PER_EPOCH == 0: tEpoch
+        else: tBlock
+
+    withTimer(timers[t]):
+      proposeBlock()
+    if attesterRatio > 0.0:
+      withTimer(timers[tAttest]):
+        handleAttestations()
+
+    verifyConsensus(blockPool.headState.data.data, attesterRatio)
+
+    if t == tEpoch:
+      echo &". slot: {shortLog(slot)} ",
+        &"epoch: {shortLog(slot.compute_epoch_at_slot)}"
+    else:
+      write(stdout, ".")
+      flushFile(stdout)
+
+  echo "Done!"
+
+  printTimers(blockPool.headState.data.data, attesters, true, timers)

research/block_sim.nim.cfg

@@ -0,0 +1,2 @@
+-u:metrics
+-d:"chronicles_sinks=json[file(block_sim.log)]"

research/simutils.nim

@@ -0,0 +1,91 @@
+import
+  stats, os, strformat,
+  ../tests/[testblockutil],
+  ../beacon_chain/[extras, ssz],
+  ../beacon_chain/spec/[beaconstate, datatypes, digest, helpers]
+
+template withTimer*(stats: var RunningStat, body: untyped) =
+  # TODO unify timing somehow
+  let start = cpuTime()
+
+  block:
+    body
+
+  let stop = cpuTime()
+  stats.push stop - start
+
+template withTimerRet*(stats: var RunningStat, body: untyped): untyped =
+  let start = cpuTime()
+  let tmp = block:
+    body
+  let stop = cpuTime()
+  stats.push stop - start
+
+  tmp
+
+func verifyConsensus*(state: BeaconState, attesterRatio: auto) =
+  if attesterRatio < 0.63:
+    doAssert state.current_justified_checkpoint.epoch == 0
+    doAssert state.finalized_checkpoint.epoch == 0
+
+  # Quorum is 2/3 of validators, and at low numbers, quantization effects
+  # can dominate, so allow for play above/below attesterRatio of 2/3.
+  if attesterRatio < 0.72:
+    return
+
+  let current_epoch = get_current_epoch(state)
+  if current_epoch >= 3:
+    doAssert state.current_justified_checkpoint.epoch + 1 >= current_epoch
+  if current_epoch >= 4:
+    doAssert state.finalized_checkpoint.epoch + 2 >= current_epoch
+
+proc loadGenesis*(validators: int, validate: bool): ref BeaconState =
+  let fn = &"genesim_{const_preset}_{validators}"
+  if fileExists(fn):
+    let res = newClone(SSZ.loadFile(fn, BeaconState))
+    if res.slot != GENESIS_SLOT:
+      echo "Can only start from genesis state"
+      quit 1
+
+    if res.validators.len != validators:
+      echo &"Supplied genesis file has {res.validators.len} validators, while {validators} where requested, running anyway"
+
+    echo &"Loaded {fn}..."
+    # TODO check that the private keys are interop keys
+    res
+  else:
+    echo "Genesis file not found, making one up (use beacon_node createTestnet to make one)"
+
+    echo "Preparing validators..."
+    let
+      flags = if validate: {} else: {skipBlsValidation}
+      deposits = makeInitialDeposits(validators, flags)
+
+    echo "Generating Genesis..."
+
+    let state =
+      initialize_beacon_state_from_eth1(Eth2Digest(), 0, deposits, flags)
+
+    echo &"Saving to {fn}..."
+    SSZ.saveFile(fn, state[])
+    state
+
+proc printTimers*[Timers: enum](
+    state: BeaconState, attesters: RunningStat, validate: bool,
+    timers: array[Timers, RunningStat]) =
+  echo "Validators: ", state.validators.len, ", epoch length: ", SLOTS_PER_EPOCH
+  echo "Validators per attestation (mean): ", attesters.mean
+
+  proc fmtTime(t: float): string = &"{t * 1000 :>12.3f}, "
+
+  echo "All time are ms"
+  echo &"{\"Average\" :>12}, {\"StdDev\" :>12}, {\"Min\" :>12}, " &
+    &"{\"Max\" :>12}, {\"Samples\" :>12}, {\"Test\" :>12}"
+
+  if not validate:
+    echo "Validation is turned off meaning that no BLS operations are performed"
+
+  for t in Timers:
+    echo fmtTime(timers[t].mean), fmtTime(timers[t].standardDeviationS),
+      fmtTime(timers[t].min), fmtTime(timers[t].max), &"{timers[t].n :>12}, ",
+      $t

research/state_sim.nim

@@ -5,13 +5,17 @@
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
+# `state_sim` runs the state transition function in isolation, creating blocks
+# and attesting to them as if the network was running as a whole.
+
 import
-  confutils, stats, times, std/monotimes,
+  confutils, stats, times,
   strformat,
   options, sequtils, random, tables,
   ../tests/[testblockutil],
   ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, validator],
-  ../beacon_chain/[attestation_pool, extras, ssz]
+  ../beacon_chain/[attestation_pool, extras, ssz],
+  ./simutils
 
 type Timers = enum
   tBlock = "Process non-epoch slot with block"
@@ -20,24 +24,6 @@ type Timers = enum
   tShuffle = "Retrieve committee once using get_beacon_committee"
   tAttest = "Combine committee attestations"
 
-template withTimer(stats: var RunningStat, body: untyped) =
-  let start = cpuTime()
-
-  block:
-    body
-
-  let stop = cpuTime()
-  stats.push stop - start
-
-template withTimerRet(stats: var RunningStat, body: untyped): untyped =
-  let start = cpuTime()
-  let tmp = block:
-    body
-  let stop = cpuTime()
-  stats.push stop - start
-
-  tmp
-
 proc jsonName(prefix, slot: auto): string =
   fmt"{prefix:04}-{shortLog(slot):08}.json"
 
@@ -46,22 +32,6 @@ proc writeJson*(fn, v: auto) =
   defer: close(f)
   Json.saveFile(fn, v, pretty = true)
 
-func verifyConsensus(state: BeaconState, attesterRatio: auto) =
-  if attesterRatio < 0.63:
-    doAssert state.current_justified_checkpoint.epoch == 0
-    doAssert state.finalized_checkpoint.epoch == 0
-
-  # Quorum is 2/3 of validators, and at low numbers, quantization effects
-  # can dominate, so allow for play above/below attesterRatio of 2/3.
-  if attesterRatio < 0.72:
-    return
-
-  let current_epoch = get_current_epoch(state)
-  if current_epoch >= 3:
-    doAssert state.current_justified_checkpoint.epoch + 1 >= current_epoch
-  if current_epoch >= 4:
-    doAssert state.finalized_checkpoint.epoch + 2 >= current_epoch
-
 cli do(slots = SLOTS_PER_EPOCH * 6,
        validators = SLOTS_PER_EPOCH * 100, # One per shard is minimum
        json_interval = SLOTS_PER_EPOCH,
@@ -69,17 +39,10 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
        prefix = 0,
        attesterRatio {.desc: "ratio of validators that attest in each round"} = 0.73,
        validate = true):
-  echo "Preparing validators..."
   let
-    # TODO should we include other skipXXXValidation flags here (e.g. StateRoot)?
     flags = if validate: {} else: {skipBlsValidation}
-    deposits = makeInitialDeposits(validators, flags)
-
-  echo "Generating Genesis..."
-
-  let
-    state = initialize_beacon_state_from_eth1(Eth2Digest(), 0, deposits, flags)
-  let genesisBlock = get_initial_beacon_block(state[])
+    state = loadGenesis(validators, validate)
+    genesisBlock = get_initial_beacon_block(state[])
 
   echo "Starting simulation..."
 
@@ -191,19 +154,4 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
 
   echo "Done!"
 
-  echo "Validators: ", validators, ", epoch length: ", SLOTS_PER_EPOCH
-  echo "Validators per attestation (mean): ", attesters.mean
-
-  proc fmtTime(t: float): string = &"{t * 1000 :>12.3f}, "
-
-  echo "All time are ms"
-  echo &"{\"Average\" :>12}, {\"StdDev\" :>12}, {\"Min\" :>12}, " &
-    &"{\"Max\" :>12}, {\"Samples\" :>12}, {\"Test\" :>12}"
-
-  if not validate:
-    echo "Validation is turned off meaning that no BLS operations are performed"
-
-  for t in Timers:
-    echo fmtTime(timers[t].mean), fmtTime(timers[t].standardDeviationS),
-      fmtTime(timers[t].min), fmtTime(timers[t].max), &"{timers[t].n :>12}, ",
-      $t
+  printTimers(state[], attesters, validate, timers)

tests/testblockutil.nim

@@ -25,7 +25,7 @@ func makeFakeHash(i: int): Eth2Digest =
   static: doAssert sizeof(bytes) <= sizeof(result.data)
   copyMem(addr result.data[0], addr bytes[0], sizeof(bytes))
 
-func hackPrivKey(v: Validator): ValidatorPrivKey =
+func hackPrivKey*(v: Validator): ValidatorPrivKey =
   ## Extract private key, per above hack
   var bytes: array[8, byte]
   static: doAssert sizeof(bytes) <= sizeof(v.withdrawal_credentials.data)