Clean 20200205 (#729)

* beacon node code cleanup
* rudimentary error checking on mainnet monitor
* start client even when sending deposit
* work around missing block number exception
* connect to testnet with web3 url
* pretty-print digests in json

beacon_chain/beacon_node.nim

@@ -218,6 +218,8 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
     mainchainMonitor = MainchainMonitor.init(
       conf.depositWeb3Url, conf.depositContractAddress,
       blockPool.headState.data.data.eth1_data.block_hash)
+    # TODO if we don't have any validators attached, we don't need a mainchain
+    #      monitor
     mainchainMonitor.start()
 
   var
@@ -624,31 +626,22 @@ proc handleProposal(node: BeaconNode, head: BlockRef, slot: Slot):
   # TODO here we advance the state to the new slot, but later we'll be
   #      proposing for it - basically, we're selecting proposer based on an
   #      empty slot
-  var cache = get_empty_per_epoch_cache()
-  node.blockPool.withState(node.blockPool.tmpState, head.atSlot(slot)):
-    let proposerIdx = get_beacon_proposer_index(state, cache)
-    if proposerIdx.isNone:
-      notice "Missing proposer index",
-        slot=slot,
-        epoch=slot.compute_epoch_at_slot,
-        num_validators=state.validators.len,
-        active_validators=
-          get_active_validator_indices(state, slot.compute_epoch_at_slot),
-        balances=state.balances
 
-      return head
+  let proposerKey = node.blockPool.getProposer(head, slot)
+  if proposerKey.isNone():
+    return head
 
-    let validator = node.getAttachedValidator(state, proposerIdx.get)
+  let validator = node.attachedValidators.getValidator(proposerKey.get())
 
-    if validator != nil:
-      return await proposeBlock(node, validator, head, slot)
+  if validator != nil:
+    return await proposeBlock(node, validator, head, slot)
 
-    trace "Expecting block proposal",
-      headRoot = shortLog(head.root),
-      slot = shortLog(slot),
-      proposer = shortLog(state.validators[proposerIdx.get].pubKey),
-      cat = "consensus",
-      pcs = "wait_for_proposal"
+  debug "Expecting block proposal",
+    headRoot = shortLog(head.root),
+    slot = shortLog(slot),
+    proposer = shortLog(proposerKey.get()),
+    cat = "consensus",
+    pcs = "wait_for_proposal"
 
   return head
 

beacon_chain/block_pool.nim

@@ -2,7 +2,7 @@ import
   bitops, chronicles, options, tables,
   ssz, beacon_chain_db, state_transition, extras,
   beacon_node_types, metrics,
-  spec/[crypto, datatypes, digest, helpers]
+  spec/[crypto, datatypes, digest, helpers, validator]
 
 declareCounter beacon_reorgs_total, "Total occurrences of reorganizations of the chain" # On fork choice
 
@@ -926,3 +926,20 @@ proc preInit*(
   db.putTailBlock(blockRoot)
   db.putHeadBlock(blockRoot)
   db.putStateRoot(blockRoot, state.slot, signedBlock.message.state_root)
+
+proc getProposer*(pool: BlockPool, head: BlockRef, slot: Slot): Option[ValidatorPubKey] =
+  pool.withState(pool.tmpState, head.atSlot(slot)):
+    var cache = get_empty_per_epoch_cache()
+
+    let proposerIdx = get_beacon_proposer_index(state, cache)
+    if proposerIdx.isNone:
+      warn "Missing proposer index",
+        slot=slot,
+        epoch=slot.compute_epoch_at_slot,
+        num_validators=state.validators.len,
+        active_validators=
+          get_active_validator_indices(state, slot.compute_epoch_at_slot),
+        balances=state.balances
+      return
+
+    return some(state.validators[proposerIdx.get()].pubkey)

beacon_chain/mainchain_monitor.nim

@@ -24,13 +24,16 @@ type
 
   QueueElement = (BlockHash, DepositData)
 
-
-proc init*(T: type MainchainMonitor, web3Url, depositContractAddress: string, startBlock: Eth2Digest): T =
-  result.new()
-  result.web3Url = web3Url
-  result.depositContractAddress = Address.fromHex(depositContractAddress)
-  result.depositQueue = newAsyncQueue[QueueElement]()
-  result.eth1Block = BlockHash(startBlock.data)
+proc init*(
+    T: type MainchainMonitor,
+    web3Url, depositContractAddress: string,
+    startBlock: Eth2Digest): T =
+  T(
+    web3Url: web3Url,
+    depositContractAddress: Address.fromHex(depositContractAddress),
+    depositQueue: newAsyncQueue[QueueElement](),
+    eth1Block: BlockHash(startBlock.data),
+  )
 
 contract(DepositContract):
   proc deposit(pubkey: Bytes48, withdrawalCredentials: Bytes32, signature: Bytes96, deposit_data_root: FixedBytes[32])
@@ -113,8 +116,18 @@ proc getGenesis*(m: MainchainMonitor): Future[BeaconState] {.async.} =
   return m.genesisState[]
 
 proc getBlockNumber(web3: Web3, hash: BlockHash): Future[Quantity] {.async.} =
-  let blk = await web3.provider.eth_getBlockByHash(hash, false)
-  return blk.number
+  debug "Querying block number", hash
+
+  try:
+    let blk = await web3.provider.eth_getBlockByHash(hash, false)
+    return blk.number
+  except CatchableError as exc:
+    # TODO this doesn't make too much sense really, but what would be a
+    #      reasonable behavior? no idea - the whole algorithm needs to be
+    #      rewritten to match the spec.
+    notice "Failed to get block number from hash, using current block instead",
+      hash, err = exc.msg
+    return await web3.provider.eth_blockNumber()
 
 proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
   if delayBeforeStart != ZeroDuration:
@@ -129,8 +142,15 @@ proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
     error "Web3 server disconnected", ulr = m.web3Url
     processFut.cancel()
 
+  # TODO this needs to implement follow distance and the rest of the honest
+  #      validator spec..
+
   let startBlkNum = await web3.getBlockNumber(m.eth1Block)
-  debug "Starting eth1 monitor", fromBlock = startBlkNum.uint64
+
+  notice "Monitoring eth1 deposits",
+    fromBlock = startBlkNum.uint64,
+    contract = $m.depositContractAddress,
+    url = m.web3Url
 
   let ns = web3.contractSender(DepositContract, m.depositContractAddress)
 
@@ -139,15 +159,17 @@ proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
       withdrawalCredentials: Bytes32,
       amount: Bytes8,
       signature: Bytes96, merkleTreeIndex: Bytes8, j: JsonNode):
+    try:
+      let blkHash = BlockHash.fromHex(j["blockHash"].getStr())
+      let amount = bytes_to_int(array[8, byte](amount))
 
-    let blkHash = BlockHash.fromHex(j["blockHash"].getStr())
-    let amount = bytes_to_int(array[8, byte](amount))
-
-    m.depositQueue.addLastNoWait((blkHash,
-      DepositData(pubkey: ValidatorPubKey.init(array[48, byte](pubkey)),
-        withdrawal_credentials: Eth2Digest(data: array[32, byte](withdrawalCredentials)),
-        amount: amount,
-        signature: ValidatorSig.init(array[96, byte](signature)))))
+      m.depositQueue.addLastNoWait((blkHash,
+        DepositData(pubkey: ValidatorPubKey.init(array[48, byte](pubkey)),
+          withdrawal_credentials: Eth2Digest(data: array[32, byte](withdrawalCredentials)),
+          amount: amount,
+          signature: ValidatorSig.init(array[96, byte](signature)))))
+    except CatchableError as exc:
+      warn "Received invalid deposit", err = exc.msg, j
 
   try:
     await processFut

beacon_chain/spec/digest.nim

@@ -20,12 +20,12 @@
 # we call this function `eth2hash`, and it outputs a `Eth2Digest`. Easy to sed :)
 
 import
-  chronicles,
+  chronicles, json_serialization,
   nimcrypto/[sha2, hash, utils],
   hashes
 
 export
-  hash.`$`
+  hash.`$`, json_serialization
 
 type
   Eth2Digest* = MDigest[32 * 8] ## `hash32` from spec
@@ -70,3 +70,9 @@ func hash*(x: Eth2Digest): Hash =
   # We just slice the first 4 or 8 bytes of the block hash
   # depending of if we are on a 32 or 64-bit platform
   result = cast[ptr Hash](unsafeAddr x)[]
+
+proc writeValue*(writer: var JsonWriter, value: Eth2Digest) =
+  writeValue(writer, value.data.toHex(true))
+
+proc readValue*(reader: var JsonReader, value: var Eth2Digest) =
+  value = Eth2Digest.fromHex(reader.readValue(string))

beacon_chain/sszdump.nim

@@ -0,0 +1,24 @@
+import
+  os,
+  ssz,
+  serialization,
+  beacon_node_types,
+  ./spec/[crypto, datatypes, digest]
+
+proc dump*(dir: string, v: AttestationData, validator: ValidatorPubKey) =
+  SSZ.saveFile(
+    dir / "att-" & $v.slot & "-" &
+    $v.index & "-" & validator.shortLog &
+    ".ssz", v)
+
+proc dump*(dir: string, v: SignedBeaconBlock, blck: BlockRef) =
+  SSZ.saveFile(
+    dir / "block-" & $v.message.slot & "-" &
+    shortLog(blck.root) & ".ssz", v)
+
+proc dump*(dir: string, v: HashedBeaconState, blck: BlockRef) =
+  SSZ.saveFile(
+    dir / "state-" & $v.data.slot & "-" &
+    shortLog(blck.root) & "-"  & shortLog(v.root) & ".ssz",
+    v.data)
+

ncli/ncli_pretty.nim

@@ -32,3 +32,4 @@ cli do(kind: string, file: string):
   of "state": printit(BeaconState)
   of "proposer_slashing": printit(ProposerSlashing)
   of "voluntary_exit": printit(VoluntaryExit)
+  else: echo "Unknown kind"

scripts/connect_to_testnet.nims

@@ -9,6 +9,7 @@ const
   genesisFile = "genesis.ssz"
   configFile = "config.yaml"
   testnetsRepo = "eth2-testnets"
+  web3Url = "wss://goerli.infura.io/ws/v3/809a18497dd74102b5f37d25aae3c85a"
 
 let
   testnetsOrg = getEnv("ETH2_TESTNETS_ORG", "eth2-clients")
@@ -123,15 +124,18 @@ cli do (testnetName {.argument.}: string):
         --random-deposits=1
         --deposits-dir="{validatorsDir}"
         --deposit-private-key={privKey}
-        --web3-url=wss://goerli.infura.io/ws/v3/809a18497dd74102b5f37d25aae3c85a
+        --web3-url={web3Url}
         {depositContractOpt}
         """, "\n", " ")
-      quit()
+      mode = Silent
+      echo "\nDeposit sent, wait for confirmation then press enter to continue"
+      discard readLineFromStdin()
 
   mode = Verbose
   execIgnoringExitCode replace(&"""{beaconNodeBinary}
     --data-dir="{dataDir}"
     --dump=true
+    --web3-url={web3Url}
     {bootstrapFileOpt}
     --state-snapshot="{testnetDir/genesisFile}" """ & depositContractOpt, "\n", " ")