Getting genesis through web3

beacon_chain/beacon_node.nim

@@ -7,7 +7,7 @@ import
   conf, time, state_transition, fork_choice, ssz, beacon_chain_db,
   validator_pool, extras, attestation_pool, block_pool, eth2_network,
   beacon_node_types, mainchain_monitor, trusted_state_snapshots, version,
-  sync_protocol, request_manager
+  sync_protocol, request_manager, genesis
 
 const
   topicBeaconBlocks = "ethereum/2.1/beacon_chain/blocks"
@@ -149,11 +149,14 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
 
     try:
       info "Importing snapshot file", path = snapshotFile
+      info "Waiting for genesis state from eth1"
 
       let
-        tailState = Json.loadFile(snapshotFile, BeaconState)
+        tailState = await getGenesisFromEth1(conf)
+        # tailState = Json.loadFile(snapshotFile, BeaconState)
         tailBlock = get_initial_beacon_block(tailState)
 
+      info "Got genesis state", hash = hash_tree_root(tailState)
       BlockPool.preInit(result.db, tailState, tailBlock)
 
     except SerializationError as err:

beacon_chain/conf.nim

@@ -77,6 +77,14 @@ type
               "If you set this to 'auto', a persistent automatically generated ID will be seleceted for each --dataDir folder"
         defaultValue: ""}: string
 
+      depositWeb3Url* {.
+        desc: "URL of the Web3 server to observe Eth1",
+        defaultValue: ""}: string
+
+      depositContractAddress* {.
+        desc: "Address of the deposit contract",
+        defaultValue: ""}: string
+
     of createTestnet:
       networkId* {.
         desc: "An unique numeric identifier for the network".}: uint8

beacon_chain/genesis.nim

@@ -0,0 +1,60 @@
+import conf, chronos, web3, json_rpc/rpcclient, json,
+  spec/[bitfield, datatypes, digest, crypto, beaconstate, helpers, validator], extras
+
+contract(DepositContract):
+  proc deposit(pubkey: Bytes48, withdrawalCredentials: Bytes32, signature: Bytes96)
+  proc Deposit(pubkey: Bytes48, withdrawalCredentials: Bytes32, amount: Bytes8, signature: Bytes96, merkleTreeIndex: Bytes8) {.event.}
+
+const MIN_GENESIS_TIME = 0
+
+type
+  QueueElement = (BlockHash, DepositData)
+
+  DepositCollector = ref object
+    deposits: seq[datatypes.Deposit]
+    queue: AsyncQueue[QueueElement]
+
+proc processDeposit(d: DepositCollector, web3: Web3): Future[BeaconState] {.async.} =
+  while true:
+    let (blkHash, data) = await d.queue.popFirst()
+
+    let blk = await web3.provider.eth_getBlockByHash(blkHash, false)
+    let dep = datatypes.Deposit(data: data)
+    d.deposits.add(dep)
+
+    if d.deposits.len >= SLOTS_PER_EPOCH and d.deposits.len >= MIN_GENESIS_ACTIVE_VALIDATOR_COUNT and blk.timestamp.uint64 >= MIN_GENESIS_TIME.uint64:
+      # This block is a genesis candidate
+      var h: Eth2Digest
+      h.data = array[32, byte](blkHash)
+      let s = initialize_beacon_state_from_eth1(h, blk.timestamp.uint64, d.deposits, {skipValidation})
+  
+      if is_valid_genesis_state(s):
+        return s
+
+proc getGenesisFromEth1*(conf: BeaconNodeConf): Future[BeaconState] {.async.} =
+  let provider = newRpcWebSocketClient()
+  await provider.connect(conf.depositWeb3Url)
+  let web3 = newWeb3(provider)
+
+  var contractAddress = Address.fromHex(conf.depositContractAddress)
+  var defaultAccount: Address
+  var ns = web3.contractSender(DepositContract, contractAddress, defaultAccount)
+
+  var deposits = DepositCollector()
+  deposits.queue = newAsyncQueue[QueueElement]()
+
+  let s = await ns.subscribe(Deposit, %*{"fromBlock": "0x0"}) do(pubkey: Bytes48, withdrawalCredentials: Bytes32, amount: Bytes8, signature: Bytes96, merkleTreeIndex: Bytes8, j: JsonNode):
+
+    let blkHash = BlockHash.fromHex(j["blockHash"].getStr())
+    let amount = bytes_to_int(array[8, byte](amount))
+
+    deposits.queue.addLastNoWait((blkHash,
+      DepositData(pubkey: ValidatorPubKey.init(array[48, byte](pubkey)),
+        withdrawal_credentials: Eth2Digest(data: array[32, byte](withdrawalCredentials)),
+        amount: amount,
+        signature: ValidatorSig.init(array[96, byte](signature)))))
+
+  let genesisState = await processDeposit(deposits, web3)
+  await s.unsubscribe()
+  return genesisState
+

beacon_chain/spec/beaconstate.nim

@@ -248,6 +248,22 @@ func initialize_beacon_state_from_eth1*(
 
   state
 
+proc initialize_beacon_state_from_eth1*(eth1_block_hash: Eth2Digest,
+    eth1_timestamp: uint64,
+    deposits: openarray[Deposit],
+    flags: UpdateFlags = {}): BeaconState =
+  # TODO: Revisit
+  initialize_beacon_state_from_eth1(deposits, eth1_timestamp, Eth1Data(deposit_count: deposits.len.uint64, deposit_root: eth1_block_hash), flags)
+
+proc is_valid_genesis_state*(state: BeaconState): bool =
+  if state.genesis_time < MIN_GENESIS_TIME:
+    return false
+  if len(get_active_validator_indices(state, GENESIS_EPOCH)) < MIN_GENESIS_ACTIVE_VALIDATOR_COUNT:
+    return false
+  return true
+
+# TODO candidate for spec?
+# https://github.com/ethereum/eth2.0-specs/blob/0.5.1/specs/core/0_beacon-chain.md#on-genesis
 func get_initial_beacon_block*(state: BeaconState): BeaconBlock =
   BeaconBlock(
     slot: GENESIS_SLOT,

beacon_chain/spec/presets/mainnet.nim

@@ -161,6 +161,9 @@ const
   MAX_VOLUNTARY_EXITS* = 2^4
   MAX_TRANSFERS* = 0
 
+  MIN_GENESIS_TIME* {.intdefine.} = 0
+  MIN_GENESIS_ACTIVE_VALIDATOR_COUNT* {.intdefine.} = 99
+
 type
   # Signature domains
   # ---------------------------------------------------------------

beacon_chain/spec/presets/minimal.nim

@@ -128,6 +128,11 @@ const
   MAX_VOLUNTARY_EXITS* = 2^4
   MAX_TRANSFERS* = 0
 
+
+  MIN_GENESIS_TIME* {.intdefine.} = 0
+  MIN_GENESIS_ACTIVE_VALIDATOR_COUNT* {.intdefine.} = 99
+
+
 type
   # Signature domains
   # ---------------------------------------------------------------

beacon_chain/validator_keygen.nim

@@ -1,9 +1,14 @@
 import
   os, ospaths, strutils, strformat,
-  chronos, blscurve, nimcrypto, json_serialization, confutils,
+  chronos, blscurve, nimcrypto, json_serialization, confutils, web3, stint,
   spec/[datatypes, digest, crypto], conf, time, ssz,
   ../tests/testutil
 
+import json_rpc/rpcclient
+
+contract(DepositContract):
+  proc deposit(pubkey: Bytes48, withdrawalCredentials: Bytes32, signature: Bytes96)
+
 proc writeTextFile(filename: string, contents: string) =
   writeFile(filename, contents)
   echo "Wrote ", filename
@@ -12,9 +17,23 @@ proc writeFile(filename: string, value: auto) =
   Json.saveFile(filename, value, pretty = true)
   echo "Wrote ", filename
 
-cli do (totalValidators: int = 125000,
-        outputDir: string = "validators",
-        generateFakeKeys = false):
+
+proc ethToWei(eth: UInt256): UInt256 =
+  eth * 1000000000000000000.u256
+
+import web3/stintjson, json
+
+proc main(totalValidators: int, outputDir: string, generateFakeKeys: bool, depositWeb3Url, depositContractAddress: string) {.async.} =
+  var web3: Web3
+  var contractAddress: Address
+  var eth1Addresses: seq[Address]
+
+  if depositWeb3Url.len > 0:
+    let provider = newRpcWebSocketClient()
+    await provider.connect(depositWeb3Url)
+    web3 = newWeb3(provider)
+    contractAddress = Address.fromHex(depositContractAddress)
+    eth1Addresses = await web3.provider.eth_accounts()
 
   for i in 0 ..< totalValidators:
     let
@@ -35,18 +54,32 @@ cli do (totalValidators: int = 125000,
       domain = 3'u64
 
     var
-      deposit = Deposit(
+      dp = Deposit(
         data: DepositData(
           amount: MAX_EFFECTIVE_BALANCE,
           pubkey: pubKey,
           withdrawal_credentials: withdrawalCredentials))
 
-    deposit.data.signature =
-      bls_sign(privkey, signing_root(deposit.data).data,
+    dp.data.signature =
+      bls_sign(privkey, signing_root(dp.data).data,
                domain)
 
     writeTextFile(privKeyFn, $privKey)
-    writeFile(depositFn, deposit)
+    writeFile(depositFn, dp)
+
+    if not web3.isNil:
+      let depositContract = web3.contractSender(DepositContract, contractAddress, eth1Addresses[i])
+      depositContract.value = 32.u256.ethToWei
+      let tx = await depositContract.deposit(Bytes48(pubKey.getBytes()), Bytes32(withdrawalCredentials.data), Bytes96(dp.data.signature.getBytes()))
 
   if generateFakeKeys:
     echo "Keys generated by this tool are only for testing!"
+
+
+cli do (totalValidators: int = 125000,
+        outputDir: string = "validators",
+        generateFakeKeys = false,
+        depositWeb3Url: string = "",
+        depositContractAddress: string = ""):
+
+  waitFor main(totalValidators, outputDir, generateFakeKeys, depositWeb3Url, depositContractAddress)

tests/simulation/run_node.sh

@@ -32,4 +32,6 @@ $BEACON_NODE_BIN \
   --tcpPort:$PORT \
   --udpPort:$PORT \
   $NAT_FLAG \
-  --stateSnapshot:$SNAPSHOT_FILE
+  --stateSnapshot:$SNAPSHOT_FILE \
+  --depositWeb3Url=$DEPOSIT_WEB3_URL \
+  --depositContractAddress=$DEPOSIT_CONTRACT_ADDRESS

tests/simulation/start.sh

@@ -7,6 +7,14 @@ set -eu
 
 # Set a default value for the env vars usually supplied by nimbus Makefile
 
+export NUM_VALIDATORS=${VALIDATORS:-100}
+export NUM_NODES=${NODES:-9}
+export NUM_MISSING_NODES=${MISSING_NODES:-1}
+
+export DEPOSIT_WEB3_URL=ws://localhost:8545
+export DEPOSIT_CONTRACT_ADDRESS=
+
+
 cd "$SIM_ROOT"
 mkdir -p "$SIMULATION_DIR"
 mkdir -p "$VALIDATORS_DIR"
@@ -25,12 +33,18 @@ LAST_VALIDATOR="$VALIDATORS_DIR/v$(printf '%07d' $LAST_VALIDATOR_NUM).deposit.js
 if [ ! -f $LAST_VALIDATOR ]; then
   if [[ -z "$SKIP_BUILDS" ]]; then
     nim c -o:"$VALIDATOR_KEYGEN_BIN" $DEFS -d:release beacon_chain/validator_keygen
+    nim c -o:"$DEPLOY_DEPOSIT_CONTRACT_BIN" $DEFS -d:release beacon_chain/deploy_deposit_contract
+
   fi
 
+  export DEPOSIT_CONTRACT_ADDRESS=$($DEPLOY_DEPOSIT_CONTRACT_BIN --depositWeb3Url=$DEPOSIT_WEB3_URL)
+
   $VALIDATOR_KEYGEN_BIN \
     --totalValidators=$NUM_VALIDATORS \
     --outputDir="$VALIDATORS_DIR" \
-    --generateFakeKeys=yes
+    --generateFakeKeys=yes \
+    --depositWeb3Url=$DEPOSIT_WEB3_URL \
+    --depositContractAddress=$DEPOSIT_CONTRACT_ADDRESS
 fi
 
 if [[ -z "$SKIP_BUILDS" ]]; then

tests/simulation/vars.sh

@@ -24,4 +24,5 @@ SNAPSHOT_FILE="${SIMULATION_DIR}/state_snapshot.json"
 NETWORK_METADATA_FILE="${SIMULATION_DIR}/network.json"
 BEACON_NODE_BIN="${BUILD_OUTPUTS_DIR}/beacon_node"
 VALIDATOR_KEYGEN_BIN="${BUILD_OUTPUTS_DIR}/validator_keygen"
+DEPLOY_DEPOSIT_CONTRACT_BIN="${BUILD_OUTPUTS_DIR}/deploy_deposit_contract"
 MASTER_NODE_ADDRESS_FILE="${SIMULATION_DIR}/node-0/beacon_node.address"