Automate the creation of the network metadata files

With these changes, running a simulation is very close to running
an actual testnet. Some checks have been added in the client to
make sure you are not connecting to an incompatible network (e.g.
a network running with a different number of shards).

beacon_chain/beacon_node.nim

@@ -15,7 +15,7 @@ const
   dataDirValidators = "validators"
   networkMetadataFile = "network.json"
   genesisFile = "genesis.json"
-  testnetsBaseUrl = "http://node-01.do-ams3.nimbus.misc.statusim.net:8000/nimbus_testnets"
+  testnetsBaseUrl = "https://serenity-testnets.status.im"
 
 # #################################################
 # Careful handling of beacon_node <-> sync_protocol
@@ -46,28 +46,25 @@ proc downloadFile(url: string): Future[string] {.async.} =
     raise newException(IOError, "Failed to download URL: " & url)
   return fileContents
 
-proc doUpdateTestnet(conf: BeaconNodeConf) {.async.} =
+proc updateTestnetMetadata(conf: BeaconNodeConf): Future[NetworkMetadata] {.async.} =
   let latestMetadata = await downloadFile(testnetsBaseUrl // $conf.network // networkMetadataFile)
 
   let localMetadataFile = conf.dataDir / networkMetadataFile
   if fileExists(localMetadataFile) and readFile(localMetadataFile).string == latestMetadata:
     return
 
+  result = Json.decode(latestMetadata, NetworkMetadata)
+
+  info "New testnet genesis data received. Starting with a fresh database."
   removeDir conf.databaseDir
   writeFile localMetadataFile, latestMetadata
 
   let newGenesis = await downloadFile(testnetsBaseUrl // $conf.network // genesisFile)
   writeFile conf.dataDir / genesisFile, newGenesis
 
-  info "New testnet genesis data received. Starting with a fresh database."
-
-proc loadTestnetMetadata(conf: BeaconNodeConf): TestnetMetadata =
-  Json.loadFile(conf.dataDir / networkMetadataFile, TestnetMetadata)
-
 proc obtainTestnetKey(conf: BeaconNodeConf): Future[ValidatorPrivKey] {.async.} =
-  await doUpdateTestnet(conf)
   let
-    metadata = loadTestnetMetadata(conf)
+    metadata = await updateTestnetMetadata(conf)
     privKeyName = validatorFileBaseName(rand(metadata.userValidatorsRange)) & ".privkey"
     privKeyContent = await downloadFile(testnetsBaseUrl // $conf.network // privKeyName)
 
@@ -81,8 +78,34 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
   new result
   result.config = conf
 
-  if conf.network in {testnet0, testnet1}:
+  template fail(args: varargs[untyped]) =
-    await doUpdateTestnet(conf)
+    stderr.write args, "\n"
+    quit 1
+
+  case conf.network
+  of "mainnet":
+    fail "The Serenity mainnet hasn't been launched yet"
+  of "testnet0", "testnet1":
+    result.networkMetadata = await updateTestnetMetadata(conf)
+  else:
+    try:
+      result.networkMetadata = Json.loadFile(conf.network, NetworkMetadata)
+    except:
+      fail "Failed to load network metadata: ", getCurrentExceptionMsg()
+
+  var metadataErrorMsg = ""
+
+  template checkCompatibility(metadataField, LOCAL_CONSTANT) =
+    let metadataValue = metadataField
+    if metadataValue != LOCAL_CONSTANT:
+      metadataErrorMsg.add " -d:" & astToStr(LOCAL_CONSTANT) & "=" & $metadataValue
+
+  checkCompatibility result.networkMetadata.numShards      , SHARD_COUNT
+  checkCompatibility result.networkMetadata.slotDuration   , SECONDS_PER_SLOT
+  checkCompatibility result.networkMetadata.slotsPerEpoch  , SLOTS_PER_EPOCH
+
+  if metadataErrorMsg.len > 0:
+    fail "To connect to the ", conf.network, " network, please compile with ", metadataErrorMsg
 
   result.attachedValidators = ValidatorPool.init
   init result.mainchainMonitor, "", Port(0) # TODO: specify geth address and port
@@ -126,10 +149,7 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
   result.network = await createEth2Node(conf)
 
   let sync = result.network.protocolState(BeaconSync)
-  sync.networkId = case conf.network
+  sync.networkId = result.networkMetadata.networkId
-                   of mainnet: 1.uint64
-                   of ephemeralNetwork: 1000.uint64
-                   of testnet0, testnet1: loadTestnetMetadata(conf).networkId
   sync.node = result
   sync.db = result.db
 
@@ -141,20 +161,16 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
   result.network.saveConnectionAddressFile(addressFile)
 
 proc connectToNetwork(node: BeaconNode) {.async.} =
-  var bootstrapNodes = newSeq[BootstrapAddr]()
+  var bootstrapNodes = node.networkMetadata.bootstrapNodes
 
-  for node in node.config.bootstrapNodes:
+  for bootNode in node.config.bootstrapNodes:
-    bootstrapNodes.add BootstrapAddr.init(node)
+    bootstrapNodes.add BootstrapAddr.init(bootNode)
 
   let bootstrapFile = string node.config.bootstrapNodesFile
   if bootstrapFile.len > 0:
     for ln in lines(bootstrapFile):
       bootstrapNodes.add BootstrapAddr.init(string ln)
 
-  if node.config.network in {testnet0, testnet1}:
-    let metadata = loadTestnetMetadata(node.config)
-    bootstrapNodes.add metadata.bootstrapNodes
-
   if bootstrapNodes.len > 0:
     info "Connecting to bootstrap nodes", bootstrapNodes
   else:
@@ -665,11 +681,40 @@ when isMainModule:
     setLogLevel(config.logLevel)
 
   case config.cmd
-  of createChain:
+  of createTestnet:
-    createStateSnapshot(
+    var deposits: seq[Deposit]
-      config.validatorsDir.string, config.numValidators, config.firstValidator,
+    for i in config.firstValidator.int ..< config.numValidators.int:
-      config.genesisOffset, config.outputStateFile.string)
+      let depositFile = config.validatorsDir /
-    quit 0
+                        validatorFileBaseName(i) & ".deposit.json"
+      deposits.add Json.loadFile(depositFile, Deposit)
+
+    let initialState = get_genesis_beacon_state(
+      deposits,
+      uint64(int(fastEpochTime() div 1000) + config.genesisOffset),
+      Eth1Data(), {})
+
+    Json.saveFile(config.outputGenesis.string, initialState, pretty = true)
+    echo "Wrote ", config.outputGenesis.string
+
+    var
+      bootstrapAddress = getPersistenBootstrapAddr(
+        config, parseIpAddress(config.bootstrapAddress), Port config.bootstrapPort)
+
+      testnetMetadata = NetworkMetadata(
+        networkId: config.networkId,
+        genesisRoot: hash_tree_root_final(initialState),
+        bootstrapNodes: @[bootstrapAddress],
+        numShards: SHARD_COUNT,
+        slotDuration: SECONDS_PER_SLOT,
+        slotsPerEpoch: SLOTS_PER_EPOCH,
+        totalValidators: config.numValidators,
+        firstUserValidator: config.firstUserValidator)
+
+    Json.saveFile(config.outputNetwork.string, testnetMetadata, pretty = true)
+    echo "Wrote ", config.outputNetwork.string
+
+  of updateTestnet:
+    discard waitFor updateTestnetMetadata(config)
 
   of importValidator:
     template reportFailureFor(keyExpr) =
@@ -693,9 +738,11 @@ when isMainModule:
         reportFailureFor config.keyFile.get.string
 
     if downloadKey:
-      if config.network in {testnet0, testnet1}:
+      if config.network in ["testnet0", "testnet1"]:
         try:
-          (waitFor obtainTestnetKey(config)).saveValidatorKey(config)
+          let key = waitFor obtainTestnetKey(config)
+          saveValidatorKey(key, config)
+          info "Imported validator", pubkey = key.pubKey
         except:
           error "Failed to download key", err = getCurrentExceptionMsg()
           quit 1
@@ -703,9 +750,6 @@ when isMainModule:
         echo "Validator keys can be downloaded only for testnets"
         quit 1
 
-  of updateTestnet:
-    waitFor doUpdateTestnet(config)
-
   of noCommand:
     waitFor synchronizeClock()
     createPidFile(config.dataDir.string / "beacon_node.pid")

beacon_chain/beacon_node_types.nim

@@ -1,7 +1,7 @@
 import # Beacon Node
   eth/[p2p, keys],
   spec/digest,
-  beacon_chain_db, conf, mainchain_monitor
+  beacon_chain_db, conf, mainchain_monitor, eth2_network
 
 import # Attestation Pool
   spec/[datatypes, crypto, digest],
@@ -25,6 +25,7 @@ type
   # #############################################
   BeaconNode* = ref object
     network*: EthereumNode
+    networkMetadata*: NetworkMetadata
     db*: BeaconChainDB
     config*: BeaconNodeConf
     keys*: KeyPair
@@ -230,3 +231,17 @@ type
 
   ValidatorPool* = object
     validators*: Table[ValidatorPubKey, AttachedValidator]
+
+  NetworkMetadata* = object
+    networkId*: uint64
+    genesisRoot*: Eth2Digest
+    bootstrapNodes*: seq[BootstrapAddr]
+    numShards*: uint64
+    slotDuration*: uint64
+    slotsPerEpoch*: uint64
+    totalValidators*: uint64
+    firstUserValidator*: uint64
+
+proc userValidatorsRange*(d: NetworkMetadata): HSlice[int, int] =
+  d.firstUserValidator.int ..< d.totalValidators.int
+

beacon_chain/conf.nim

@@ -11,37 +11,32 @@ type
 
   StartUpCommand* = enum
     noCommand
-    createChain
+    createTestnet
     importValidator
     updateTestnet
 
-  Network* = enum
-    ephemeralNetwork
-    testnet0
-    testnet1
-    mainnet
-
   BeaconNodeConf* = object
     logLevel* {.
       desc: "Sets the log level",
       defaultValue: enabledLogLevel.}: LogLevel
 
     network* {.
-      desc: "The network Nimbus should connect to"
+      desc: "The network Nimbus should connect to. " &
+            "Possible values: testnet0, testnet1, mainnet, custom-network.json"
       longform: "network"
       shortform: "n"
-      defaultValue: testnet0.}: Network
+      defaultValue: "testnet0".}: string
+
+    dataDir* {.
+      desc: "The directory where nimbus will store all blockchain data."
+      shortform: "d"
+      defaultValue: config.defaultDataDir().}: OutDir
 
     case cmd* {.
       command
       defaultValue: noCommand.}: StartUpCommand
 
     of noCommand:
-      dataDir* {.
-        desc: "The directory where nimbus will store all blockchain data."
-        shortform: "d"
-        defaultValue: config.defaultDataDir().}: OutDir
-
       bootstrapNodes* {.
         desc: "Specifies one or more bootstrap nodes to use when connecting to the network."
         longform: "bootstrapNode"
@@ -74,27 +69,43 @@ type
         desc: "Json file specifying a recent state snapshot"
         shortform: "s".}: Option[TypedInputFile[BeaconState, Json, "json"]]
 
-    of createChain:
+    of createTestnet:
+      networkId* {.
+        desc: "An unique numeric identifier for the network".}: uint64
+
       validatorsDir* {.
         desc: "Directory containing validator descriptors named vXXXXXXX.deposit.json"
         shortform: "d".}: InputDir
 
       numValidators* {.
-        desc: "The number of validators in the newly created chain".}: int
+        desc: "The number of validators in the newly created chain".}: uint64
 
       firstValidator* {.
-        desc: "index of first validator to add to validator list"
+        desc: "Index of first validator to add to validator list"
-        defaultValue: 0.}: int
+        defaultValue: 0 .}: uint64
+
+      firstUserValidator* {.
+        desc: "The first validator index that will free for taking from a testnet participant"
+        defaultValue: 0 .}: uint64
+
+      bootstrapAddress* {.
+        desc: "The public IP address that will be advertised as a bootstrap node for the testnet"
+        defaultValue: "127.0.0.1".}: string
+
+      bootstrapPort* {.
+        desc: "The TCP/UDP port that will be used by the bootstrap node"
+        defaultValue: config.defaultPort().}: int
 
       genesisOffset* {.
         desc: "Seconds from now to add to genesis time"
         shortForm: "g"
         defaultValue: 5 .}: int
 
-      outputStateFile* {.
+      outputGenesis* {.
-        desc: "Output file where to write the initial state snapshot"
+        desc: "Output file where to write the initial state snapshot".}: OutFile
-        longform: "out"
+
-        shortform: "o".}: OutFile
+      outputNetwork* {.
+        desc: "Output file where to write the initial state snapshot".}: OutFile
 
     of importValidator:
       keyFile* {.
@@ -107,10 +118,6 @@ type
       discard
 
 proc defaultDataDir*(conf: BeaconNodeConf): string =
-  if conf.network == ephemeralNetwork:
-    getCurrentDir() / "beacon-node-cache"
-
-  else:
   let dataDir = when defined(windows):
     "AppData" / "Roaming" / "Nimbus"
   elif defined(macosx):
@@ -118,10 +125,17 @@ proc defaultDataDir*(conf: BeaconNodeConf): string =
   else:
     ".cache" / "nimbus"
 
-    getHomeDir() / dataDir / "BeaconNode" / $conf.network
+  let networkId = if conf.network in ["testnet0", "testnet1", "mainnet"]:
+    conf.network
+  else:
+    # TODO: This seems silly. Perhaps we should error out here and ask
+    # the user to specify dataDir as well.
+    "tempnet"
+
+  getHomeDir() / dataDir / "BeaconNode" / networkId
 
 proc defaultPort*(conf: BeaconNodeConf): int =
-  (if conf.network == testnet0: 9630 else: 9632) + ord(useRLPx)
+  (if conf.network == "testnet0": 9630 else: 9632) + ord(useRLPx)
 
 proc validatorFileBaseName*(validatorIdx: int): string =
   # there can apparently be tops 4M validators so we use 7 digits..

beacon_chain/eth2_network.nim

@@ -27,6 +27,26 @@ when useRLPx:
     else:
       parseIpAddress("127.0.0.1")
 
+  proc ensureNetworkKeys(conf: BeaconNodeConf): KeyPair =
+    let privateKeyFile = conf.dataDir / "network.privkey"
+    var privKey: PrivateKey
+    if not fileExists(privateKeyFile):
+      privKey = newPrivateKey()
+      createDir conf.dataDir.string
+      writeFile(privateKeyFile, $privKey)
+    else:
+      privKey = initPrivateKey(readFile(privateKeyFile).string)
+
+    KeyPair(seckey: privKey, pubkey: privKey.getPublicKey())
+
+  proc getPersistenBootstrapAddr*(conf: BeaconNodeConf,
+                                  ip: IpAddress, port: Port): BootstrapAddr =
+    let
+      keys = ensureNetworkKeys(conf)
+      address = Address(ip: ip, tcpPort: port, udpPort: port)
+
+    initENode(keys.pubKey, address)
+
   proc writeValue*(writer: var JsonWriter, value: BootstrapAddr) {.inline.} =
     writer.writeValue $value
 
@@ -34,22 +54,14 @@ when useRLPx:
     value = initENode reader.readValue(string)
 
   proc createEth2Node*(conf: BeaconNodeConf): Future[EthereumNode] {.async.} =
-    let privateKeyFile = conf.dataDir / "network.privkey"
-    var privKey: PrivateKey
-    if not fileExists(privateKeyFile):
-      privKey = newPrivateKey()
-      writeFile(privateKeyFile, $privKey)
-    else:
-      privKey = initPrivateKey(readFile(privateKeyFile).string)
-
-    # TODO there are more networking options to add here: local bind ip, ipv6
-    #      etc.
     let
-      keys = KeyPair(seckey: privKey, pubkey: privKey.getPublicKey())
+      keys = ensureNetworkKeys(conf)
       address = Address(ip: parseNat(conf.nat),
                         tcpPort: Port conf.tcpPort,
                         udpPort: Port conf.udpPort)
 
+    # TODO there are more networking options to add here: local bind ip, ipv6
+    #      etc.
     return newEthereumNode(keys, address, 0,
                            nil, clientId, minPeers = 1)
 
@@ -107,15 +119,3 @@ else:
   proc loadConnectionAddressFile*(filename: string): PeerInfo =
     Json.loadFile(filename, PeerInfo)
 
-type
-  TestnetMetadata* = object
-    networkId*: uint64
-    genesisRoot*: Eth2Digest
-    bootstrapNodes*: BootstrapAddr
-    totalValidators*: int
-    userValidatorsStart*: int
-    userValidatorsEnd*: int
-
-proc userValidatorsRange*(d: TestnetMetadata): HSlice[int, int] =
-  d.userValidatorsStart .. d.userValidatorsEnd
-

beacon_chain/trusted_state_snapshots.nim

@@ -29,19 +29,3 @@ proc obtainTrustedStateSnapshot*(db: BeaconChainDB): Future[BeaconState] {.async
 
   doAssert(false, "Not implemented")
 
-proc createStateSnapshot*(
-    validatorDir: string, numValidators, firstValidator, genesisOffset: int,
-    outFile: string) =
-
-  var deposits: seq[Deposit]
-  for i in firstValidator..<numValidators:
-    deposits.add Json.loadFile(validatorDir / &"v{i:07}.deposit.json", Deposit)
-
-  let initialState = get_genesis_beacon_state(
-    deposits,
-    uint64(int(fastEpochTime() div 1000) + genesisOffset),
-    Eth1Data(), {})
-
-  var vr: Validator
-  Json.saveFile(outFile, initialState, pretty = true)
-  echo "Wrote ", outFile

tests/simulation/run_node.sh

@@ -4,8 +4,6 @@ set -eux
 
 . $(dirname $0)/vars.sh
 
-BOOTSTRAP_NODES_FLAG="--bootstrapNodesFile:$MASTER_NODE_ADDRESS_FILE"
-
 if [[ "$1" == "0" ]]; then
   BOOTSTRAP_NODES_FLAG=""
 fi
@@ -17,7 +15,7 @@ PORT=$(printf '5%04d' ${1})
 MYIP=$(curl -s ifconfig.me)
 
 $BEACON_NODE_BIN \
-  --network:ephemeralNetwork \
+  --network:$NETWORK_METADATA_FILE \
   --dataDir:$DATA_DIR \
   --validator:${V_PREFIX}0.privkey \
   --validator:${V_PREFIX}1.privkey \
@@ -32,5 +30,5 @@ $BEACON_NODE_BIN \
   --tcpPort:$PORT \
   --udpPort:$PORT \
   --nat:extip:$MYIP \
-  --stateSnapshot:$SNAPSHOT_FILE \
+  --stateSnapshot:$SNAPSHOT_FILE
-  $BOOTSTRAP_NODES_FLAG
+

tests/simulation/start.sh

@@ -41,10 +41,14 @@ if [[ -z "$SKIP_BUILDS" ]]; then
 fi
 
 if [ ! -f $SNAPSHOT_FILE ]; then
-  $BEACON_NODE_BIN createChain \
+  $BEACON_NODE_BIN --dataDir=$SIMULATION_DIR/node-0 createTestnet \
-    --validatorsDir:$VALIDATORS_DIR \
+    --networkId=1000 \
-    --out:$SNAPSHOT_FILE \
+    --validatorsDir=$VALIDATORS_DIR \
     --numValidators=$NUM_VALIDATORS \
+    --outputGenesis=$SNAPSHOT_FILE \
+    --outputNetwork=$NETWORK_METADATA_FILE \
+    --bootstrapAddress=127.0.0.1 \
+    --bootstrapPort=50001 \
     --genesisOffset=5 # Delay in seconds
 fi
 

tests/simulation/vars.sh

@@ -16,6 +16,7 @@ GIT_ROOT="$($PWD_CMD)"
 SIMULATION_DIR="$SIM_ROOT/data"
 VALIDATORS_DIR="$SIM_ROOT/validators"
 SNAPSHOT_FILE="$SIMULATION_DIR/state_snapshot.json"
+NETWORK_METADATA_FILE="$SIMULATION_DIR/network.json"
 BEACON_NODE_BIN=$BUILD_OUTPUTS_DIR/beacon_node
 VALIDATOR_KEYGEN_BIN=$BUILD_OUTPUTS_DIR/validator_keygen
 MASTER_NODE_ADDRESS_FILE="$SIMULATION_DIR/node-0/beacon_node.address"