Embed the Altona metadata in the NBC executable

.gitmodules

@@ -178,3 +178,8 @@
 	url = https://github.com/status-im/nim-chronicles-tail.git
 	ignore = dirty
 	branch = master
+[submodule "vendor/eth2-testnets"]
+	path = vendor/eth2-testnets
+	url = https://github.com/eth2-clients/eth2-testnets.git
+	ignore = dirty
+	branch = master

beacon_chain/beacon_node.nim

@@ -23,7 +23,7 @@ import
   conf, time, beacon_chain_db, validator_pool, extras,
   attestation_pool, block_pool, eth2_network, eth2_discovery,
   beacon_node_common, beacon_node_types, block_pools/block_pools_types,
-  nimbus_binary_common,
+  nimbus_binary_common, network_metadata,
   mainchain_monitor, version, ssz/[merkleization], sszdump,
   sync_protocol, request_manager, keystore_management, interop, statusbar,
   sync_manager, validator_duties, validator_api, attestation_aggregation
@@ -148,7 +148,7 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
         fatal "Web3 URL not specified"
         quit 1
 
-      if conf.depositContractAddress.len == 0:
+      if conf.depositContractAddress.isNone:
         fatal "Deposit contract address not specified"
         quit 1
 
@@ -162,7 +162,7 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
       #      that would do only this - see Paul's proposal for this.
       mainchainMonitor = MainchainMonitor.init(
         web3Provider(conf.web3Url),
-        conf.depositContractAddress,
+        conf.depositContractAddress.get,
         Eth1Data(block_hash: conf.depositContractDeployedAt.get, deposit_count: 0))
       mainchainMonitor.start()
 
@@ -203,10 +203,10 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
 
   if mainchainMonitor.isNil and
      conf.web3Url.len > 0 and
-     conf.depositContractAddress.len > 0:
+     conf.depositContractAddress.isSome:
     mainchainMonitor = MainchainMonitor.init(
       web3Provider(conf.web3Url),
-      conf.depositContractAddress,
+      conf.depositContractAddress.get,
       blockPool.headState.data.data.eth1_data)
     # TODO if we don't have any validators attached, we don't need a mainchain
     #      monitor
@@ -1114,10 +1114,43 @@ proc createWalletInteractively(conf: BeaconNodeConf): OutFile {.raises: [Defect]
       keystore_management.burnMem(confirmedPassword)
 
 programMain:
-  let config = makeBannerAndConfig(clientId, BeaconNodeConf)
+  var config = makeBannerAndConfig(clientId, BeaconNodeConf)
 
   setupMainProc(config.logLevel)
 
+  if config.eth2Network.isSome:
+    let
+      networkName = config.eth2Network.get
+      metadata = case toLowerAscii(networkName)
+        of "altona":
+          altonaMetadata
+        else:
+          if fileExists(networkName):
+            Json.loadFile(networkName, Eth2NetworkMetadata)
+          else:
+            fatal "Unrecognized network name", networkName
+            quit 1
+
+    if config.cmd == noCommand:
+      for node in metadata.bootstrapNodes:
+        config.bootstrapNodes.add node
+
+    template checkForIncompatibleOption(flagName, fieldName) =
+      # TODO: This will have to be reworked slightly when we introduce config files.
+      # We'll need to keep track of the "origin" of the config value, so we can
+      # discriminate between values from config files that can be overridden and
+      # regular command-line options (that may conflict).
+      if config.fieldName.isSome:
+        fatal "Invalid CLI arguments specified. You must not specify '--network' and '" & flagName & "' at the same time",
+            networkParam = networkName, `flagName` = config.fieldName.get
+        quit 1
+
+    checkForIncompatibleOption "deposit-contract", depositContractAddress
+    checkForIncompatibleOption "deposit-contract-block", depositContractDeployedAt
+
+    config.depositContractAddress = some metadata.depositContractAddress
+    config.depositContractDeployedAt = some metadata.depositContractDeployedAt
+
   case config.cmd
   of createTestnet:
     var

beacon_chain/beacon_node_common.nim

@@ -49,6 +49,7 @@ type
     topicAggregateAndProofs*: string
     forwardSyncLoop*: Future[void]
     onSecondLoop*: Future[void]
+    genesisSnapshotContent*: string
 
 const
   MaxEmptySlotCount* = uint64(10*60) div SECONDS_PER_SLOT

beacon_chain/conf.nim

@@ -2,13 +2,14 @@
 
 import
   os, options,
-  chronicles, confutils, json_serialization,
-  confutils/defs, confutils/std/net,
-  chronicles/options as chroniclesOptions,
-  spec/[crypto, keystore, digest]
+  chronicles, chronicles/options as chroniclesOptions,
+  confutils, confutils/defs, confutils/std/net,
+  json_serialization, web3/ethtypes,
+  network_metadata, spec/[crypto, keystore, digest]
 
 export
-  defs, enabledLogLevel, parseCmdArg, completeCmdArg
+  defs, enabledLogLevel, parseCmdArg, completeCmdArg,
+  network_metadata
 
 type
   ValidatorKeyPath* = TypedInputFile[ValidatorPrivKey, Txt, "privkey"]
@@ -32,11 +33,7 @@ type
   VCStartUpCmd* = enum
     VCNoCommand
 
-  Eth1Network* = enum
-    custom
-    mainnet
-    rinkeby
-    goerli
+  Web3Url* = distinct string
 
   BeaconNodeConf* = object
     logLevel* {.
@@ -44,10 +41,9 @@ type
       desc: "Sets the log level"
       name: "log-level" }: string
 
-    eth1Network* {.
-      defaultValue: goerli
-      desc: "The Eth1 network tracked by the beacon node"
-      name: "eth1-network" }: Eth1Network
+    eth2Network* {.
+      desc: "The Eth2 network to join"
+      name: "network" }: Option[string]
 
     dataDir* {.
       defaultValue: config.defaultDataDir()
@@ -61,9 +57,8 @@ type
       name: "web3-url" }: string
 
     depositContractAddress* {.
-      defaultValue: ""
       desc: "Address of the deposit contract"
-      name: "deposit-contract" }: string
+      name: "deposit-contract" }: Option[Eth1Address]
 
     depositContractDeployedAt* {.
       desc: "The Eth1 block hash where the deposit contract has been deployed"
@@ -412,6 +407,13 @@ proc createDumpDirs*(conf: BeaconNodeConf) =
       # Dumping is mainly a debugging feature, so ignore these..
       warn "Cannot create dump directories", msg = err.msg
 
+func parseCmdArg*(T: type Eth1Address, input: TaintedString): T
+                 {.raises: [ValueError, Defect].} =
+  fromHex(T, string input)
+
+func completeCmdArg*(T: type Eth1Address, input: TaintedString): seq[string] =
+  return @[]
+
 func parseCmdArg*(T: type Eth2Digest, input: TaintedString): T
                  {.raises: [ValueError, Defect].} =
   fromHex(T, string input)
@@ -453,3 +455,4 @@ func defaultAdminListenAddress*(conf: BeaconNodeConf|ValidatorClientConf): Valid
 template writeValue*(writer: var JsonWriter,
                      value: TypedInputFile|InputFile|InputDir|OutPath|OutDir|OutFile) =
   writer.writeValue(string value)
+

beacon_chain/deposit_contract.nim

@@ -75,12 +75,12 @@ proc main() {.async.} =
   case cfg.cmd
   of StartUpCommand.deploy:
     let receipt = await web3.deployContract(contractCode)
-    echo "0x", receipt.contractAddress.get, ";", receipt.blockHash
+    echo receipt.contractAddress.get, ";", receipt.blockHash
   of StartUpCommand.drain:
     let sender = web3.contractSender(Deposit, Address.fromHex(cfg.contractAddress))
     discard await sender.drain().send(gasPrice = 1)
 
   of StartUpCommand.sendEth:
-    echo "0x", await sendEth(web3, cfg.toAddress, cfg.valueEth.parseInt)
+    echo await sendEth(web3, cfg.toAddress, cfg.valueEth.parseInt)
 
 when isMainModule: waitFor main()

beacon_chain/keystore_management.nim

@@ -1,7 +1,7 @@
 import
   os, strutils, terminal,
   chronicles, chronos, blscurve, nimcrypto, json_serialization, serialization,
-  web3, stint, eth/keys, confutils,
+  web3, stint, eth/common/eth_types, eth/keys, confutils,
   spec/[datatypes, digest, crypto, keystore], conf, ssz/merkleization, merkle_minimal
 
 export
@@ -15,7 +15,8 @@ const
   depositFileName* = "deposit.json"
 
 type
- DelayGenerator* = proc(): chronos.Duration {.closure, gcsafe.}
+  Eth1Address* = eth_types.EthAddress
+  DelayGenerator* = proc(): chronos.Duration {.closure, gcsafe.}
 
 {.push raises: [Defect].}
 
@@ -165,7 +166,8 @@ proc loadDeposits*(depositsDir: string): seq[Deposit] =
 # TODO: async functions should note take `seq` inputs because
 #       this leads to full copies.
 proc sendDeposits*(deposits: seq[Deposit],
-                   web3Url, depositContractAddress, privateKey: string,
+                   web3Url, privateKey: string,
+                   depositContractAddress: Eth1Address,
                    delayGenerator: DelayGenerator = nil) {.async.} =
   var web3 = await newWeb3(web3Url)
   if privateKey.len != 0:
@@ -177,8 +179,8 @@ proc sendDeposits*(deposits: seq[Deposit],
       return
     web3.defaultAccount = accounts[0]
 
-  let contractAddress = Address.fromHex(depositContractAddress)
-  let depositContract = web3.contractSender(DepositContract, contractAddress)
+  let depositContract = web3.contractSender(DepositContract,
+                                            Address depositContractAddress)
 
   for i, dp in deposits:
     let status = await depositContract.deposit(
@@ -202,7 +204,7 @@ proc sendDeposits*(config: BeaconNodeConf,
   await sendDeposits(
     deposits,
     config.web3Url,
-    config.depositContractAddress,
     config.depositPrivateKey,
+    config.depositContractAddress.get,
     delayGenerator)
 

beacon_chain/mainchain_monitor.nim

@@ -1,13 +1,14 @@
 import
   deques, tables, hashes, options, strformat,
-  chronos, web3, web3/ethtypes, json, chronicles, eth/async_utils,
+  chronos, web3, web3/ethtypes as web3Types, json, chronicles,
+  eth/common/eth_types, eth/async_utils,
   spec/[datatypes, digest, crypto, beaconstate, helpers],
   merkle_minimal
 
 from times import epochTime
 
 export
-  ethtypes
+  web3Types
 
 contract(DepositContract):
   proc deposit(pubkey: Bytes48,
@@ -28,8 +29,10 @@ contract(DepositContract):
 # Exceptions being reported from Chronos's asyncfutures2.
 
 type
+  Eth1Address = eth_types.EthAddress
   Eth1BlockNumber* = uint64
   Eth1BlockTimestamp* = uint64
+  Eth1BlockHeader = web3Types.BlockHeader
 
   Eth1Block* = ref object
     number*: Eth1BlockNumber
@@ -56,7 +59,7 @@ type
 
     eth1Chain: Eth1Chain
 
-    depositQueue: AsyncQueue[BlockHeader]
+    depositQueue: AsyncQueue[Eth1BlockHeader]
     runFut: Future[void]
 
   DataProvider* = object of RootObj
@@ -458,11 +461,11 @@ template getBlockProposalData*(m: MainchainMonitor, state: BeaconState): untyped
 
 proc init*(T: type MainchainMonitor,
            dataProviderFactory: DataProviderFactory,
-           depositContractAddress: string,
+           depositContractAddress: Eth1Address,
            startPosition: Eth1Data): T =
-  T(depositQueue: newAsyncQueue[BlockHeader](),
+  T(depositQueue: newAsyncQueue[Eth1BlockHeader](),
     dataProviderFactory: dataProviderFactory,
-    depositContractAddress: Address.fromHex(depositContractAddress),
+    depositContractAddress: Address depositContractAddress,
     eth1Chain: Eth1Chain(knownStart: startPosition))
 
 proc isCandidateForGenesis(timeNow: float, blk: Eth1Block): bool =
@@ -742,7 +745,7 @@ proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
       contract = $m.depositContractAddress,
       url = m.dataProviderFactory.desc
 
-    await dataProvider.onBlockHeaders do (blk: BlockHeader)
+    await dataProvider.onBlockHeaders do (blk: Eth1BlockHeader)
                                          {.raises: [Defect], gcsafe}:
       try:
         m.depositQueue.addLastNoWait(blk)

beacon_chain/network_metadata.nim

@@ -0,0 +1,70 @@
+import
+  os, strutils,
+  stew/byteutils, nimcrypto/hash,
+  eth/common/[eth_types, eth_types_json_serialization]
+
+# ATTENTION! This file will produce a large C file, because we are inlining
+# genesis states as C literals in the generated code (and blobs in the final
+# binary). It makes sense to keep the file small and separated from the rest
+# of the module in order go gain maximum efficiency in incremental compilation.
+#
+# TODO:
+# We can compress the embedded states with snappy before embedding them here.
+
+{.push raises: [Defect].}
+
+export
+  eth_types_json_serialization
+
+type
+  Eth1Address* = eth_types.EthAddress
+  Eth1BlockHash* = eth_types.Hash256
+
+  Eth1Network* = enum
+    mainnet
+    rinkeby
+    goerli
+
+  Eth2Network* = enum
+    customEth2Network
+    altona
+
+  Eth2NetworkMetadata* = object
+    eth1Network*: Eth1Network
+
+    # Parsing `enr.Records` is still not possible at compile-time
+    bootstrapNodes*: seq[string]
+
+    depositContractAddress*: Eth1Address
+    depositContractDeployedAt*: Eth1BlockHash
+
+    # Please note that we are using `string` here because SSZ.decode
+    # is not currently usable at compile time and we want to load the
+    # network metadata into a constant.
+    #
+    # We could have also used `seq[byte]`, but this results in a lot
+    # more generated code that slows down compilation. The impact on
+    # compilation times of embedding the genesis as a string is roughly
+    # 0.1s on my machine (you can test this by choosing an invalid name
+    # for the genesis file below).
+    #
+    # `genesisData` will have `len == 0` for networks with a still
+    # unknown genesis state.
+    genesisData*: string
+
+proc loadEth2NetworkMetadata*(path: string): Eth2NetworkMetadata
+                             {.raises: [CatchableError, Defect].} =
+  let
+    genesisPath = path / "genesis.ssz"
+
+  Eth2NetworkMetadata(
+    eth1Network: goerli,
+    bootstrapNodes: readFile(path / "bootstrap_nodes.txt").split("\n"),
+    depositContractAddress: Eth1Address.fromHex readFile(path / "deposit_contract.txt").strip,
+    depositContractDeployedAt: Eth1BlockHash.fromHex readFile(path / "deposit_contract_block.txt").strip,
+    genesisData: if fileExists(genesisPath): readFile(genesisPath) else: "")
+
+const
+  altonaMetadata* = loadEth2NetworkMetadata(
+    currentSourcePath.parentDir / ".." / "vendor" / "eth2-testnets" / "shared" / "altona")
+

vendor/eth2-testnets

@@ -0,0 +1 @@
+Subproject commit 61d94737812fc40438b70ea49a6a87d17c2f3d4c

vendor/nim-confutils

@@ -1 +1 @@
-Subproject commit a76faa5eec2454309753bbe99a68fb3cc25782f1
+Subproject commit f9415621f87287524d26aa99a94e2613b237cc3c

vendor/nim-stew

@@ -1 +1 @@
-Subproject commit 152eb1b58cd618a175dc2ae6fba39e620115c356
+Subproject commit 61d5cfc37677f2b434d43c06d06695b00e56613b

vendor/nim-web3

@@ -1 +1 @@
-Subproject commit a75519fe1264ea861fb65eb2ffec1d6566ebd033
+Subproject commit 227927ddc80e7d6bd432f70fe8c803286b46e770