implement '--import': import rlp encoded block(s), validate, write to db and quit

nimbus/conf_utils.nim

@@ -0,0 +1,46 @@
+# Nimbus
+# Copyright (c) 2021 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+import
+  std/[terminal, os],
+  chronicles, eth/trie/db, eth/[common, rlp], stew/[io2, byteutils],
+  ./config, ./genesis, ./p2p/chain,
+  ./db/[db_chain, select_backend, storage_types]
+
+type
+  # trick the rlp decoder
+  # so we can separate the body and header
+  EthHeader = object
+    header: BlockHeader
+
+proc importRlpBlock*(importFile: string, chainDB: BasechainDB) =
+  let res = io2.readAllBytes(importFile)
+  if res.isErr:
+    error "failed to import", fileName = importFile
+    quit(QuitFailure)
+
+  var chain = newChain(chainDB)
+  # the encoded rlp can contains one or more blocks
+  var rlp = rlpFromBytes(res.get)
+
+  # separate the header and the body
+  # TODO: probably we need to put it in one struct
+  var headers: seq[BlockHeader]
+  var bodies : seq[BlockBody]
+
+  while true:
+    headers.add rlp.read(EthHeader).header
+    bodies.add rlp.readRecordType(BlockBody, false)
+    if not rlp.hasData:
+      break
+
+  let valid = chain.persistBlocks(headers, bodies)
+  if valid == ValidationResult.Error:
+    error "failed to import rlp encoded blocks", fileName = importFile
+    quit(QuitFailure)

nimbus/config.nim

@@ -165,6 +165,7 @@ type
     # Ref is used so that it can be shared between components
     rng*: ref BrHmacDrbgContext
     accounts*: Table[EthAddress, NimbusAccount]
+    importFile*: string
 
   CustomGenesisConfig = object
     chainId*: ChainId
@@ -200,6 +201,9 @@ var nimbusConfig {.threadvar.}: NimbusConfiguration
 
 proc getConfiguration*(): NimbusConfiguration {.gcsafe.}
 
+proc `$`*(c: ChainId): string =
+  $(c.int)
+
 proc toFork*(c: ChainConfig, number: BlockNumber): Fork =
   if number >= c.berlinBlock: FkBerlin
   elif number >= c.istanbulBlock: FkIstanbul
@@ -234,6 +238,7 @@ proc privateChainConfig*(): ChainConfig =
 proc publicChainConfig*(id: PublicNetwork): ChainConfig =
   # For some public networks, NetworkId and ChainId value are identical
   # but that is not always the case
+
   result = case id
   of MainNet:
     ChainConfig(
@@ -307,8 +312,6 @@ proc publicChainConfig*(id: PublicNetwork): ChainConfig =
     doAssert(false, "No chain config for " & $id)
     ChainConfig()
 
-  result.chainId = ChainId(id)
-
 proc processCustomGenesisConfig(customGenesis: JsonNode): ConfigStatus =
   ## Parses Custom Genesis Block config options when customnetwork option provided
 
@@ -329,7 +332,7 @@ proc processCustomGenesisConfig(customGenesis: JsonNode): ConfigStatus =
     when T is string:
       c[field].getStr()
     elif T is Uint256:
-      parseHexInt(c[field].getStr()).u256
+      hexToUint[256](c[field].getStr())
     elif T is bool:
       c[field].getBool()
     elif T is Hash256:
@@ -346,7 +349,8 @@ proc processCustomGenesisConfig(customGenesis: JsonNode): ConfigStatus =
       hexToSeqByte(c[field].getStr())
     elif T is int64:
       fromHex[int64](c[field].getStr())
-
+    elif T is ChainId:
+      c[field].getInt().T
 
   template validateConfigValue(chainDetails, field, jtype, T: untyped, checkError: static[bool] = true) =
     let fieldName = field.astToStr()
@@ -433,6 +437,24 @@ proc processCustomGenesisConfig(customGenesis: JsonNode): ConfigStatus =
     timestamp:        timestamp
   )
 
+proc customGenesis(key, value: string): ConfigStatus =
+  if value == "":
+    error "No genesis block config provided for custom network", network=key
+    result = ErrorParseOption
+  else:
+    try:
+      result = processCustomGenesisConfig(parseFile(value))
+    except IOError:
+      error "Genesis block config file not found", invalidFileName=value
+      result = ErrorParseOption
+    except JsonParsingError:
+      error "Invalid genesis block config file format", invalidFileName=value
+      result = ErrorIncorrectOption
+    except:
+      var msg = getCurrentExceptionMsg()
+      error "Error loading genesis block config file", invalidFileName=msg
+      result = Error
+
 proc processList(v: string, o: var seq[string]) =
   ## Process comma-separated list of strings.
   if len(v) > 0:
@@ -537,30 +559,6 @@ proc processPrivateKey(v: string, o: var PrivateKey): ConfigStatus =
 
   result = ErrorParseOption
 
-# proc processHexBytes(v: string, o: var seq[byte]): ConfigStatus =
-#   ## Convert hexadecimal string to seq[byte].
-#   try:
-#     o = fromHex(v)
-#     result = Success
-#   except CatchableError:
-#     result = ErrorParseOption
-
-# proc processHexString(v: string, o: var string): ConfigStatus =
-#   ## Convert hexadecimal string to string.
-#   try:
-#     o = parseHexStr(v)
-#     result = Success
-#   except CatchableError:
-#     result = ErrorParseOption
-
-# proc processJson(v: string, o: var JsonNode): ConfigStatus =
-#   ## Convert string to JSON.
-#   try:
-#     o = parseJson(v)
-#     result = Success
-#   except CatchableError:
-#     result = ErrorParseOption
-
 proc processPruneList(v: string, flags: var PruneMode): ConfigStatus =
   var list = newSeq[string]()
   processList(v, list)
@@ -583,6 +581,8 @@ proc processEthArguments(key, value: string): ConfigStatus =
     config.dataDir = value
   of "prune":
     result = processPruneList(value, config.prune)
+  of "import":
+    config.importFile = value
   else:
     result = EmptyOption
 
@@ -676,22 +676,8 @@ proc processNetArguments(key, value: string): ConfigStatus =
   elif skey == "kovan":
     config.net.setNetwork(KovanNet)
   elif skey == "customnetwork":
-    if value == "":
+    result = customGenesis(key, value)
-      error "No genesis block config provided for custom network", network=key
+    config.net.networkId = NetworkId(CustomNet)
-      result = ErrorParseOption
-    else:
-      try:
-        result = processCustomGenesisConfig(parseFile(value))
-      except IOError:
-        error "Genesis block config file not found", invalidFileName=value
-        result = ErrorParseOption
-      except JsonParsingError:
-        error "Invalid genesis block config file format", invalidFileName=value
-        result = ErrorIncorrectOption
-      except:
-        var msg = getCurrentExceptionMsg()
-        error "Error loading genesis block config file", invalidFileName=msg
-        result = Error
   elif skey == "networkid":
     var res = 0
     result = processInteger(value, res)
@@ -924,6 +910,7 @@ ETHEREUM OPTIONS:
   --keystore:<value>      Directory for the keystore (default = inside the datadir)
   --datadir:<value>       Base directory for all blockchain-related data
   --prune:<value>         Blockchain prune mode(full or archive)
+  --import:<path>         import rlp encoded block(s), validate, write to db and quit
 
 NETWORKING OPTIONS:
   --bootnodes:<value>     Comma separated enode URLs for P2P discovery bootstrap (set v4+v5 instead for light servers)
@@ -944,7 +931,7 @@ NETWORKING OPTIONS:
   --rinkeby               Use Ethereum Rinkeby Test Network
   --ident:<value>         Client identifier (default is '$1')
   --protocols:<value>     Enable specific set of protocols (default: $4)
-  --customnetwork         Use custom genesis block for private Ethereum Network (as /path/to/genesis.json)
+  --customnetwork:<path>  Use custom genesis block for private Ethereum Network (as /path/to/genesis.json)
 
 WHISPER OPTIONS:
   --shh-maxsize:<value>   Max message size accepted (default: $5)

nimbus/genesis.nim

@@ -1,5 +1,5 @@
 import
-  tables, json, times,
+  tables, json, times, strutils,
   eth/[common, rlp, trie], stint, stew/[byteutils],
   chronicles, eth/trie/db,
   db/[db_chain, state_db], genesis_alloc, config, constants
@@ -34,9 +34,16 @@ func decodePrealloc(data: seq[byte]): GenesisAlloc =
 
 proc customNetPrealloc(genesisBlock: JsonNode): GenesisAlloc =
   result = newTable[EthAddress, GenesisAccount]()
-  for address, balance in genesisBlock.pairs():
+  for address, account in genesisBlock.pairs():
-    let balance = fromHex(UInt256,balance["balance"].getStr())
+    var acc = GenesisAccount(
-    result[parseAddress(address)] = GenesisAccount(balance: balance)
+      balance: fromHex(UInt256, account["balance"].getStr),
+      code: hexToSeqByte(account["code"].getStr),
+      nonce: parseHexInt(account["nonce"].getStr).AccountNonce
+    )
+    let storage = account["storage"]
+    for k, v in storage:
+      acc.storage[fromHex(UInt256, k)] = fromHex(UInt256, v.getStr)
+    result[parseAddress(address)] = acc
 
 proc defaultGenesisBlockForNetwork*(id: PublicNetwork): Genesis =
   result = case id

nimbus/nimbus.nim

@@ -15,7 +15,7 @@ import
   eth/p2p/rlpx_protocols/[eth_protocol, les_protocol, whisper_protocol],
   eth/p2p/blockchain_sync, eth/net/nat, eth/p2p/peer_pool,
   config, genesis, rpc/[common, p2p, debug, whisper, key_storage], p2p/chain,
-  eth/trie/db, metrics, metrics/chronicles_support, utils
+  eth/trie/db, metrics, metrics/chronicles_support, utils, ./conf_utils
 
 ## TODO:
 ## * No IPv6 support
@@ -36,10 +36,6 @@ type
 
 proc start(nimbus: NimbusNode) =
   var conf = getConfiguration()
-  let res = conf.loadKeystoreFiles()
-  if res.isErr:
-    echo res.error()
-    quit(QuitFailure)
 
   ## logging
   setLogLevel(conf.debug.logLevel)
@@ -47,6 +43,26 @@ proc start(nimbus: NimbusNode) =
     defaultChroniclesStream.output.outFile = nil # to avoid closing stdout
     discard defaultChroniclesStream.output.open(conf.debug.logFile, fmAppend)
 
+  createDir(conf.dataDir)
+  let trieDB = trieDB newChainDb(conf.dataDir)
+  var chainDB = newBaseChainDB(trieDB,
+    conf.prune == PruneMode.Full,
+    conf.net.networkId.toPublicNetwork())
+  chainDB.populateProgress()
+
+  if canonicalHeadHashKey().toOpenArray notin trieDB:
+    initializeEmptyDb(chainDb)
+    doAssert(canonicalHeadHashKey().toOpenArray in trieDB)
+    
+  if conf.importFile.len > 0:
+    importRlpBlock(conf.importFile, chainDB)
+    quit(QuitSuccess)
+
+  let res = conf.loadKeystoreFiles()
+  if res.isErr:
+    echo res.error()
+    quit(QuitFailure)
+
   # metrics logging
   if conf.debug.logMetrics:
     proc logMetrics(udata: pointer) {.closure, gcsafe.} =
@@ -81,18 +97,6 @@ proc start(nimbus: NimbusNode) =
       if extPorts.isSome:
         (address.tcpPort, address.udpPort) = extPorts.get()
 
-  createDir(conf.dataDir)
-  let trieDB = trieDB newChainDb(conf.dataDir)
-  var chainDB = newBaseChainDB(trieDB,
-    conf.prune == PruneMode.Full,
-    conf.net.networkId.toPublicNetwork())
-
-  chainDB.populateProgress()
-
-  if canonicalHeadHashKey().toOpenArray notin trieDB:
-    initializeEmptyDb(chainDb)
-    doAssert(canonicalHeadHashKey().toOpenArray in trieDB)
-
   nimbus.ethNode = newEthereumNode(keypair, address, conf.net.networkId,
                                    nil, nimbusClientId,
                                    addAllCapabilities = false,