add premix support tools

premix/downloader.nim

@@ -0,0 +1,115 @@
+import
+  json_rpc/[rpcclient], json, parser, httputils, strutils,
+  eth_common, chronicles, ../nimbus/[utils], rlp, nimcrypto
+
+logScope:
+  topics = "downloader"
+
+type
+  Block* = object
+    header*: BlockHeader
+    body*: BlockBody
+    traces*: JsonNode
+    receipts*: seq[Receipt]
+    jsonData*: JsonNode
+
+  DownloadFlags* = enum
+    DownloadReceipts
+    DownloadTxTrace
+
+proc request*(methodName: string, params: JsonNode): JsonNode =
+  var client = newRpcHttpClient()
+  client.httpMethod(MethodPost)
+  waitFor client.connect("localhost", Port(8545))
+  var r = waitFor client.call(methodName, params)
+  if r.error:
+    result = newJNull()
+  else:
+    result = r.result
+  client.transp.close()
+
+proc requestBlockBody(n: JsonNode, blockNumber: BlockNumber): BlockBody =
+  let txs = n["transactions"]
+  if txs.len > 0:
+    result.transactions = newSeqOfCap[Transaction](txs.len)
+    for tx in txs:
+      let txn = parseTransaction(tx)
+      result.transactions.add txn
+
+  let uncles = n["uncles"]
+  if uncles.len > 0:
+    result.uncles = newSeqOfCap[BlockHeader](uncles.len)
+    let blockNumber = blockNumber.prefixHex
+    for i in 0 ..< uncles.len:
+      let idx = i.prefixHex
+      let uncle = request("eth_getUncleByBlockNumberAndIndex", %[%blockNumber, %idx])
+      if uncle.kind == JNull:
+        error "requested uncle not available", blockNumber=blockNumber, uncleIdx=i
+        raise newException(ValueError, "Error when retrieving block uncles")
+      result.uncles.add parseBlockHeader(uncle)
+
+proc requestReceipts(n: JsonNode): seq[Receipt] =
+  let txs = n["transactions"]
+  if txs.len > 0:
+    result = newSeqOfCap[Receipt](txs.len)
+    for tx in txs:
+      let txHash = tx["hash"]
+      let rec = request("eth_getTransactionReceipt", %[txHash])
+      if rec.kind == JNull:
+        error "requested receipt not available", txHash=txHash
+        raise newException(ValueError, "Error when retrieving block receipts")
+      result.add parseReceipt(rec)
+
+proc requestTxTraces(n: JsonNode): JsonNode =
+  let txs = n["transactions"]
+  if txs.len > 0:
+    result = newJArray()
+    for tx in txs:
+      let txHash = tx["hash"]
+      let txTrace = request("debug_traceTransaction", %[txHash])
+      if txTrace.kind == JNull:
+        error "requested trace not available", txHash=txHash
+        raise newException(ValueError, "Error when retrieving transaction trace")
+      result.add txTrace
+
+proc requestBlock*(blockNumber: BlockNumber, flags: set[DownloadFlags] = {}): Block =
+  var header = request("eth_getBlockByNumber", %[%blockNumber.prefixHex, %true])
+  if header.kind == JNull:
+    error "requested block not available", blockNumber=blockNumber
+    raise newException(ValueError, "Error when retrieving block header")
+
+  result.jsonData   = header
+  result.header     = parseBlockHeader(header)
+  result.body       = requestBlockBody(header, blockNumber)
+
+  if DownloadTxTrace in flags:
+    result.traces     = requestTxTraces(header)
+
+  let
+    txRoot        = calcTxRoot(result.body.transactions).prefixHex
+    txRootOK      = result.header.txRoot.prefixHex
+    ommersHash    = rlpHash(result.body.uncles).prefixHex
+    ommersHashOK  = result.header.ommersHash.prefixHex
+    headerHash    = rlpHash(result.header).prefixHex
+    headerHashOK  = header["hash"].getStr().toLowerAscii
+
+  if DownloadReceipts in flags:
+    result.receipts   = requestReceipts(header)
+    let
+      receiptRoot   = calcReceiptRoot(result.receipts).prefixHex
+      receiptRootOK = result.header.receiptRoot.prefixHex
+    if receiptRoot != receiptRootOK:
+      debug "wrong receipt root", receiptRoot, receiptRootOK, blockNumber
+      raise newException(ValueError, "Error when validating receipt root")
+
+  if txRoot != txRootOK:
+    debug "wrong tx root", txRoot, txRootOK, blockNumber
+    raise newException(ValueError, "Error when validating tx root")
+
+  if ommersHash != ommersHashOK:
+    debug "wrong ommers hash", ommersHash, ommersHashOK, blockNumber
+    raise newException(ValueError, "Error when validating ommers hash")
+
+  if headerHash != headerHashOK:
+    debug "wrong header hash", headerHash, headerHashOK, blockNumber
+    raise newException(ValueError, "Error when validating block header hash")

premix/parser.nim

@@ -0,0 +1,129 @@
+import
+  json, strutils, times, options, os,
+  rlp, httputils, nimcrypto, chronicles,
+  stint, eth_common, byteutils
+
+import
+  ../nimbus/[transaction, rpc/hexstrings]
+
+func hexToInt(s: string, T: typedesc[SomeInteger]): T =
+  var i = 0
+  if s[i] == '0' and (s[i+1] in {'x', 'X'}): inc(i, 2)
+  if s.len - i > sizeof(T) * 2:
+    raise newException(ValueError, "input hex too big for destination int")
+  while i < s.len:
+    result = result shl 4 or readHexChar(s[i]).T
+    inc(i)
+
+proc prefixHex*(x: Hash256): string =
+  "0x" & toLowerAscii($x)
+
+proc prefixHex*(x: int64 | uint64 | byte | int): string =
+  encodeQuantity(x.uint64).toLowerAscii
+
+proc prefixHex*(x: openArray[byte]): string =
+  "0x" & toHex(x, true)
+
+proc prefixHex*(x: UInt256): string =
+  "0x" & stint.toHex(x)
+
+proc prefixHex*(x: string): string =
+  "0x" & toLowerAscii(x)
+
+type
+  SomeData = EthAddress | BloomFilter | BlockNonce
+
+proc fromJson(n: JsonNode, name: string, x: var SomeData) =
+  hexToByteArray(n[name].getStr(), x)
+  assert(x.prefixHex == toLowerAscii(n[name].getStr()))
+
+proc fromJson(n: JsonNode, name: string, x: var Hash256) =
+  hexToByteArray(n[name].getStr(), x.data)
+  assert(x.prefixHex == toLowerAscii(n[name].getStr()))
+
+proc fromJson(n: JsonNode, name: string, x: var Blob) =
+  x = hexToSeqByte(n[name].getStr())
+  assert(x.prefixHex == toLowerAscii(n[name].getStr()))
+
+proc fromJson(n: JsonNode, name: string, x: var UInt256) =
+  x = UInt256.fromHex(n[name].getStr())
+  assert(x.prefixHex == toLowerAscii(n[name].getStr()))
+
+proc fromJson(n: JsonNode, name: string, x: var SomeInteger) =
+  x = hexToInt(n[name].getStr(), type(x))
+  assert(x.prefixHex == toLowerAscii(n[name].getStr()))
+
+proc fromJson(n: JsonNode, name: string, x: var EthTime) =
+  x = initTime(hexToInt(n[name].getStr(), int64), 0)
+  assert(x.toUnix.prefixHex == toLowerAscii(n[name].getStr()))
+
+proc parseBlockHeader*(n: JsonNode): BlockHeader =
+  n.fromJson "parentHash", result.parentHash
+  n.fromJson "sha3Uncles", result.ommersHash
+  n.fromJson "miner", result.coinbase
+  n.fromJson "stateRoot", result.stateRoot
+  n.fromJson "transactionsRoot", result.txRoot
+  n.fromJson "receiptsRoot", result.receiptRoot
+  n.fromJson "logsBloom", result.bloom
+  n.fromJson "difficulty", result.difficulty
+  n.fromJson "number", result.blockNumber
+  n.fromJson "gasLimit", result.gasLimit
+  n.fromJson "gasUsed", result.gasUsed
+  n.fromJson "timestamp", result.timestamp
+  n.fromJson "extraData", result.extraData
+  n.fromJson "mixHash", result.mixDigest
+  n.fromJson "nonce", result.nonce
+
+proc parseTransaction*(n: JsonNode): Transaction =
+  n.fromJson "nonce", result.accountNonce
+  n.fromJson "gasPrice", result.gasPrice
+  n.fromJson "gas", result.gasLimit
+
+  result.isContractCreation = n["to"].kind == JNull
+  if not result.isContractCreation:
+    n.fromJson "to", result.to
+
+  n.fromJson "value", result.value
+  n.fromJson "input", result.payload
+  n.fromJson "v", result.V
+  n.fromJson "r", result.R
+  n.fromJson "s", result.S
+
+  var sender = result.getSender()
+  assert sender.prefixHex == n["from"].getStr()
+  assert n["hash"].getStr() == result.rlpHash().prefixHex
+
+proc parseLog(n: JsonNode): Log =
+  n.fromJson "address", result.address
+  n.fromJson "data", result.data
+  let topics = n["topics"]
+  result.topics = newSeqOfCap[Topic](n.len)
+  var topicHash: Topic
+  for tp in topics:
+    hexToByteArray(tp.getStr(), topicHash)
+    result.topics.add topicHash
+
+proc parseLogs(n: JsonNode): seq[Log] =
+  if n.len > 0:
+    result = newSeqOfCap[Log](n.len)
+    for log in n:
+      result.add parseLog(log)
+  else:
+    result = @[]
+
+proc parseReceipt*(n: JsonNode): Receipt =
+  if n.hasKey("root"):
+    var hash: Hash256
+    n.fromJson "root", hash
+    result.stateRootOrStatus = hashOrStatus(hash)
+  else:
+    var status: int
+    n.fromJson "status", status
+    result.stateRootOrStatus = hashOrStatus(status == 1)
+
+  n.fromJson "cumulativeGasUsed", result.cumulativeGasUsed
+  n.fromJson "logsBloom", result.bloom
+  result.logs = parseLogs(n["logs"])
+
+proc headerHash*(n: JsonNode): Hash256 =
+  n.fromJson "hash", result

premix/persist.nim

@@ -0,0 +1,92 @@
+# use this module to quickly populate db with data from geth/parity
+
+import
+  eth_common, stint, byteutils, nimcrypto,
+  chronicles, rlp, downloader
+
+import
+  eth_trie/[hexary, db, defs],
+  ../nimbus/db/[storage_types, db_chain, select_backend],
+  ../nimbus/[genesis, utils, config],
+  ../nimbus/p2p/chain
+
+# TODO: move this one and the one in nimbus.nim to db_chain.nim
+proc initializeEmptyDb(db: BaseChainDB) =
+  echo "Writing genesis to DB"
+  let networkId = getConfiguration().net.networkId.toPublicNetwork()
+  if networkId == CustomNet:
+    raise newException(Exception, "Custom genesis not implemented")
+  else:
+    defaultGenesisBlockForNetwork(networkId).commit(db)
+
+const
+  manualCommit = nimbus_db_backend == "lmdb"
+
+template persistToDb(db: ChainDB, body: untyped) =
+  when manualCommit:
+    if not db.txBegin(): assert(false)
+  body
+  when manualCommit:
+    if not db.txCommit(): assert(false)
+
+proc main() =
+  # 97 block with uncles
+  # 46147 block with first transaction
+  # 46400 block with transaction
+  # 46402 block with first contract: failed
+  # 47205 block with first success contract
+  # 48712 block with 5 transactions
+  # 48915 block with contract
+  # 49018 first problematic block
+  # 49439 first block with contract call
+  # 52029 first block with receipts logs
+  # 66407 failed transaction
+
+  var conf = getConfiguration()
+  let db = newChainDb(conf.dataDir)
+  let trieDB = trieDB db
+  let chainDB = newBaseChainDB(trieDB, false)
+
+  # move head to block number ...
+  #var parentBlock = requestBlock(49438.u256)
+  #chainDB.setHead(parentBlock.header)
+
+  if canonicalHeadHashKey().toOpenArray notin trieDB:
+    persistToDb(db):
+      initializeEmptyDb(chainDB)
+    assert(canonicalHeadHashKey().toOpenArray in trieDB)
+
+  var head = chainDB.getCanonicalHead()
+  var blockNumber = head.blockNumber + 1
+  var chain = newChain(chainDB)
+
+  const
+    numBlocksToCommit = 128
+    numBlocksToDownload = 20000
+
+  var headers = newSeqOfCap[BlockHeader](numBlocksToCommit)
+  var bodies  = newSeqOfCap[BlockBody](numBlocksToCommit)
+  var one     = 1.u256
+
+  var numBlocks = 0
+  for _ in 0 ..< numBlocksToDownload:
+    info "REQUEST HEADER", blockNumber=blockNumber
+    var thisBlock = requestBlock(blockNumber)
+
+    headers.add thisBlock.header
+    bodies.add thisBlock.body
+    inc numBlocks
+    blockNumber += one
+
+    if numBlocks == numBlocksToCommit:
+      persistToDb(db):
+        discard chain.persistBlocks(headers, bodies)
+      numBlocks = 0
+      headers.setLen(0)
+      bodies.setLen(0)
+
+  if numBlocks > 0:
+    persistToDb(db):
+      discard chain.persistBlocks(headers, bodies)
+
+main()