augment debug tool: fix, compile, update report page, iterate

premix/assets/js/index.js

@@ -252,6 +252,7 @@ function headerRenderer(nimbus, geth) {
   }
 
   let container = $('#headerContainer').empty();
+  $('#headerTitle').text('Block #' + parseInt(geth.block.number, 16));
 
   let ncs = deepCopy(nimbus.stateDump.after);
   let gcs = deepCopy(geth.accounts);

premix/debug.nim

@@ -1,7 +1,7 @@
 import
   json, os, stint, eth_trie/db, byteutils, eth_common,
   ../nimbus/db/[db_chain], chronicles, ../nimbus/vm_state,
-  ../nimbus/p2p/executor
+  ../nimbus/p2p/executor, premixcore, prestate, ../nimbus/tracer
 
 proc prepareBlockEnv(node: JsonNode, memoryDB: TrieDatabaseRef) =
   let state = node["state"]
@@ -11,7 +11,7 @@ proc prepareBlockEnv(node: JsonNode, memoryDB: TrieDatabaseRef) =
     let value = hexToSeqByte(v.getStr())
     memoryDB.put(key, value)
 
-proc executeBlock(memoryDB: TrieDatabaseRef, blockNumber: Uint256) =
+proc executeBlock(blockEnv: JsonNode, memoryDB: TrieDatabaseRef, blockNumber: Uint256) =
   let
     parentNumber = blockNumber - 1
     chainDB = newBaseChainDB(memoryDB, false)
@@ -28,6 +28,21 @@ proc executeBlock(memoryDB: TrieDatabaseRef, blockNumber: Uint256) =
   else:
     info "block validation success", validationResult, blockNumber
 
+  dumpDebuggingMetaData(chainDB, header, body, vmState.receipts, false)
+  let
+    fileName = "debug" & $blockNumber & ".json"
+    nimbus   = json.parseFile(fileName)
+    geth     = blockEnv["geth"]
+
+  processNimbusData(nimbus)
+
+  # premix data goes to report page
+  generatePremixData(nimbus, geth)
+
+  # prestate data goes to debug tool and contains data
+  # needed to execute single block
+  generatePrestate(nimbus, geth, blockNumber, parent, header, body)
+
 proc main() =
   if paramCount() == 0:
     echo "usage: debug blockxxx.json"
@@ -39,6 +54,6 @@ proc main() =
     blockNumber = UInt256.fromHex(blockEnv["blockNumber"].getStr())
 
   prepareBlockEnv(blockEnv, memoryDB)
-  executeBlock(memoryDB, blockNumber)
+  executeBlock(blockEnv, memoryDB, blockNumber)
 
 main()

premix/index.html

@@ -108,7 +108,7 @@
         <!-- Header Page -->
         <div class="uk-section-small uk-section-default">
           <div class="uk-container uk-container-medium">
-            <h3>Header Post State Accounts</h3>
+            <h3>Header Post State Accounts <span id="headerTitle" class="uk-text-primary uk-text-small">Block #</span></h3>
             <div id="headerContainer">
             </div>
           </div>

premix/premix.nim

@@ -2,22 +2,7 @@ import
   json, downloader, stint, strutils, os,
   ../nimbus/tracer, chronicles, prestate,
   js_tracer, eth_common, byteutils, parser,
-  nimcrypto
-
-proc fakeAlloc(n: JsonNode) =
-  const
-    chunk = repeat('0', 64)
-
-  for i in 1 ..< n.len:
-    if not n[i].hasKey("memory"): return
-    let
-      prevMem = n[i-1]["memory"]
-      currMem = n[i]["memory"]
-
-    if currMem.len > prevMem.len:
-      let diff = currMem.len - prevMem.len
-      for _ in 0 ..< diff:
-        prevMem.add %chunk
+  nimcrypto, premixcore
 
 proc jsonTracer(tracer: string): JsonNode =
   result = %{ "tracer": %tracer }
@@ -99,32 +84,6 @@ proc requestBlockState(postState: JsonNode, thisBlock: Block) =
       for x in trace["storageProof"]:
         account["storage"][x["key"].getStr] = padding(x["value"].getStr())
 
-proc copyAccount(acc: JsonNode): JsonNode =
-  result = newJObject()
-  if acc.hasKey("name"):
-    result["name"] = newJString(acc["name"].getStr)
-  result["balance"] = newJString(acc["balance"].getStr)
-  result["nonce"] = newJString(acc["nonce"].getStr)
-  result["code"] = newJString(acc["code"].getStr)
-  var storage = newJObject()
-  for k, v in acc["storage"]:
-    storage[k] = newJString(v.getStr)
-  result["storage"] = storage
-  result["storageRoot"] = newJString(acc["storageRoot"].getStr)
-  result["codeHash"] = newJString(acc["codeHash"].getStr)
-
-proc updateAccount(a, b: JsonNode) =
-  if b.hasKey("name"):
-    a["name"] = newJString(b["name"].getStr)
-  a["balance"] = newJString(b["balance"].getStr)
-  a["nonce"] = newJString(b["nonce"].getStr)
-  a["code"] = newJString(b["code"].getStr)
-  var storage = a["storage"]
-  for k, v in b["storage"]:
-    storage[k] = newJString(v.getStr)
-  a["storageRoot"] = newJString(b["storageRoot"].getStr)
-  a["codeHash"] = newJString(b["codeHash"].getStr)
-
 proc processPostState(postState: JsonNode): JsonNode =
   var accounts = newJObject()
 
@@ -137,17 +96,6 @@ proc processPostState(postState: JsonNode): JsonNode =
 
   result = accounts
 
-proc removePostStateDup(nimbus: JsonNode) =
-  let postState = nimbus["stateDump"]["after"]
-  var accounts = newJObject()
-  for acc in postState:
-    let address = acc["address"].getStr
-    if accounts.hasKey(address):
-      updateAccount(accounts[address], acc)
-    else:
-      accounts[address] = copyAccount(acc)
-  nimbus["stateDump"]["after"] = accounts
-
 proc requestPostState(thisBlock: Block): JsonNode =
   let postState = requestPostState(thisBlock.jsonData, postStateTracer, thisBlock)
   requestBlockState(postState, thisBlock)
@@ -159,15 +107,11 @@ proc requestPostState(thisBlock: Block): JsonNode =
   requestBlockState(postState, thisBlock, addresses)
   processPostState(postState)
 
-proc generatePremixData(nimbus: JsonNode, blockNumber: Uint256, thisBlock: Block, accounts: JsonNode) =
+proc generateGethData(thisBlock: Block, blockNumber: Uint256, accounts: JsonNode): JsonNode =
   let
     receipts = toJson(thisBlock.receipts)
-    txTraces = nimbus["txTraces"]
 
-  for trace in txTraces:
-    trace["structLogs"].fakeAlloc()
-
-  var geth = %{
+  let geth = %{
     "blockNumber": %blockNumber.toHex,
     "txTraces": thisBlock.traces,
     "receipts": receipts,
@@ -175,13 +119,7 @@ proc generatePremixData(nimbus: JsonNode, blockNumber: Uint256, thisBlock: Block
     "accounts": accounts
   }
 
-  var premixData = %{
-    "nimbus": nimbus,
-    "geth": geth
-  }
-
-  var data = "var premixData = " & premixData.pretty & "\n"
-  writeFile("premixData.js", data)
+  result = geth
 
 proc printDebugInstruction(blockNumber: Uint256) =
   var text = """
@@ -206,17 +144,18 @@ proc main() =
       blockNumber = UInt256.fromHex(nimbus["blockNumber"].getStr())
       thisBlock   = downloader.requestBlock(blockNumber, {DownloadReceipts, DownloadTxTrace})
       accounts    = requestPostState(thisBlock)
+      geth        = generateGethData(thisBlock, blockNumber, accounts)
+      parentNumber = blockNumber - 1.u256
+      parentBlock  = requestBlock(parentNumber)
 
-    # remove duplicate accounts with same address
-    # and only take newest one
-    removePostStateDup(nimbus)
+    processNimbusData(nimbus)
 
     # premix data goes to report page
-    generatePremixData(nimbus, blockNumber, thisBlock, accounts)
+    generatePremixData(nimbus, geth)
 
     # prestate data goes to debug tool and contains data
     # needed to execute single block
-    generatePrestate(nimbus, blockNumber, thisBlock)
+    generatePrestate(nimbus, geth, blockNumber, parentBlock.header, thisBlock.header, thisBlock.body)
 
     printDebugInstruction(blockNumber)
   except:

premix/premixcore.nim

@@ -0,0 +1,72 @@
+import json, strutils
+
+proc fakeAlloc(n: JsonNode) =
+  const
+    chunk = repeat('0', 64)
+
+  for i in 1 ..< n.len:
+    if not n[i].hasKey("memory"): return
+    let
+      prevMem = n[i-1]["memory"]
+      currMem = n[i]["memory"]
+
+    if currMem.len > prevMem.len:
+      let diff = currMem.len - prevMem.len
+      for _ in 0 ..< diff:
+        prevMem.add %chunk
+
+proc copyAccount*(acc: JsonNode): JsonNode =
+  result = newJObject()
+  if acc.hasKey("name"):
+    result["name"] = newJString(acc["name"].getStr)
+  result["balance"] = newJString(acc["balance"].getStr)
+  result["nonce"] = newJString(acc["nonce"].getStr)
+  result["code"] = newJString(acc["code"].getStr)
+  var storage = newJObject()
+  for k, v in acc["storage"]:
+    storage[k] = newJString(v.getStr)
+  result["storage"] = storage
+  result["storageRoot"] = newJString(acc["storageRoot"].getStr)
+  result["codeHash"] = newJString(acc["codeHash"].getStr)
+
+proc updateAccount*(a, b: JsonNode) =
+  if b.hasKey("name"):
+    a["name"] = newJString(b["name"].getStr)
+  a["balance"] = newJString(b["balance"].getStr)
+  a["nonce"] = newJString(b["nonce"].getStr)
+  a["code"] = newJString(b["code"].getStr)
+  var storage = a["storage"]
+  for k, v in b["storage"]:
+    storage[k] = newJString(v.getStr)
+  a["storageRoot"] = newJString(b["storageRoot"].getStr)
+  a["codeHash"] = newJString(b["codeHash"].getStr)
+
+proc removePostStateDup(nimbus: JsonNode) =
+  let postState = nimbus["stateDump"]["after"]
+  var accounts = newJObject()
+  for acc in postState:
+    let address = acc["address"].getStr
+    if accounts.hasKey(address):
+      updateAccount(accounts[address], acc)
+    else:
+      accounts[address] = copyAccount(acc)
+  nimbus["stateDump"]["after"] = accounts
+
+proc processNimbusData*(nimbus: JsonNode) =
+  # remove duplicate accounts with same address
+  # and only take newest one
+  removePostStateDup(nimbus)
+
+  let txTraces = nimbus["txTraces"]
+
+  for trace in txTraces:
+    trace["structLogs"].fakeAlloc()
+
+proc generatePremixData*(nimbus, geth: JsonNode) =
+  var premixData = %{
+    "nimbus": nimbus,
+    "geth": geth
+  }
+
+  var data = "var premixData = " & premixData.pretty & "\n"
+  writeFile("premixData.js", data)

premix/prestate.nim

@@ -3,39 +3,35 @@ import
   ../nimbus/db/[db_chain, storage_types], rlp, eth_common,
   ../nimbus/p2p/chain, ../nimbus/tracer
 
-proc generatePrestate*(nimbus: JsonNode, blockNumber: Uint256, thisBlock: Block) =
+proc generatePrestate*(nimbus, geth: JsonNode, blockNumber: Uint256, parent, header: BlockHeader, body: BlockBody) =
   let
-    state  = nimbus["state"]
-    parentNumber = blockNumber - 1.u256
-    parentBlock  = requestBlock(parentNumber)
-    headerHash   = rlpHash(thisBlock.header)
+    state = nimbus["state"]
+    headerHash = rlpHash(header)
+    #parentNumber = parent.blockNumber
 
   var
     memoryDB = newMemoryDB()
     chainDB = newBaseChainDB(memoryDB, false)
 
-  chainDB.setHead(parentBlock.header, true)
-  chainDB.persistTransactions(blockNumber, thisBlock.body.transactions)
-  discard chainDB.persistUncles(thisBlock.body.uncles)
+  chainDB.setHead(parent, true)
+  chainDB.persistTransactions(blockNumber, body.transactions)
+  discard chainDB.persistUncles(body.uncles)
 
-  memoryDB.put(genericHashKey(headerHash).toOpenArray, rlp.encode(thisBlock.header))
-  chainDB.addBlockNumberToHashLookup(thisBlock.header)
+  memoryDB.put(genericHashKey(headerHash).toOpenArray, rlp.encode(header))
+  chainDB.addBlockNumberToHashLookup(header)
 
   for k, v in state:
     let key = hexToSeqByte(k)
     let value = hexToSeqByte(v.getStr())
     memoryDB.put(key, value)
 
-  let
-    chain = newChain(chainDB)
-    parent = chainDB.getBlockHeader(parentNumber)
-    header = chainDB.getBlockHeader(blockNumber)
-    body = chainDB.getBlockBody(headerHash)
-    headers = @[header]
-    bodies = @[body]
+  #discard chainDB.getBlockHeader(parentNumber)
+  #discard chainDB.getBlockHeader(blockNumber)
+  #discard chainDB.getBlockBody(headerHash)
 
   var metaData = %{
-    "blockNumber": %blockNumber.toHex
+    "blockNumber": %blockNumber.toHex,
+    "geth": geth
   }
 
   metaData.dumpMemoryDB(memoryDB)