add report page

nimbus/tracer.nim

@@ -32,11 +32,11 @@ proc toJson*(receipts: seq[Receipt]): JsonNode =
   for receipt in receipts:
     result.add receipt.toJson
 
-proc getSender(tx: Transaction): EthAddress =
+proc getSender*(tx: Transaction): EthAddress =
   if not tx.getSender(result):
     raise newException(ValueError, "Could not get sender")
 
-proc getRecipient(tx: Transaction): EthAddress =
+proc getRecipient*(tx: Transaction): EthAddress =
   if tx.isContractCreation:
     let sender = tx.getSender()
     result = generateAddress(sender, tx.accountNonce)

premix/assets/js/index.js

@@ -0,0 +1,229 @@
+var premix = function() {
+  function chunkSubstr(str, size) {
+    const numChunks = Math.ceil(str.length / size)
+    const chunks = new Array(numChunks)
+
+    for (let i = 0, o = 0; i < numChunks; ++i, o += size) {
+      chunks[i] = str.substr(o, size)
+    }
+
+    return chunks
+  }
+
+  function split32(text) {
+    if(text.length > 32) {
+      let chunks = chunkSubstr(text, 32);
+      let result = "";
+      for(var x of chunks) {
+        result += '<div>'+x+'</div>';
+      }
+      return result;
+    } else {
+      return text;
+    }
+  }
+
+  return {
+    fields: ['op', 'pc', 'gas', 'gasCost', 'depth'],
+
+    newTable: function(container) {
+      let table = $('<table class="uk-table uk-table-divider"/>').appendTo(container);
+      $('<thead><tr><th>Field</th><th>Nimbus</th><th>Geth</th></tr></thead>').appendTo(table);
+      return $('<tbody></tbody>').appendTo(table);
+    },
+
+    renderRow: function(body, nimbus, geth, x) {
+      let row = $('<tr/>').appendTo(body);
+      let ncr = nimbus[x].toString().toLowerCase();
+      let gcr = geth[x].toString().toLowerCase();
+      let cls = ncr == gcr ? '' : 'class="uk-text-danger"';
+      $(`<td ${cls}>${split32(x)}</td>`).appendTo(row);
+      $(`<td ${cls}>${split32(ncr)}</td>`).appendTo(row);
+      $(`<td ${cls}>${split32(gcr)}</td>`).appendTo(row);
+    },
+
+    newSection: function(container, title, colored) {
+      let section = $('<div class="uk-section uk-section-xsmall tm-horizontal-overflow"></div>').appendTo(container);
+      section.addClass(colored ? "uk-section-secondary uk-light" : "uk-section-muted");
+      let contentDiv = $('<div class="uk-container uk-margin-small-left uk-margin-small-right"></div>').appendTo(section);
+      $(`<h4>${title}</h4>`).appendTo(contentDiv);
+      return contentDiv;
+    }
+
+  };
+}();
+
+function windowResize() {
+  let bodyHeight = $(window).height();
+  $('#opCodeSideBar').css('height', parseInt(bodyHeight) - 80);
+}
+
+function renderTrace(title, nimbus, geth) {
+  let container = $('#opCodeContainer').empty();
+  let body = premix.newTable(container);
+  for(var x of premix.fields) {
+    premix.renderRow(body, nimbus, geth, x);
+  }
+
+  function renderExtra(name) {
+    let nk = Object.keys(nimbus[name]);
+    let gk = Object.keys(geth[name]);
+    let keys = new Set(nk.concat(gk));
+
+    if(keys.size > 0) {
+      let section = premix.newSection(container, name);
+      let body = premix.newTable(section);
+      for(var key of keys) {
+        premix.renderRow(body, nimbus[name], geth[name], key);
+      }
+      $('<hr class="uk-divider-icon">').appendTo(container);
+    }
+  }
+
+  renderExtra("memory");
+  renderExtra("stack");
+  renderExtra("storage");
+}
+
+function opCodeRenderer(txId, nimbus, geth) {
+  function analyze(nimbus, geth) {
+    for(var x of premix.fields) {
+      if(nimbus[x] != geth[x]) return false;
+    }
+    // TODO: analyze stack, storage, mem
+    return true;
+  }
+
+  txId = parseInt(txId);
+  var ncs = nimbus.txTraces[txId].structLogs;
+  var gcs = geth.txTraces[txId].structLogs;
+  var sideBar = $('#opCodeSideBar').empty();
+  $('#opCodeTitle').text(`Tx #${(txId+1)}`);
+
+  for(var i in ncs) {
+    var pc = ncs[i];
+    if(!analyze(ncs[i], gcs[i])) {
+      var nav = $(`<li><a class="tm-text-danger" rel="${i}" href="#">${pc.pc + ' ' + pc.op}</a></li>`).appendTo(sideBar);
+    } else {
+      var nav = $(`<li><a rel="${i}" href="#">${pc.pc + ' ' + pc.op}</a></li>`).appendTo(sideBar);
+    }
+    nav.children('a').click(function(ev) {
+      let idx = this.rel;
+      $('#sideBar li').removeClass('uk-active');
+      $(this).parent().addClass('uk-active');
+      renderTrace('tx', ncs[idx], gcs[idx]);
+    });
+  }
+
+  renderTrace("tx", ncs[0], gcs[0]);
+  windowResize();
+}
+
+function transactionsRenderer(txId, nimbus, geth) {
+  $('#transactionsTitle').text(`Tx #${(txId+1)}`);
+  let container = $('#transactionsContainer').empty();
+
+  function renderTx(nimbus, geth) {
+    let body = premix.newTable(container);
+    const fields = ["gas", "returnValue", "cumulativeGasUsed", "bloom"];
+    for(var x of fields) {
+      premix.renderRow(body, nimbus, geth, x);
+    }
+
+    // TODO: receipt logs
+  }
+
+  txId = parseInt(txId);
+  let ntx = nimbus.txTraces[txId];
+  let gtx = geth.txTraces[txId];
+
+  let ncr = $.extend({
+    gas: ntx.gas,
+    returnValue: ntx.returnValue
+  },
+    nimbus.receipts[txId]
+  );
+
+  let gcr = $.extend({
+    gas: gtx.gas,
+    returnValue: "0x" + gtx.returnValue
+  },
+    geth.receipts[txId]
+  );
+
+  renderTx(ncr, gcr);
+}
+
+function headerRenderer(nimbus, geth) {
+  let container = $('#headerContainer').empty();
+
+  let ncs = nimbus.stateDump.after;
+  let gcs = geth.accounts;
+
+
+  for(var n of ncs) {
+    n.address = '0x' + n.address;
+    n.balance = '0x' + n.balance;
+    n.code = '0x' + n.code;
+    let g = gcs[n.address];
+
+  }
+
+          /*"name": "internalTx0",
+          "address": "0000000000000000000000000000000000000004",
+          "nonce": "0000000000000000",
+          "balance": "0",
+          "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
+          "code": "",
+          "storageRoot": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+          "storage": {}*/
+
+        /*"0xf927a40c8b7f6e07c5af7fa2155b4864a4112b13": {
+        "balance": "0x607c9cea65ef7e19dd8",
+        "nonce": "0000000000000000",
+        "code": "0x",
+        "storage": {}*/
+
+
+}
+
+function generateNavigation(txs, nimbus, geth) {
+  function navAux(menuId, renderer) {
+    let menu = $(menuId).click(function(ev) {
+      renderer(0, nimbus, geth);
+    });
+
+    if(txs.length == 0) {
+      menu.parent().addClass('uk-disabled');
+    } else if(txs.length > 1) {
+      $('<span uk-icon="icon: triangle-down"></span>').appendTo(menu);
+      let div  = $('<div uk-dropdown="mode: hover;"/>').appendTo(menu.parent());
+      let list = $('<ul class="uk-nav uk-dropdown-nav"/>').appendTo(div);
+
+      for(var i in txs) {
+        let id = i.toString();
+        $(`<li class="uk-active"><a rel="${id}" href="#">TX #${id}</a></li>`).appendTo(list);
+      }
+
+      list.find('li a').click(function(ev) {
+        renderer(this.rel, nimbus, geth);
+      });
+    }
+  }
+
+  navAux('#opCodeMenu', opCodeRenderer);
+  navAux('#transactionsMenu', transactionsRenderer);
+
+  $('#headerMenu').click(function(ev) {
+    headerRenderer(nimbus, geth);
+  });
+}
+
+$(document).ready(function() {
+
+  var nimbus = premixData.nimbus;
+  var geth = premixData.geth;
+  var transactions = geth.block.transactions;
+
+  generateNavigation(transactions, nimbus, geth);
+});

premix/index.html

@@ -0,0 +1,164 @@
+<html class="uk-height-1-1">
+  <head>
+      <title id="windowTitle">Premix Report Page</title>
+      <meta http-equiv="Content-type" content="text/html; charset=utf-8">
+      <meta name="viewport" content="width=device-width, initial-scale=1">
+
+      <script src="assets/js/jquery.min.js"></script>
+      <script src="assets/js/uikit.min.js"></script>
+      <script src="assets/js/uikit-icons.min.js"></script>
+      <script src="assets/js/index.js"></script>
+      <script src="premixData.js"></script>
+      <link rel="stylesheet" href="assets/css/uikit.min.css" />
+      <style>
+        body {
+          font: 12px normal Arial, Helvetica, sans-serif;
+        }
+        .tm-horizontal-overflow {
+          overflow-x: auto;
+        }
+        .tm-sidebar {
+          position: fixed;
+          overflow-y: auto;
+          padding-left:40px;
+        }
+        .uk-nav-default > li > a.tm-text-danger {
+          color: #f0506e;
+        }
+        .uk-nav-default > li > a.tm-text-danger:hover {
+          color: red;
+        }
+        .uk-nav-default > li.uk-active > a.tm-text-danger {
+          color: red;
+        }
+      </style>
+  </head>
+
+  <body onresize="windowResize()" class="uk-height-1-1">
+
+    <div class="uk-section-small uk-background-primary uk-light" uk-sticky="bottom: #offset">
+      <!-- Navigation -->
+      <div class="uk-overlay uk-position-left uk-flex uk-flex-middle">
+        <h1>Premix Report Page</h1>
+      </div>
+
+      <div class="uk-position-right uk-overlay">
+        <ul class="uk-subnav uk-subnav-divider" uk-switcher="connect:#switcherSection">
+          <li><a id="opCodeMenu" href="#">OpCode</a></li>
+          <li><a id="transactionsMenu" href="#">Transactions</a></li>
+          <li><a id="headerMenu" href="#">Header</a></li>
+          <li class="uk-active"><a href="#">Help</a></li>
+        </ul>
+      </div>
+      <!-- Navigation -->
+    </div>
+
+    <ul id="switcherSection" class="uk-switcher">
+
+      <li>
+        <!-- Opcode Page -->
+        <div class="uk-grid-collapse" uk-grid>
+
+          <div class="uk-width-1-5@m">
+            <ul id="opCodeSideBar" class="tm-sidebar uk-nav uk-nav-default uk-height-1-1 uk-width-1-5@m">
+                <!-- op code traces navigation sidebar -->
+            </ul>
+          </div>
+
+          <div class="uk-width-4-5@m">
+
+            <div class="uk-section-small uk-section-default">
+              <div class="uk-container uk-container-expand">
+                <h3>Opcode Trace <span id="opCodeTitle" class="uk-text-primary uk-text-small">Tx #</span></h3>
+                <div id="opCodeContainer">
+                </div>
+              </div>
+            </div>
+
+            <div class="uk-section uk-section-small uk-section-secondary uk-light">
+              <div class="uk-container uk-text-center">
+                <h2>Have You Found The Bug?</h2>
+              </div>
+
+              <ul class="uk-subnav uk-subnav-divider uk-flex uk-flex-center" uk-margin>
+                <li><a href="https://github.com/status-im/nimbus"><span uk-icon="github" class="uk-margin-small-right"></span>Github</a></li>
+                <li><a href="https://gitter.im/status-im/nimbus"><span uk-icon="gitter" class="uk-margin-small-right"></span>Gitter</a></li>
+              </ul>
+
+            </div>
+
+          </div>
+        </div>
+        <!-- Opcode Page -->
+      </li>
+
+      <li>
+        <!-- Transactions Page -->
+        <div class="uk-section-small uk-section-default">
+          <div class="uk-container uk-container-medium">
+            <h3>Transactions Post State Accounts <span id="transactionsTitle" class="uk-text-primary uk-text-small">Tx #</span></h3>
+            <div id="transactionsContainer">
+            </div>
+          </div>
+        </div>
+        <!-- Transactions Page -->
+      </li>
+
+      <li>
+        <!-- Header Page -->
+        <div class="uk-section-small uk-section-default">
+          <div class="uk-container uk-container-medium">
+            <h3>Header Post State Accounts</h3>
+            <div id="headerContainer">
+            </div>
+          </div>
+        </div>
+        <!-- Header Page -->
+      </li>
+
+      <li>
+        <!-- Help Page -->
+        <div class="uk-section-small uk-section-default">
+          <div class="uk-container uk-container-xsmall">
+            <h2>Help</h2>
+
+            <p>The top navigation is ordered from left to right like the picture below.</p>
+
+            <ul class="uk-subnav uk-subnav-divider" uk-margin>
+              <li><span>OpCode</span><span uk-icon="icon: triangle-down"></span></li>
+              <li><span>Transactions</span><span uk-icon="icon: triangle-down"></span></li>
+              <li><span>Header</span></li>
+              <li><span>Help</span></li>
+            </ul>
+
+            <p>
+              Usually you will start from left, then move to the right to find out where the bug might located.
+              If you see <span class="uk-text-warning">red colored text</span>, it means you already found the difference between nimbus and other ethereum
+              client tracing result.
+            </p>
+
+            <p>
+              If there is no red colored text in opcode section, it means the bug might be located in transactions section,
+              or in header section.
+            </p>
+
+            <p>
+              Once you already located the bug, you can use <span class="uk-text-primary">debug.nim</span> located in <span class="uk-text-primary">&#47;premix</span>
+              to sort things out until there is no more error or it pass block validation.
+            </p>
+
+            <p>
+              Block with multiple transactions will have triangle pointing downward in the navigation bar.
+              Usually only the first transaction with red colored text have the problem, but it might affect the rest
+              of other transactions. In opcode section, the same thing happened, perhaps only the first instruction
+              with red colored text have the problem, but it will affect the rest of other instructions.
+            </p>
+          </div>
+        </div>
+        <!-- Help Page -->
+      </li>
+
+    </ul>
+
+  </body>
+</html>

premix/js_tracer.nim

@@ -1,6 +1,5 @@
-{
+const postStateTracer* = """{
 	postState: null,
-  accounts: [$1],
 
 	// lookupAccount injects the specified account into the postState object.
 	lookupAccount: function(addr, db){
@@ -22,10 +21,7 @@
 		var idx = toHex(key);
 
 		if (this.postState[acc].storage[idx] === undefined) {
-			var val = toHex(db.getState(addr, key));
+      this.postState[acc].storage[idx] = "";//toHex(db.getState(addr, key));
-			if (val != "0x0000000000000000000000000000000000000000000000000000000000000000") {
-				this.postState[acc].storage[idx] = toHex(db.getState(addr, key));
-			}
 		}
 	},
 
@@ -35,10 +31,6 @@
 		this.lookupAccount(ctx.from, db);
     this.lookupAccount(ctx.to, db);
 
-    for(var i in this.accounts) {
-      this.lookupAccount(toAddress(this.accounts[i]), db)
-    }
-
 		// Return the assembled allocations (postState)
 		return this.postState;
 	},
@@ -73,3 +65,4 @@
 	// fault is invoked when the actual execution of an opcode fails.
 	fault: function(log, db) {}
 }
+"""

premix/premix.nim

@@ -1,6 +1,8 @@
 import
   json, downloader, stint, strutils, os,
-  ../nimbus/tracer, chronicles, prestate
+  ../nimbus/tracer, chronicles, prestate,
+  js_tracer, eth_common, byteutils, parser,
+  nimcrypto
 
 proc fakeAlloc(n: JsonNode) =
   const
@@ -17,52 +19,49 @@ proc fakeAlloc(n: JsonNode) =
       for _ in 0 ..< diff:
         prevMem.add %chunk
 
+proc jsonTracer(tracer: string): JsonNode =
+  result = %{ "tracer": %tracer }
+
+proc requestTrace(txHash, tracer: JsonNode): JsonNode =
+  let txTrace = request("debug_traceTransaction", %[txHash, tracer])
+  if txTrace.kind == JNull:
+    error "requested postState not available", txHash=txHash
+    raise newException(ValueError, "Error when retrieving transaction postState")
+  result = txTrace
+
 proc requestPostState(n: JsonNode, jsTracer: string): JsonNode =
   let txs = n["transactions"]
-  if txs.len > 0:
+  result = newJArray()
-    result = newJArray()
+  if txs.len == 0: return
-    let tracer = %{
-      "tracer": %jsTracer
-    }
-    for tx in txs:
-      let txHash = tx["hash"]
-      let txTrace = request("debug_traceTransaction", %[txHash, tracer])
-      if txTrace.kind == JNull:
-        error "requested postState not available", txHash=txHash
-        raise newException(ValueError, "Error when retrieving transaction postState")
-      result.add txTrace
-
-proc requestLastPostState(n: JsonNode, jsTracer: string, arr: JsonNode) =
-  let txs = n["transactions"]
-  if txs.len > 0:
-    let tracer = %{
-      "tracer": %jsTracer
-    }
 
+  let tracer = jsonTracer(jsTracer)
+  for tx in txs:
+    if tx["to"].kind != JNull:
+      result.add newJObject()
+      continue
     let
-      tx = txs[txs.len - 1]
       txHash = tx["hash"]
-      txTrace = request("debug_traceTransaction", %[txHash, tracer])
+      txTrace = requestTrace(txHash, tracer)
-    if txTrace.kind == JNull:
+    result.add txTrace
-      error "requested postState not available", txHash=txHash
-      raise newException(ValueError, "Error when retrieving transaction postState")
-    arr.add txTrace
 
-proc requestPostState(thisBlock: Block): JsonNode =
+proc padding(x: string): JsonNode =
-  let
+  let val = x.substr(2)
-    tmp = readFile("poststate_tracer.js.template")
+  let pad = repeat('0', 64 - val.len)
-    tracer = tmp % [ $thisBlock.jsonData["miner"] ]
+  result = newJString("0x" & pad & val)
-  result = requestPostState(thisBlock.jsonData, tracer)
 
-  var uncles = ""
+proc requestBlockState(postState: JsonNode, thisBlock: Block) =
-  for i, uncle in thisBlock.body.uncles:
+  let number = %(thisBlock.header.blockNumber.prefixHex)
-    uncles.add $uncle.coinbase
-    if i < thisBlock.body.uncles.len - 1:
-      uncles.add ", "
 
-  if uncles.len > 0:
+  for state in postState:
-    let tracer = tmp % [uncles]
+    for address, account in state:
-    requestLastPostState(thisBlock.jsonData, tracer, result)
+      var storage = newJArray()
+      for k, _ in account["storage"]:
+        storage.add %k
+      let trace = request("eth_getProof", %[%address, storage, number])
+      account["codeHash"] = trace["codeHash"]
+      account["storageHash"] = trace["storageHash"]
+      for x in trace["storageProof"]:
+        account["storage"][x["key"].getStr] = padding(x["value"].getStr())
 
 proc copyAccount(acc: JsonNode): JsonNode =
   result = newJObject()
@@ -82,6 +81,24 @@ proc updateAccount(a, b: JsonNode) =
   for k, v in b["storage"]:
     storage[k] = newJString(v.getStr)
 
+proc requestBlockState(postState: JsonNode, thisBlock: Block, addresses: seq[EthAddress]) =
+  let number = %(thisBlock.header.blockNumber.prefixHex)
+
+  var txTrace = newJObject()
+  for a in addresses:
+    let address = a.prefixHex
+    let trace = request("eth_getProof", %[%address, %[], number])
+    let account = %{
+      "codeHash": trace["codeHash"],
+      "storageHash": trace["storageHash"],
+      "balance": trace["balance"],
+      "nonce": trace["nonce"],
+      "code": newJString("0x"),
+      "storage": newJObject()
+    }
+    txTrace[address] = account
+  postState.add txTrace
+
 proc processPostState(postState: JsonNode): JsonNode =
   var accounts = newJObject()
 
@@ -94,7 +111,18 @@ proc processPostState(postState: JsonNode): JsonNode =
 
   result = accounts
 
-proc generatePremixData(nimbus: JsonNode, blockNumber: Uint256, thisBlock: Block, postState, accounts: JsonNode) =
+proc requestPostState(thisBlock: Block): JsonNode =
+  let postState = requestPostState(thisBlock.jsonData, postStateTracer)
+  requestBlockState(postState, thisBlock)
+
+  var addresses = @[thisBlock.header.coinbase]
+  for uncle in thisBlock.body.uncles:
+    addresses.add uncle.coinbase
+
+  requestBlockState(postState, thisBlock, addresses)
+  processPostState(postState)
+
+proc generatePremixData(nimbus: JsonNode, blockNumber: Uint256, thisBlock: Block, accounts: JsonNode) =
   let
     receipts = toJson(thisBlock.receipts)
     txTraces = nimbus["txTraces"]
@@ -107,7 +135,6 @@ proc generatePremixData(nimbus: JsonNode, blockNumber: Uint256, thisBlock: Block
     "txTraces": thisBlock.traces,
     "receipts": receipts,
     "block": thisBlock.jsonData,
-    "postState": postState,
     "accounts": accounts
   }
 
@@ -141,10 +168,9 @@ proc main() =
       nimbus      = json.parseFile(paramStr(1))
       blockNumber = UInt256.fromHex(nimbus["blockNumber"].getStr())
       thisBlock   = downloader.requestBlock(blockNumber, {DownloadReceipts, DownloadTxTrace})
-      postState   = requestPostState(thisBlock)
+      accounts    = requestPostState(thisBlock)
-      accounts    = processPostState(postState)
 
-    generatePremixData(nimbus, blockNumber, thisBlock, postState, accounts)
+    generatePremixData(nimbus, blockNumber, thisBlock, accounts)
     generatePrestate(nimbus, blockNumber, thisBlock)
     printDebugInstruction(blockNumber)
   except: