Various fixes towards state reconstruction

nimbus/db/backends/rocksdb_backend.nim

@@ -21,8 +21,12 @@ proc newChainDB*(basePath: string): ChainDB =
 
 proc get*(db: ChainDB, key: openarray[byte]): seq[byte] =
   let s = db.store.getBytes(key)
-  if not s.ok: raiseKeyReadError(key)
-  return s.value
+  if s.ok:
+    return s.value
+  elif s.error.len == 0:
+    discard
+  else:
+    raiseKeyReadError(key)
 
 proc put*(db: ChainDB, key, value: openarray[byte]) =
   let s = db.store.put(key, value)

nimbus/db/backends/sqlite_backend.nim

@@ -69,10 +69,12 @@ proc get*(db: ChainDB, key: openarray[byte]): seq[byte] =
     var
       resStart = columnBlob(db.selectStmt, 0)
       resLen   = columnBytes(db.selectStmt, 0)
-      resSeq   = newSeq[byte](resLen)
-    copyMem(resSeq.baseAddr, resStart, resLen)
-    return resSeq
-  else: raiseKeySearchError(key)
+    result = newSeq[byte](resLen)
+    copyMem(result.baseAddr, resStart, resLen)
+  of SQLITE_DONE:
+    discard
+  else:
+    raiseKeyReadError(key)
 
 proc put*(db: ChainDB, key, value: openarray[byte]) =
   template check(op) =

nimbus/db/db_chain.nim

@@ -31,13 +31,10 @@ proc `$`*(db: BaseChainDB): string =
   result = "BaseChainDB"
 
 proc getBlockHeader*(self: BaseChainDB; blockHash: Hash256, output: var BlockHeader): bool =
-  try:
-    let blk = self.db.get(genericHashKey(blockHash).toOpenArray).toRange
-    if blk.len != 0:
-      output = rlp.decode(blk, BlockHeader)
-      result = true
-  except KeyError:
-    discard
+  let data = self.db.get(genericHashKey(blockHash).toOpenArray).toRange
+  if data.len != 0:
+    output = rlp.decode(data, BlockHeader)
+    result = true
 
 proc getBlockHeader*(self: BaseChainDB, blockHash: Hash256): BlockHeader =
   ## Returns the requested block header as specified by block hash.
@@ -47,11 +44,10 @@ proc getBlockHeader*(self: BaseChainDB, blockHash: Hash256): BlockHeader =
     raise newException(BlockNotFound, "No block with hash " & blockHash.data.toHex)
 
 proc getHash(self: BaseChainDB, key: DbKey, output: var Hash256): bool {.inline.} =
-  try:
-    output = rlp.decode(self.db.get(key.toOpenArray).toRange, Hash256)
+  let data = self.db.get(key.toOpenArray).toRange
+  if data.len != 0:
+    output = rlp.decode(data, Hash256)
     result = true
-  except KeyError:
-    discard
 
 proc getCanonicalHead*(self: BaseChainDB): BlockHeader =
   var headHash: Hash256
@@ -145,13 +141,13 @@ proc setAsCanonicalChainHead(self: BaseChainDB; headerHash: Hash256): seq[BlockH
   for h in newCanonicalHeaders:
     self.addBlockNumberToHashLookup(h)
 
-  self.db.put(canonicalHeadHashKey().toOpenArray, rlp.encode(header.hash).toOpenArray)
+  self.db.put(canonicalHeadHashKey().toOpenArray, rlp.encode(headerHash).toOpenArray)
 
   return newCanonicalHeaders
 
 proc headerExists*(self: BaseChainDB; blockHash: Hash256): bool =
   ## Returns True if the header with the given block hash is in our DB.
-  self.db.contains(blockHash.data)
+  self.db.contains(genericHashKey(blockHash).toOpenArray)
 
 iterator getBlockTransactionData(self: BaseChainDB, transactionRoot: Hash256): BytesRange =
   var transactionDb = initHexaryTrie(self.db, transactionRoot)
@@ -183,14 +179,15 @@ iterator getBlockTransactions(self: BaseChainDB; transactionRoot: Hash256;
 
 proc persistHeaderToDb*(self: BaseChainDB; header: BlockHeader): seq[BlockHeader] =
   let isGenesis = header.parentHash == GENESIS_PARENT_HASH
+  let headerHash = header.blockHash
   if not isGenesis and not self.headerExists(header.parentHash):
     raise newException(ParentNotFound, "Cannot persist block header " &
-        $header.hash & " with unknown parent " & $header.parentHash)
-  self.db.put(genericHashKey(header.hash).toOpenArray, rlp.encode(header).toOpenArray)
+        $headerHash & " with unknown parent " & $header.parentHash)
+  self.db.put(genericHashKey(headerHash).toOpenArray, rlp.encode(header).toOpenArray)
 
   let score = if isGenesis: header.difficulty
               else: self.getScore(header.parentHash).u256 + header.difficulty
-  self.db.put(blockHashToScoreKey(header.hash).toOpenArray, rlp.encode(score).toOpenArray)
+  self.db.put(blockHashToScoreKey(headerHash).toOpenArray, rlp.encode(score).toOpenArray)
 
   self.addBlockNumberToHashLookup(header)
 
@@ -198,10 +195,10 @@ proc persistHeaderToDb*(self: BaseChainDB; header: BlockHeader): seq[BlockHeader
   try:
     headScore = self.getScore(self.getCanonicalHead().hash)
   except CanonicalHeadNotFound:
-    return self.setAsCanonicalChainHead(header.hash)
+    return self.setAsCanonicalChainHead(headerHash)
 
   if score > headScore.u256:
-    result = self.setAsCanonicalChainHead(header.hash)
+    result = self.setAsCanonicalChainHead(headerHash)
 
 proc addTransactionToCanonicalChain(self: BaseChainDB, txHash: Hash256,
     blockHeader: BlockHeader, index: int) =

nimbus/db/storage_types.nim

@@ -73,7 +73,7 @@ template raiseKeyWriteError*(key: auto) =
   raise newException(StorageError, "failed to write key " & $key)
 
 template raiseKeySearchError*(key: auto) =
-  raise newException(KeyError, "failure during search for key " & $key)
+  raise newException(StorageError, "failure during search for key " & $key)
 
 template raiseKeyDeletionError*(key: auto) =
   raise newException(StorageError, "failure to delete key " & $key)

nimbus/nimbus.nim

@@ -14,7 +14,7 @@ import
   config, genesis, rpc/[common, p2p], p2p/chain,
   eth_trie
 
-const UseSqlite = true
+const UseSqlite = false
 
 when UseSqlite:
   import db/backends/sqlite_backend

nimbus/p2p/chain.nim

@@ -1,5 +1,5 @@
-import ../db/db_chain, eth_common, chronicles, ../vm_state, ../vm_types, ../transaction,
-  ../vm/[computation, interpreter_dispatch, message]
+import ../db/[db_chain, state_db], eth_common, chronicles, ../vm_state, ../vm_types, ../transaction, ranges,
+  ../vm/[computation, interpreter_dispatch, message], ../constants
 
 
 type
@@ -35,17 +35,28 @@ method persistBlocks*(c: Chain, headers: openarray[BlockHeader], bodies: openarr
   assert(headers.len == bodies.len)
 
   for i in 0 ..< headers.len:
-    let head = c.db.getCanonicalHead()
-    # assert(head.blockNumber == headers[i].blockNumber - 1)
-    let vmState = newBaseVMState(head, c.db)
-    if bodies[i].transactions.len != 0:
-      # echo "block: ", headers[i].blockNumber
-      for t in bodies[i].transactions:
-        var msg: Message
-        # echo "trns: ", t
+    echo "Persisting block: ", headers[i].blockNumber
+    if headers[i].txRoot != BLANK_ROOT_HASH:
+      let head = c.db.getCanonicalHead()
+      # assert(head.blockNumber == headers[i].blockNumber - 1)
+      let vmState = newBaseVMState(head, c.db)
+      let stateDb = newAccountStateDB(c.db.db, head.stateRoot)
+      if bodies[i].transactions.len != 0:
+        # echo "block: ", headers[i].blockNumber
+        for t in bodies[i].transactions:
+          var sender: EthAddress
+          if t.getSender(sender):
+            echo "Sender: ", sender
+            let code = stateDb.getCode(sender)
+            debug "Transaction", sender, to = t.to, value = t.value, hasCode = code.len != 0
+            let msg = newMessage(t.gasLimit, t.gasPrice, t.to, sender, t.value, t.payload, code.toSeq)
+      assert(false, "Dont know how to persist transactions")
 
-        # let msg = newMessage(t.gasLimit, t.gasPrice, t.to, t.getSender, 
+    if headers[i].ommersHash != EMPTY_UNCLE_HASH:
+      debug "Ignoring ommers", blockNumber = headers[i].blockNumber
+      
+    discard c.db.persistHeaderToDb(headers[i])
+    assert(c.db.getCanonicalHead().blockHash == headers[i].blockHash)
 
-      # let c = newBaseComputation(vmState, 
 
   discard

nimbus/transaction.nim

@@ -49,7 +49,7 @@ proc toSignature*(transaction: Transaction): Signature =
   bytes[0..31] = transaction.R.toByteArrayBE()
   bytes[32..63] = transaction.S.toByteArrayBE()
   # TODO: V will become a byte or range soon.
-  bytes[64] = transaction.V
+  bytes[64] = transaction.V - 27 # TODO: 27 should come from somewhere
   initSignature(bytes)
 
 proc getSender*(transaction: Transaction, output: var EthAddress): bool =

nimbus/vm/computation.nim

@@ -25,7 +25,7 @@ proc newBaseComputation*(vmState: BaseVMState, blockNumber: UInt256, message: Me
   result.children = @[]
   result.accountsToDelete = initTable[EthAddress, EthAddress]()
   result.logEntries = @[]
-  result.code = newCodeStreamFromUnescaped(message.code) # TODO: what is the best repr
+  result.code = newCodeStream(message.code)
   # result.rawOutput = "0x"
   result.gasCosts = blockNumber.toFork.forkToSchedule
 
@@ -67,7 +67,7 @@ proc prepareChildMessage*(
     to,
     value,
     data,
-    code.bytesToHex, # TODO: use seq[byte] for Message as well
+    code,
     childOptions)
 
 func output*(c: BaseComputation): seq[byte] =

nimbus/vm/message.nim

@@ -39,7 +39,7 @@ proc newMessage*(
     sender: EthAddress,
     value: UInt256,
     data: seq[byte],
-    code: string,
+    code: seq[byte],
     options: MessageOptions = newMessageOptions()): Message =
 
   validateGte(options.depth, minimum=0, title="Message.depth")

nimbus/vm_types.nim

@@ -90,7 +90,7 @@ type
     # Not in EVMC API
 
     # TODO: Done via callback function (v)table in EVMC
-    code*:                    string    # TODO: seq[byte] is probably a better representation
+    code*:                    seq[byte]
 
     internalOrigin*:          EthAddress
     internalCodeAddress*:     EthAddress

tests/test_opcode.nim

@@ -6,7 +6,7 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
-  unittest, tables, parseutils,
+  unittest, tables, parseutils, byteutils,
   eth_trie/[types, memdb], eth_common/eth_types,
   ../nimbus/[constants, vm_types, vm_state],
   ../nimbus/vm/interpreter,
@@ -32,7 +32,7 @@ proc testCode(code: string, initialGas: GasInt, blockNum: UInt256): BaseComputat
     sender=ZERO_ADDRESS, #fixture{"exec"}{"caller"}.getStr,
     value=0.u256,
     data = @[],
-    code=code,
+    code=code.hexToSeqByte,
     gas=initial_gas,
     gasPrice=1) # What is this used for?
     # gasPrice=fixture{"exec"}{"gasPrice"}.getHexadecimalInt.u256,

tests/test_storage_backends.nim

@@ -49,6 +49,8 @@ template backendTests(DB) =
         keyA notin db
         keyB in db
 
+      check db.get(keyA) == @[]
+
       check db.get(keyB) == value1
       db.del(keyA)
 

tests/test_vm_json.nim

@@ -22,11 +22,6 @@ suite "vm json tests":
   jsonTest("VMTests", testFixture)
 
 
-proc stringFromBytes(x: ByteRange): string =
-  result = newString(x.len)
-  for i in 0 ..< x.len:
-    result[i] = char(x[i])
-
 proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus) =
   var fixture: JsonNode
   for label, child in fixtures:
@@ -47,13 +42,13 @@ proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus) =
   var memDb = newMemDB()
   var vmState = newBaseVMState(header, newBaseChainDB(trieDB memDb))
   let fexec = fixture["exec"]
-  var code = ""
+  var code: seq[byte]
   vmState.mutateStateDB:
     setupStateDB(fixture{"pre"}, db)
     let address = fexec{"address"}.getStr.parseAddress
-    code = stringFromBytes db.getCode(address)
+    code = db.getCode(address).toSeq
 
-  code = fexec{"code"}.getStr
+  code = fexec{"code"}.getStr.hexToSeqByte
   let toAddress = fexec{"address"}.getStr.parseAddress
   let message = newMessage(
       to = toAddress,
@@ -67,7 +62,7 @@ proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus) =
                                   createAddress = toAddress))
 
   #echo fixture{"exec"}
-  var c = newCodeStreamFromUnescaped(code)
+  var c = newCodeStream(code)
   when defined(nimbusdebug):
     c.displayDecompiled()