Populate the persistent databases with the empty RLP key.

Also implements transactional block persistence. Two issues
in the transaction processing code have been discovered that
might affect other usages such as the CALL instruction.

The main fix gets us past block 49000.

You may need to clean up your database.

nimbus/db/backends/caching_backend.nim

@@ -1,5 +1,7 @@
-import ranges, eth_trie, tables, sets
-import ../storage_types
+import
+  ranges, tables, sets,
+  eth_trie/db,
+  ../storage_types
 
 type
   CachingDB* = ref object of RootObj

nimbus/db/backends/rocksdb_backend.nim

@@ -1,4 +1,4 @@
-import os, rocksdb, ranges, eth_trie/db_tracing
+import os, rocksdb, ranges, eth_trie/[db_tracing, constants]
 import ../storage_types
 
 type
@@ -7,18 +7,6 @@ type
 
   ChainDB* = RocksChainDB
 
-proc newChainDB*(basePath: string): ChainDB =
-  result.new()
-  let
-    dataDir = basePath / "data"
-    backupsDir = basePath / "backups"
-
-  createDir(dataDir)
-  createDir(backupsDir)
-
-  let s = result.store.init(dataDir, backupsDir)
-  if not s.ok: raiseStorageInitError()
-
 proc get*(db: ChainDB, key: openarray[byte]): seq[byte] =
   let s = db.store.getBytes(key)
   if s.ok:
@@ -46,3 +34,18 @@ proc del*(db: ChainDB, key: openarray[byte]) =
 
 proc close*(db: ChainDB) =
   db.store.close
+
+proc newChainDB*(basePath: string): ChainDB =
+  result.new()
+  let
+    dataDir = basePath / "data"
+    backupsDir = basePath / "backups"
+
+  createDir(dataDir)
+  createDir(backupsDir)
+
+  let s = result.store.init(dataDir, backupsDir)
+  if not s.ok: raiseStorageInitError()
+
+  put(result, emptyRlpHash.data, emptyRlp)
+

nimbus/db/backends/sqlite_backend.nim

@@ -1,5 +1,5 @@
 import
-  os, sqlite3, ranges, ranges/ptr_arith, eth_trie/db_tracing,
+  os, sqlite3, ranges, ranges/ptr_arith, eth_trie/[db_tracing, constants],
   ../storage_types
 
 type
@@ -9,6 +9,8 @@ type
 
   ChainDB* = SqliteChainDB
 
+proc put*(db: ChainDB, key, value: openarray[byte])
+
 proc newChainDB*(basePath: string, inMemory = false): ChainDB =
   result.new()
   let dbPath = if inMemory: ":memory:" else: basePath / "nimbus.db"
@@ -54,6 +56,8 @@ proc newChainDB*(basePath: string, inMemory = false): ChainDB =
 
   result.deleteStmt = prepare "DELETE FROM trie_nodes WHERE key = ?;"
 
+  put(result, emptyRlpHash.data, emptyRlp)
+
 proc bindBlob(s: Pstmt, n: int, blob: openarray[byte]): int32 =
   sqlite3.bind_blob(s, n.int32, blob.baseAddr, blob.len.int32, nil)
 

nimbus/db/db_chain.nim

@@ -7,7 +7,7 @@
 
 import
   tables, sequtils, algorithm,
-  rlp, ranges, state_db, nimcrypto, eth_trie/[types, hexary], eth_common, byteutils,
+  rlp, ranges, state_db, nimcrypto, eth_trie/[hexary, db], eth_common, byteutils,
   ../errors, ../block_types, ../utils/header, ../constants, ./storage_types.nim
 
 type

nimbus/db/state_db.nim

@@ -7,7 +7,7 @@
 
 import
   sequtils, strformat, tables,
-  chronicles, eth_common, nimcrypto, rlp, eth_trie/[hexary, memdb],
+  chronicles, eth_common, nimcrypto, rlp, eth_trie/[hexary, db],
   ../constants, ../errors, ../validation, ../account
 
 logScope:

nimbus/genesis.nim

@@ -1,6 +1,8 @@
-import db/[db_chain, state_db], genesis_alloc, eth_common, tables, stint,
-    byteutils, times, config, rlp, ranges, block_types, eth_trie,
-    eth_trie/memdb, account, constants, nimcrypto, chronicles
+import
+  times, tables,
+  eth_common, stint, byteutils, rlp, ranges, block_types, nimcrypto,
+  chronicles, eth_trie, eth_trie/db,
+  db/[db_chain, state_db], genesis_alloc, config, account, constants
 
 type
   Genesis* = object

nimbus/nimbus.nim

@@ -12,7 +12,7 @@ import
   asyncdispatch2, json_rpc/rpcserver, eth_keys,
   eth_p2p, eth_p2p/rlpx_protocols/[eth],
   config, genesis, rpc/[common, p2p], p2p/chain,
-  eth_trie
+  eth_trie/db
 
 const UseSqlite = false
 

nimbus/p2p/chain.nim

@@ -1,10 +1,9 @@
 import ../db/[db_chain, state_db], eth_common, chronicles, ../vm_state, ../vm_types, ../transaction, ranges,
   ../vm/[computation, interpreter_dispatch, message], ../constants, stint, nimcrypto,
   ../vm_state_transactions,
-  eth_trie/memdb, eth_trie, rlp,
+  eth_trie/db, eth_trie, rlp,
   sugar
 
-
 type
   Chain* = ref object of AbstractChainDB
     db: BaseChainDB
@@ -101,7 +100,7 @@ proc processTransaction(db: var AccountStateDB, t: Transaction, sender: EthAddre
   return gasUsed.u256 * t.gasPrice.u256
 
 proc calcTxRoot(transactions: openarray[Transaction]): Hash256 =
-  var tr = initHexaryTrie(trieDB(newMemDB()))
+  var tr = initHexaryTrie(newMemoryDB())
   for i, t in transactions:
     tr.put(rlp.encode(i).toRange, rlp.encode(t).toRange)
   return tr.rootHash
@@ -112,6 +111,9 @@ method persistBlocks*(c: Chain, headers: openarray[BlockHeader], bodies: openarr
 
   let blockReward = 5.u256 * pow(10.u256, 18) # 5 ETH
 
+  let transaction = c.db.db.beginTransaction()
+  defer: transaction.dispose()
+
   echo "Persisting blocks: ", headers[0].blockNumber, " - ", headers[^1].blockNumber
   for i in 0 ..< headers.len:
     let head = c.db.getCanonicalHead()
@@ -158,6 +160,8 @@ method persistBlocks*(c: Chain, headers: openarray[BlockHeader], bodies: openarr
       echo "Wrong state root in block ", headers[i].blockNumber, ". Expected: ", headers[i].stateRoot, ", Actual: ", stateDb.rootHash, " arrived from ", c.db.getCanonicalHead().stateRoot
     assert(headers[i].stateRoot == stateDb.rootHash)
 
- 
     discard c.db.persistHeaderToDb(headers[i])
     assert(c.db.getCanonicalHead().blockHash == headers[i].blockHash)
+
+  transaction.commit()
+

nimbus/rpc/common.nim

@@ -6,10 +6,11 @@
 # at your option.
 # This file may not be copied, modified, or distributed except according to
 # those terms.
-import strutils, nimcrypto, eth_common, stint, eth_trie/[memdb, types]
+
 import
-  json_rpc/server, ../vm_state, ../db/[db_chain, state_db],
-  ../constants, ../config, hexstrings
+  strutils,
+  nimcrypto, eth_common, stint, json_rpc/server,
+  ../vm_state, ../db/[db_chain, state_db], ../constants, ../config, hexstrings
 
 proc setupCommonRPC*(server: RpcServer) =
   server.rpc("web3_clientVersion") do() -> string:

nimbus/rpc/p2p.nim

@@ -6,11 +6,14 @@
 # at your option.
 # This file may not be copied, modified, or distributed except according to
 # those terms.
+
 import
-  nimcrypto, json_rpc/rpcserver, eth_p2p, hexstrings, strutils, stint,
-  ../config, ../vm_state, ../constants, eth_trie/[memdb, types], eth_keys,
-  ../db/[db_chain, state_db, storage_types], eth_common, rpc_types, byteutils,
-  ranges/typedranges, times, ../utils/header, rlp, ../transaction
+  strutils, times,
+  nimcrypto, json_rpc/rpcserver, hexstrings, stint, byteutils, ranges/typedranges,
+  eth_common, eth_p2p, eth_keys, eth_trie/db, rlp,
+  ../utils/header, ../transaction, ../config, ../vm_state, ../constants,
+  ../db/[db_chain, state_db, storage_types],
+  rpc_types
 
 #[
   Note:
@@ -52,14 +55,14 @@ func headerFromTag(chain:BaseChainDB, blockTag: string): BlockHeader =
     result = chain.getBlockHeader(blockNum)
 
 proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
-  
-  func getAccountDb(header: BlockHeader, readOnly = true): AccountStateDb = 
+
+  func getAccountDb(header: BlockHeader, readOnly = true): AccountStateDb =
     ## Retrieves the account db from canonical head
     let vmState = newBaseVMState(header, chain)
     result = vmState.chaindb.getStateDb(vmState.blockHeader.hash, readOnly)
 
   func accountDbFromTag(tag: string, readOnly = true): AccountStateDb =
-    result = getAccountDb(chain.headerFromTag(tag))  
+    result = getAccountDb(chain.headerFromTag(tag))
 
   proc getBlockBody(hash: KeccakHash): BlockBody =
     if not chain.getBlockBody(hash, result):
@@ -68,7 +71,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
   rpcsrv.rpc("net_version") do() -> uint:
     let conf = getConfiguration()
     result = conf.net.networkId
-  
+
   rpcsrv.rpc("eth_syncing") do() -> JsonNode:
     ## Returns SyncObject or false when not syncing.
     # TODO: Requires PeerPool to check sync state.
@@ -84,7 +87,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
       result = %sync
     else:
       result = newJBool(false)
-  
+
   rpcsrv.rpc("eth_coinbase") do() -> EthAddress:
     ## Returns the current coinbase address.
     result = chain.getCanonicalHead().coinbase
@@ -238,7 +241,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
     ## Generates and returns an estimate of how much gas is necessary to allow the transaction to complete.
     ## The transaction will not be added to the blockchain. Note that the estimate may be significantly more than
     ## the amount of gas actually used by the transaction, for a variety of reasons including EVM mechanics and node performance.
-    ## 
+    ##
     ## call: the transaction call object.
     ## quantityTag:  integer block number, or the string "latest", "earliest" or "pending", see the default block parameter.
     ## Returns the amount of gas used.
@@ -256,7 +259,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
       startIdx = 0
     for i in 0 ..< blockBody.uncles.len:
       rawData[startIdx .. startIdx + 32] = blockBody.uncles[i].hash.data
-      startIdx += 32    
+      startIdx += 32
     result.sha3Uncles = keccak256.digest(rawData)
 
     result.logsBloom = some(header.bloom)
@@ -317,7 +320,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
     result.gasPrice = transaction.gasPrice
     result.gas = accountGas
     result.input = transaction.payload
-  
+
   rpcsrv.rpc("eth_getTransactionByHash") do(data: HexDataStr) -> TransactionObject:
     ## Returns the information about a transaction requested by transaction hash.
     ##
@@ -393,7 +396,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
       idx.inc
 
   rpcsrv.rpc("eth_getUncleByBlockHashAndIndex") do(data: HexDataStr, quantity: int) -> Option[BlockObject]:
-    ## Returns information about a uncle of a block by hash and uncle index position.  
+    ## Returns information about a uncle of a block by hash and uncle index position.
     ##
     ## data: hash of block.
     ## quantity: the uncle's index position.
@@ -428,12 +431,12 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
     ## [A] "A in first position (and anything after)"
     ## [null, B] "anything in first position AND B in second position (and anything after)"
     ## [A, B] "A in first position AND B in second position (and anything after)"
-    ## [[A, B], [A, B]] "(A OR B) in first position AND (A OR B) in second position (and anything after)"  
+    ## [[A, B], [A, B]] "(A OR B) in first position AND (A OR B) in second position (and anything after)"
     ##
     ## filterOptions: settings for this filter.
     ## Returns integer filter id.
     discard
-  
+
   rpcsrv.rpc("eth_newBlockFilter") do() -> int:
     ## Creates a filter in the node, to notify when a new block arrives.
     ## To check if the state has changed, call eth_getFilterChanges.
@@ -449,7 +452,7 @@ proc setupEthRpc*(node: EthereumNode, chain: BaseChainDB, rpcsrv: RpcServer) =
     discard
 
   rpcsrv.rpc("eth_uninstallFilter") do(filterId: int) -> bool:
-    ## Uninstalls a filter with given id. Should always be called when watch is no longer needed. 
+    ## Uninstalls a filter with given id. Should always be called when watch is no longer needed.
     ## Additonally Filters timeout when they aren't requested with eth_getFilterChanges for a period of time.
     ##
     ## filterId: The filter id.

tests/test_caching_db_backend.nim

@@ -1,5 +1,8 @@
-
-import ../nimbus/db/backends/caching_backend, eth_trie, eth_trie/memdb, unittest
+#[
+import
+  unittest,
+  eth_trie/db,
+  ../nimbus/db/backends/caching_backend
 
 let
   key1 = [0.byte, 0, 1]
@@ -42,4 +45,4 @@ suite "Caching DB backend":
       mdb.get(key2) == @value2
       mdb.get(key3) == @value3
       key4 notin mdb
-
+]#

tests/test_generalstate_json.nim

@@ -8,7 +8,7 @@
 import
   unittest, strformat, strutils, tables, json, ospaths, times,
   byteutils, ranges/typedranges, nimcrypto/[keccak, hash],
-  rlp, eth_trie/[types, memdb], eth_common,
+  rlp, eth_trie/db, eth_common,
   eth_keys,
   ./test_helpers,
   ../nimbus/[constants, errors],

tests/test_opcode.nim

@@ -7,7 +7,7 @@
 
 import
   unittest, tables, parseutils, byteutils,
-  eth_trie/[types, memdb], eth_common/eth_types,
+  eth_trie/db, eth_common/eth_types,
   ../nimbus/[constants, vm_types, vm_state],
   ../nimbus/vm/interpreter,
   ../nimbus/utils/header,
@@ -18,7 +18,6 @@ from eth_common import GasInt
 
 proc testCode(code: string, initialGas: GasInt, blockNum: UInt256): BaseComputation =
   let header = BlockHeader(blockNumber: blockNum)
-  var memDb = newMemDB()
   var vmState = newBaseVMState(header, newBaseChainDB(newMemoryDb()))
 
   # coinbase: "",

tests/test_vm_json.nim

@@ -8,7 +8,7 @@
 import
   unittest, strformat, strutils, sequtils, tables, json, ospaths, times,
   byteutils, ranges/typedranges, nimcrypto/[keccak, hash],
-  rlp, eth_trie/[types, memdb], eth_common,
+  rlp, eth_trie/db, eth_common,
   ./test_helpers,
   ../nimbus/[constants, errors],
   ../nimbus/[vm_state, vm_types],
@@ -39,7 +39,6 @@ proc testFixture(fixtures: JsonNode, testStatusIMPL: var TestStatus) =
     stateRoot: emptyRlpHash
     )
 
-  var memDb = newMemDB()
   var vmState = newBaseVMState(header, newBaseChainDB(newMemoryDB()))
   let fexec = fixture["exec"]
   var code: seq[byte]