multi root state trie implementation

nimbus/db/state_db.nim

@@ -17,20 +17,25 @@ type
   AccountStateDB* = ref object
     trie: SecureHexaryTrie
     originalRoot: KeccakHash   # will be updated for every transaction
+    transactionID: TransactionID
 
   ReadOnlyStateDB* = distinct AccountStateDB
 
+template trieDB(stateDB: AccountStateDB): TrieDatabaseRef =
+  HexaryTrie(stateDB.trie).db
+
 proc rootHash*(db: AccountStateDB): KeccakHash =
   db.trie.rootHash
 
 proc `rootHash=`*(db: AccountStateDB, root: KeccakHash) =
-  db.trie = initSecureHexaryTrie(HexaryTrie(db.trie).db, root, db.trie.isPruning)
+  db.trie = initSecureHexaryTrie(trieDB(db), root, db.trie.isPruning)
 
 proc newAccountStateDB*(backingStore: TrieDatabaseRef,
                         root: KeccakHash, pruneTrie: bool): AccountStateDB =
   result.new()
   result.trie = initSecureHexaryTrie(backingStore, root, pruneTrie)
   result.originalRoot = root
+  result.transactionID = backingStore.getTransactionID()
 
 template createRangeFromAddress(address: EthAddress): ByteRange =
   ## XXX: The name of this proc is intentionally long, because it
@@ -80,13 +85,13 @@ template createTrieKeyFromSlot(slot: UInt256): ByteRange =
   # pad32(int_to_big_endian(slot))
   # morally equivalent to toByteRange_Unnecessary but with different types
 
-template getAccountTrie(stateDb: AccountStateDB, account: Account): auto =
+template getAccountTrie(db: AccountStateDB, account: Account): auto =
   # TODO: implement `prefix-db` to solve issue #228 permanently.
   # the `prefix-db` will automatically insert account address to the
   # underlying-db key without disturb how the trie works.
   # it will create virtual container for each account.
   # see nim-eth#9
-  initSecureHexaryTrie(HexaryTrie(stateDb.trie).db, account.storageRoot, false)
+  initSecureHexaryTrie(trieDB(db), account.storageRoot, false)
 
 # XXX: https://github.com/status-im/nimbus/issues/142#issuecomment-420583181
 proc setStorageRoot*(db: var AccountStateDB, address: EthAddress, storageRoot: Hash256) =
@@ -114,7 +119,7 @@ proc setStorage*(db: var AccountStateDB,
   # map slothash back to slot value
   # see iterator storage below
   var
-    triedb = HexaryTrie(db.trie).db
+    triedb = trieDB(db)
     # slotHash can be obtained from accountTrie.put?
     slotHash = keccakHash(slot.toByteArrayBE)
   triedb.put(slotHashToSlotKey(slotHash.data).toOpenArray, rlp.encode(slot))
@@ -125,7 +130,7 @@ proc setStorage*(db: var AccountStateDB,
 iterator storage*(db: AccountStateDB, address: EthAddress): (UInt256, UInt256) =
   let
     storageRoot = db.getStorageRoot(address)
-    triedb = HexaryTrie(db.trie).db
+    triedb = trieDB(db)
   var trie = initHexaryTrie(triedb, storageRoot)
 
   for key, value in trie:
@@ -167,7 +172,7 @@ proc setCode*(db: AccountStateDB, address: EthAddress, code: ByteRange) =
 
   let
     newCodeHash = keccakHash(code.toOpenArray)
-    triedb = HexaryTrie(db.trie).db
+    triedb = trieDB(db)
 
   if code.len != 0:
     triedb.put(contractHashKey(newCodeHash).toOpenArray, code.toOpenArray)
@@ -176,7 +181,7 @@ proc setCode*(db: AccountStateDB, address: EthAddress, code: ByteRange) =
   db.setAccount(address, account)
 
 proc getCode*(db: AccountStateDB, address: EthAddress): ByteRange =
-  let triedb = HexaryTrie(db.trie).db
+  let triedb = trieDB(db)
   let data = triedb.get(contractHashKey(db.getCodeHash(address)).toOpenArray)
   data.toRange
 
@@ -213,12 +218,16 @@ proc getCommittedStorage*(db: AccountStateDB, address: EthAddress, slot: UInt256
   let tmpHash = db.rootHash
   db.rootHash = db.originalRoot
   var exists: bool
+  shortTimeReadOnly(trieDB(db), db.transactionID):
     (result, exists) = db.getStorage(address, slot)
   db.rootHash = tmpHash
 
 proc updateOriginalRoot*(db: AccountStateDB) =
   ## this proc will be called for every transaction
   db.originalRoot = db.rootHash
+  # no need to rollback or dispose
+  # transactionID, it will be handled elsewhere
+  db.transactionID = trieDB(db).getTransactionID()
 
 proc rootHash*(db: ReadOnlyStateDB): KeccakHash {.borrow.}
 proc getAccount*(db: ReadOnlyStateDB, address: EthAddress): Account {.borrow.}

nimbus/p2p/executor.nim

@@ -1,5 +1,5 @@
 import options, sets,
-  eth/[common, bloom], stew/ranges, chronicles, nimcrypto,
+  eth/[common, bloom, trie/db], stew/ranges, chronicles, nimcrypto,
   ../db/[db_chain, state_db],
   ../utils, ../constants, ../transaction,
   ../vm_state, ../vm_types, ../vm_state_transactions,
@@ -119,6 +119,9 @@ const
   ]
 
 proc processBlock*(chainDB: BaseChainDB, header: BlockHeader, body: BlockBody, vmState: BaseVMState): ValidationResult =
+  var dbTx = chainDB.db.beginTransaction()
+  defer: dbTx.dispose()
+
   if chainDB.config.daoForkSupport and header.blockNumber == chainDB.config.daoForkBlock:
     vmState.mutateStateDB:
       db.applyDAOHardFork()
@@ -183,3 +186,8 @@ proc processBlock*(chainDB: BaseChainDB, header: BlockHeader, body: BlockBody, v
   if header.receiptRoot != receiptRoot:
     debug "wrong receiptRoot in block", blockNumber=header.blockNumber, actual=receiptRoot, expected=header.receiptRoot
     return ValidationResult.Error
+
+  # `applyDeletes = false`
+  # preserve previous block stateRoot
+  # while still benefits from trie pruning
+  dbTx.commit(applyDeletes = false)

tests/test_blockchain_json.nim

@@ -271,7 +271,10 @@ proc assignBlockRewards(minedBlock: PlainBlock, vmState: BaseVMState, fork: Fork
   if minedBlock.header.txRoot != txRoot:
     raise newException(ValidationError, "wrong txRoot")
 
-proc processBlock(vmState: BaseVMState, minedBlock: PlainBlock, fork: Fork) =
+proc processBlock(chainDB: BaseChainDB, vmState: BaseVMState, minedBlock: PlainBlock, fork: Fork) =
+  var dbTx = chainDB.db.beginTransaction()
+  defer: dbTx.dispose()
+
   vmState.receipts = newSeq[Receipt](minedBlock.transactions.len)
   vmState.cumulativeGasUsed = 0
 
@@ -288,6 +291,11 @@ proc processBlock(vmState: BaseVMState, minedBlock: PlainBlock, fork: Fork) =
 
   assignBlockRewards(minedBlock, vmState, fork, vmState.chainDB)
 
+  # `applyDeletes = false`
+  # preserve previous block stateRoot
+  # while still benefits from trie pruning
+  dbTx.commit(applyDeletes = false)
+
 func validateBlockUnchanged(a, b: PlainBlock): bool =
   result = rlp.encode(a) == rlp.encode(b)
 
@@ -513,7 +521,8 @@ proc importBlock(tester: var Tester, chainDB: BaseChainDB,
   let tracerFlags: set[TracerFlags] = if tester.trace: {TracerFlags.EnableTracing} else : {}
   tester.vmState = newBaseVMState(parentHeader.stateRoot, baseHeaderForImport, chainDB, tracerFlags)
 
-  processBlock(tester.vmState, result, fork)
+  processBlock(chainDB, tester.vmState, result, fork)
+
   result.header.stateRoot = tester.vmState.blockHeader.stateRoot
   result.header.parentHash = parentHeader.hash
   result.header.difficulty = baseHeaderForImport.difficulty
@@ -626,9 +635,8 @@ proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus, debugMode = fal
       continue
 
     var tester = parseTester(fixture, testStatusIMPL)
-    # TODO: implement journalDB in AccountStateDB
+    # TODO: do we need another test with pruneTrie = false?
-    # then turn on state trie pruning
+    var chainDB = newBaseChainDB(newMemoryDb(), pruneTrie = false)
-    var chainDB = newBaseChainDB(newMemoryDb(), false)
 
     echo "TESTING: ", fixtureName
     if not tester.good: continue

tests/test_generalstate_json.nim

@@ -87,9 +87,10 @@ proc dumpDebugData(tester: Tester, vmState: BaseVMState, sender: EthAddress, gas
 
 proc testFixtureIndexes(tester: Tester, testStatusIMPL: var TestStatus) =
   var tracerFlags: set[TracerFlags] = if tester.trace: {TracerFlags.EnableTracing} else : {}
-  # TODO: implement journalDB in AccountStateDB
+
-  # then turn on state trie pruning
+  # TODO: do we need another test with pruneTrie = false?
-  var vmState = newGST_VMState(emptyRlpHash, tester.header, newBaseChainDB(newMemoryDb(), false), tracerFlags)
+  var chainDB = newBaseChainDB(newMemoryDb(), pruneTrie = true)
+  var vmState = newGST_VMState(emptyRlpHash, tester.header, chainDB, tracerFlags)
   var gasUsed: GasInt
   let sender = tester.tx.getSender()