Jordan/accounts cache scenario (#904)

* crash test scenario

details:
  Example code for inspecting nested block chain and accounts cache
  database transaction framework. There seems to be a pathological
  case where the system crashes after a rollback (as appeared in the
  tx-pool packer code.)

* simplified crash scenario

* Workable solution (as suggested by Andri)

details:
  Avoiding db.rollback() (db.commit() is OK) while vmState.stateDB is
  alive.

* Rename text_txcrash => test_accounts_cache

why:
  Unit tests covers part of accounts_cache handling

* comment update

tests/all_tests.nim

@@ -11,6 +11,7 @@ import ../test_macro
 
 cliBuilder:
   import  ./test_code_stream,
+          ./test_accounts_cache,
           ./test_gas_meter,
           ./test_memory,
           ./test_stack,

tests/test_accounts_cache.nim

@@ -0,0 +1,379 @@
+# Nimbus
+# Copyright (c) 2018-2019 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+import
+  std/[os, sequtils, strformat, strutils, tables],
+  ../nimbus/[chain_config, config, constants, genesis],
+  ../nimbus/db/[accounts_cache, db_chain],
+  ../nimbus/p2p/chain,
+  ../nimbus/transaction,
+  ../nimbus/vm_state,
+  ../nimbus/vm_types,
+  ./test_clique/undump,
+  eth/[common, p2p, trie/db],
+  unittest2
+
+type
+  CaptureSpecs = tuple
+    network: NetworkID
+    file: string
+    numBlocks: int
+    numTxs: int
+
+const
+  baseDir = [".", "tests", ".." / "tests", $DirSep] # path containg repo
+  repoDir = ["replay", "status", "test_clique"]     # alternative repo paths
+
+  goerliCapture: CaptureSpecs = (
+    network: GoerliNet,
+    # file: "goerli68161.txt.gz",
+    file: "goerli51840.txt.gz",
+    numBlocks: 5500,  # unconditionally load blocks
+    numTxs:      10)  # txs following (not in block chain)
+
+  goerliCapture1: CaptureSpecs = (
+    GoerliNet, goerliCapture.file, 5500, 10000)
+
+  mainCapture: CaptureSpecs = (
+    MainNet, "mainnet843841.txt.gz", 50000, 3000)
+
+var
+  xdb: BaseChainDB
+  txs: seq[Transaction]
+  txi: seq[int] # selected index into txs[] (crashable sender addresses)
+
+# ------------------------------------------------------------------------------
+# Helpers
+# ------------------------------------------------------------------------------
+
+proc findFilePath(file: string): string =
+  result = "?unknown?" / file
+  for dir in baseDir:
+    for repo in repoDir:
+      let path = dir / repo / file
+      if path.fileExists:
+        return path
+
+proc pp*(a: EthAddress): string =
+  a.mapIt(it.toHex(2)).join[32 .. 39].toLowerAscii
+
+proc pp*(tx: Transaction): string =
+  # "(" & tx.ecRecover.value.pp & "," & $tx.nonce & ")"
+  "(" & tx.getSender.pp & "," & $tx.nonce & ")"
+
+proc pp*(h: KeccakHash): string =
+  h.data.mapIt(it.toHex(2)).join[52 .. 63].toLowerAscii
+
+proc pp*(tx: Transaction; vmState: BaseVMState): string =
+  let address = tx.getSender
+  "(" & address.pp &
+    "," & $tx.nonce &
+    ";" & $vmState.readOnlyStateDB.getNonce(address) &
+    "," & $vmState.readOnlyStateDB.getBalance(address) &
+    ")"
+
+# ------------------------------------------------------------------------------
+# Private functions
+# ------------------------------------------------------------------------------
+
+proc blockChainForTesting*(network: NetworkID): BaseChainDB =
+  result = newBaseChainDB(
+    newMemoryDb(),
+    id = network,
+    params = network.networkParams)
+  result.populateProgress
+  initializeEmptyDB(result)
+
+proc importBlocks(cdb: BaseChainDB; h: seq[BlockHeader]; b: seq[BlockBody]) =
+  if cdb.newChain.persistBlocks(h,b) != ValidationResult.OK:
+    raiseAssert "persistBlocks() failed at block #" & $h[0].blockNumber
+
+proc getVmState(cdb: BaseChainDB; number: BlockNumber): BaseVMState =
+  let
+    topHeader = cdb.getBlockHeader(number)
+    accounts = AccountsCache.init(cdb.db, topHeader.stateRoot, cdb.pruneTrie)
+  result = accounts.newBaseVMState(topHeader, cdb)
+
+# ------------------------------------------------------------------------------
+# Crash test function, finding out about how the transaction framework works ..
+# ------------------------------------------------------------------------------
+
+proc modBalance(ac: var AccountsCache, address: EthAddress) =
+  ## This function is crucial for profucing the crash. If must
+  ## modify the balance so that the database gets written.
+  # ac.blindBalanceSetter(address)
+  ac.addBalance(address, 1.u256)
+
+
+proc runTrial2ok(vmState: BaseVMState; inx: int) =
+  ## Run two blocks, the first one with *rollback*.
+  let eAddr = txs[inx].getSender
+
+  block:
+    let accTx = vmState.stateDB.beginSavepoint
+    vmState.stateDB.modBalance(eAddr)
+    vmState.stateDB.rollback(accTx)
+
+  block:
+    let accTx = vmState.stateDB.beginSavepoint
+    vmState.stateDB.modBalance(eAddr)
+    vmState.stateDB.commit(accTx)
+
+  vmState.stateDB.persist(clearCache = false)
+
+
+proc runTrial3(vmState: BaseVMState; inx: int; rollback: bool) =
+  ## Run three blocks, the second one optionally with *rollback*.
+  let eAddr = txs[inx].getSender
+
+  block:
+    let accTx = vmState.stateDB.beginSavepoint
+    vmState.stateDB.modBalance(eAddr)
+    vmState.stateDB.commit(accTx)
+    vmState.stateDB.persist(clearCache = false)
+
+  block:
+    let accTx = vmState.stateDB.beginSavepoint
+    vmState.stateDB.modBalance(eAddr)
+
+    if rollback:
+      vmState.stateDB.rollback(accTx)
+      break
+
+    vmState.stateDB.commit(accTx)
+    vmState.stateDB.persist(clearCache = false)
+
+  block:
+    let accTx = vmState.stateDB.beginSavepoint
+    vmState.stateDB.modBalance(eAddr)
+    vmState.stateDB.commit(accTx)
+    vmState.stateDB.persist(clearCache = false)
+
+
+proc runTrial3crash(vmState: BaseVMState; inx: int; noisy = false) =
+  ## Run three blocks with extra db frames and *rollback*.
+  let eAddr = txs[inx].getSender
+
+  block:
+    let dbTx = xdb.db.beginTransaction()
+
+    block:
+      let accTx = vmState.stateDB.beginSavepoint
+      vmState.stateDB.modBalance(eAddr)
+      vmState.stateDB.commit(accTx)
+      vmState.stateDB.persist(clearCache = false)
+
+    block:
+      let accTx = vmState.stateDB.beginSavepoint
+      vmState.stateDB.modBalance(eAddr)
+      vmState.stateDB.rollback(accTx)
+
+    # The following statement will cause a crash at the next `persist()` call.
+    dbTx.rollback()
+
+  # In order to survive without an exception in the next `persist()` call, the
+  # following function could be added to db/accounts_cache.nim:
+  #
+  #   proc clobberRootHash*(ac: AccountsCache; root: KeccakHash; prune = true) =
+  #     ac.trie = initSecureHexaryTrie(ac.db, rootHash, prune)
+  #
+  # Then, beginning this very function `runTrial3crash()` with
+  #
+  #   let stateRoot = vmState.stateDB.rootHash
+  #
+  # the survival statement would be to re-assign the state-root via
+  #
+  #   vmState.stateDB.clobberRootHash(stateRoot)
+  #
+  # Also mind this comment from Andri:
+  #
+  #   [..] but as a reminder, only reinit the ac.trie is not enough, you
+  #   should consider the accounts in the cache too. if there is any accounts
+  #   in the cache they must in sync with the new rootHash.
+  #
+  block:
+    let dbTx = xdb.db.beginTransaction()
+
+    block:
+      let accTx = vmState.stateDB.beginSavepoint
+      vmState.stateDB.modBalance(eAddr)
+      vmState.stateDB.commit(accTx)
+
+      try:
+        vmState.stateDB.persist(clearCache = false)
+      except AssertionError as e:
+        if noisy:
+          let msg = e.msg.rsplit($DirSep,1)[^1]
+          echo &"*** runVmExec({eAddr.pp}): {e.name}: {msg}"
+        dbTx.dispose()
+        raise e
+
+      vmState.stateDB.persist(clearCache = false)
+
+    dbTx.commit()
+
+
+proc runTrial4(vmState: BaseVMState; inx: int; rollback: bool) =
+  ## Like `runTrial3()` but with four blocks and extra db transaction frames.
+  let eAddr = txs[inx].getSender
+
+  block:
+    let dbTx = xdb.db.beginTransaction()
+
+    block:
+      let accTx = vmState.stateDB.beginSavepoint
+      vmState.stateDB.modBalance(eAddr)
+      vmState.stateDB.commit(accTx)
+      vmState.stateDB.persist(clearCache = false)
+
+    block:
+      let accTx = vmState.stateDB.beginSavepoint
+      vmState.stateDB.modBalance(eAddr)
+      vmState.stateDB.commit(accTx)
+      vmState.stateDB.persist(clearCache = false)
+
+    block:
+      let accTx = vmState.stateDB.beginSavepoint
+      vmState.stateDB.modBalance(eAddr)
+
+      if rollback:
+        vmState.stateDB.rollback(accTx)
+        break
+
+      vmState.stateDB.commit(accTx)
+      vmState.stateDB.persist(clearCache = false)
+
+    # There must be no dbTx.rollback() here unless `vmState.stateDB` is
+    # discarded and/or re-initialised.
+    dbTx.commit()
+
+  block:
+    let dbTx = xdb.db.beginTransaction()
+
+    block:
+      let accTx = vmState.stateDB.beginSavepoint
+      vmState.stateDB.modBalance(eAddr)
+      vmState.stateDB.commit(accTx)
+      vmState.stateDB.persist(clearCache = false)
+
+    dbTx.commit()
+
+# ------------------------------------------------------------------------------
+# Test Runner
+# ------------------------------------------------------------------------------
+
+proc runner(noisy = true; capture = goerliCapture) =
+  let
+    loadBlocks = capture.numBlocks.u256
+    loadTxs = capture.numTxs
+    fileInfo = capture.file.splitFile.name.split(".")[0]
+    filePath = capture.file.findFilePath
+
+  txs.reset
+  xdb = capture.network.blockChainForTesting
+
+  suite &"StateDB nesting scenarios":
+    var topNumber: BlockNumber
+
+    test &"Import from {fileInfo}":
+      # Import minimum amount of blocks, then collect transactions
+      for chain in filePath.undumpNextGroup:
+        let leadBlkNum = chain[0][0].blockNumber
+        topNumber = chain[0][^1].blockNumber
+
+        if loadTxs <= txs.len:
+          break
+
+        # Verify Genesis
+        if leadBlkNum == 0.u256:
+          doAssert chain[0][0] == xdb.getBlockHeader(0.u256)
+          continue
+
+        # Import block chain blocks
+        if leadBlkNum < loadBlocks:
+          xdb.importBlocks(chain[0],chain[1])
+          continue
+
+        # Import transactions
+        for inx in 0 ..< chain[0].len:
+          let blkTxs = chain[1][inx].transactions
+
+          # Continue importing up until first non-trivial block
+          if txs.len == 0 and blkTxs.len == 0:
+            xdb.importBlocks(@[chain[0][inx]],@[chain[1][inx]])
+            continue
+
+          # Load transactions
+          txs.add blkTxs
+
+
+    test &"Collect unique sender addresses from {txs.len} txs," &
+        &" head=#{xdb.getCanonicalHead.blockNumber}, top=#{topNumber}":
+      var seen: Table[EthAddress,bool]
+      for n,tx in txs:
+        let a = tx.getSender
+        if not seen.hasKey(a):
+          seen[a] = true
+          txi.add n
+
+    test &"Run {txi.len} two-step trials with rollback":
+      let dbTx = xdb.db.beginTransaction()
+      defer: dbTx.dispose()
+      for n in txi:
+        let vmState = xdb.getVmState(xdb.getCanonicalHead.blockNumber)
+        vmState.runTrial2ok(n)
+
+    test &"Run {txi.len} three-step trials with rollback":
+      let dbTx = xdb.db.beginTransaction()
+      defer: dbTx.dispose()
+      for n in txi:
+        let vmState = xdb.getVmState(xdb.getCanonicalHead.blockNumber)
+        vmState.runTrial3(n, rollback = true)
+
+    test &"Run {txi.len} three-step trials with extra db frame rollback" &
+        " throwing Exceptions":
+      let dbTx = xdb.db.beginTransaction()
+      defer: dbTx.dispose()
+      for n in txi:
+        let vmState = xdb.getVmState(xdb.getCanonicalHead.blockNumber)
+        expect AssertionError:
+          vmState.runTrial3crash(n, noisy)
+
+    test &"Run {txi.len} tree-step trials without rollback":
+      let dbTx = xdb.db.beginTransaction()
+      defer: dbTx.dispose()
+      for n in txi:
+        let vmState = xdb.getVmState(xdb.getCanonicalHead.blockNumber)
+        vmState.runTrial3(n, rollback = false)
+
+    test &"Run {txi.len} four-step trials with rollback and db frames":
+      let dbTx = xdb.db.beginTransaction()
+      defer: dbTx.dispose()
+      for n in txi:
+        let vmState = xdb.getVmState(xdb.getCanonicalHead.blockNumber)
+        vmState.runTrial4(n, rollback = true)
+
+# ------------------------------------------------------------------------------
+# Main function(s)
+# ------------------------------------------------------------------------------
+
+proc accountsCacheMain*(noisy = defined(debug)) =
+  noisy.runner
+
+when isMainModule:
+  var noisy = defined(debug)
+  #noisy = true
+
+  noisy.runner # mainCapture
+  # noisy.runner goerliCapture2
+
+# ------------------------------------------------------------------------------
+# End
+# ------------------------------------------------------------------------------

vendor/nim-stew

@@ -1 +1 @@
-Subproject commit a61869c256725c077222c861029292db16b35541
+Subproject commit 3b9e906a185a8bca3ac3abf39a8e8befe8bf6dcc