nimbus-eth1/tests/test_aristo.nim

# Nimbus - Types, data structures and shared utilities used in network sync
#
# Copyright (c) 2018-2021 Status Research & Development GmbH
# Licensed under either of
#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
#    http://www.apache.org/licenses/LICENSE-2.0)
#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
#    http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or
# distributed except according to those terms.

## Re-invented implementation for Merkle Patricia Tree named as Aristo Trie

import
  std/[os, strformat, strutils],
  chronicles,
  eth/[common, p2p],
  rocksdb,
  unittest2,
  ../nimbus/db/aristo/[aristo_desc, aristo_merge],
  ../nimbus/db/core_db,
  ../nimbus/db/core_db/[legacy_persistent, persistent],
  ../nimbus/core/chain,
  ../nimbus/sync/snap/worker/db/[rocky_bulk_load, snapdb_accounts, snapdb_desc],
  ./replay/[pp, undump_accounts, undump_storages],
  ./test_sync_snap/[snap_test_xx, test_accounts, test_types],
  ./test_aristo/[
    test_backend, test_filter, test_helpers, test_transcode, test_tx]

const
  baseDir = [".", "..", ".."/"..", $DirSep]
  repoDir = [".", "tests", "nimbus-eth1-blobs"]
  subDir = ["replay", "test_sync_snap", "replay"/"snap"]

  # Reference file for finding the database directory
  sampleDirRefFile = "sample0.txt.gz"

  # Standard test samples
  accSample = snapTest0
  storSample = snapTest4

  # Number of database slots available
  nTestDbInstances = 9

type
  TestDbs = object
    ## Provide enough spare empty databases
    persistent: bool
    dbDir: string
    baseDir: string # for cleanup
    subDir: string  # for cleanup
    cdb: array[nTestDbInstances,CoreDbRef]

# ------------------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------------------

proc findFilePath(
     file: string;
     baseDir: openArray[string] = baseDir;
     repoDir: openArray[string] = repoDir;
     subDir: openArray[string] = subDir;
       ): Result[string,void] =
  for dir in baseDir:
    if dir.dirExists:
      for repo in repoDir:
        if (dir / repo).dirExists:
          for sub in subDir:
            if (dir / repo / sub).dirExists:
              let path = dir / repo / sub / file
              if path.fileExists:
                return ok(path)
  echo "*** File not found \"", file, "\"."
  err()

proc getTmpDir(sampleDir = sampleDirRefFile): string =
  sampleDir.findFilePath.value.splitFile.dir

proc setTraceLevel {.used.} =
  discard
  when defined(chronicles_runtime_filtering) and loggingEnabled:
    setLogLevel(LogLevel.TRACE)

proc setErrorLevel {.used.} =
  discard
  when defined(chronicles_runtime_filtering) and loggingEnabled:
    setLogLevel(LogLevel.ERROR)

# ------------------------------------------------------------------------------
# Private functions
# ------------------------------------------------------------------------------

proc flushDbDir(s: string; subDir = "") =
  if s != "":
    let baseDir = s / "tmp"
    for n in 0 ..< nTestDbInstances:
      let instDir = if subDir == "": baseDir / $n else: baseDir / subDir / $n
      if (instDir / "nimbus" / "data").dirExists:
        # Typically under Windows: there might be stale file locks.
        try: instDir.removeDir except CatchableError: discard
    try: (baseDir / subDir).removeDir except CatchableError: discard
    block dontClearUnlessEmpty:
      for w in baseDir.walkDir:
        break dontClearUnlessEmpty
      try: baseDir.removeDir except CatchableError: discard


proc flushDbs(db: TestDbs) =
  if db.persistent:
    for n in 0 ..< nTestDbInstances:
      if db.cdb[n].isNil or db.cdb[n].dbType != LegacyDbPersistent:
        break
      db.cdb[n].toLegacyBackend.rocksStoreRef.store.db.rocksdb_close
    db.baseDir.flushDbDir(db.subDir)

proc testDbs(
    workDir: string;
    subDir: string;
    instances: int;
    persistent: bool;
      ): TestDbs =
  if workDir == "" or not persistent:
    result.persistent = false
    result.dbDir = "*notused*"
  else:
    result.persistent = true
    result.baseDir = workDir
    result.subDir = subDir
    if subDir != "":
      result.dbDir = workDir / "tmp" / subDir
    else:
      result.dbDir = workDir / "tmp"
  if result.persistent:
    workDir.flushDbDir(subDir)
    for n in 0 ..< min(result.cdb.len, instances):
      result.cdb[n] = newCoreDbRef(LegacyDbPersistent, result.dbDir / $n)

proc snapDbRef(cdb: CoreDbRef; pers: bool): SnapDbRef =
  if pers: SnapDbRef.init(cdb)
  else: SnapDbRef.init(newCoreDbRef LegacyDbMemory)

proc snapDbAccountsRef(
    cdb: CoreDbRef;
    root: Hash256;
    pers: bool;
      ): SnapDbAccountsRef =
  SnapDbAccountsRef.init(cdb.snapDbRef(pers), root, Peer())

# ------------------------------------------------------------------------------
# Test Runners: accounts and accounts storages
# ------------------------------------------------------------------------------

proc miscRunner(noisy =true) =
  suite &"Aristo: Miscellaneous tests":
    test &"VertexID recyling lists":
      noisy.test_transcodeVidRecycleLists()


proc transcodeRunner(noisy =true; sample=accSample; stopAfter=high(int)) =
  let
    accLst = sample.to(seq[UndumpAccounts])
    root = accLst[0].root
    tmpDir = getTmpDir()
    db = tmpDir.testDbs(sample.name&"-transcode", instances=2, persistent=true)
    info = if db.persistent: &"persistent db on \"{db.baseDir}\""
           else: "in-memory db"
    fileInfo = sample.file.splitPath.tail.replace(".txt.gz","")

  defer:
    db.flushDbs

  suite &"Aristo: transcoding {fileInfo} accounts for {info}":

    # New common descriptor for this sub-group of tests
    let
      desc = db.cdb[0].snapDbAccountsRef(root, db.persistent)
      hexaDb = desc.hexaDb
      getFn = desc.getAccountFn
      dbg = if noisy: hexaDb else: nil

    # Borrowed from `test_sync_snap/test_accounts.nim`
    test &"Importing {accLst.len} list items to persistent database":
      if db.persistent:
        accLst.test_accountsImport(desc, true)
      else:
        skip()

    test "Trancoding database records: RLP, NodeRef, Blob, VertexRef":
      noisy.showElapsed("test_transcoder()"):
        noisy.test_transcodeAccounts(
          db.cdb[0].toLegacyBackend.rocksStoreRef, stopAfter)


proc accountsRunner(
    noisy = true;
    sample = accSample;
    resetDb = false;
    cmpBackends = true;
      ) =
  let
    accLst = sample.to(seq[UndumpAccounts]).to(seq[ProofTrieData])
    fileInfo = sample.file.splitPath.tail.replace(".txt.gz","")
    listMode = if resetDb: "" else: ", merged data lists"
    baseDir = getTmpDir() / sample.name & "-accounts"
    dbDir = baseDir / "tmp"

  defer:
    try: baseDir.removeDir except CatchableError: discard

  suite &"Aristo: accounts data dump from {fileInfo}{listMode}":

    test &"Merge {accLst.len} proof & account lists to database":
      check noisy.testTxMergeProofAndKvpList(accLst, dbDir, resetDb)

    test &"Compare {accLst.len} account lists on database backends":
      if cmpBackends:
        check noisy.testBackendConsistency(accLst, dbDir, resetDb)
      else:
        skip()

    test &"Delete accounts database, successively {accLst.len} entries":
      check noisy.testTxMergeAndDelete(accLst, dbDir)

    test &"Distributed backend access {accLst.len} entries":
      check noisy.testDistributedAccess(accLst, dbDir)


proc storagesRunner(
    noisy = true;
    sample = storSample;
    resetDb = false;
    oops: KnownHasherFailure = @[];
    cmpBackends = true;
      ) =
  let
    stoLst = sample.to(seq[UndumpStorages]).to(seq[ProofTrieData])
    fileInfo = sample.file.splitPath.tail.replace(".txt.gz","")
    listMode = if resetDb: "" else: ", merged data lists"
    baseDir = getTmpDir() / sample.name & "-storage"
    dbDir = baseDir / "tmp"

  defer:
    try: baseDir.removeDir except CatchableError: discard

  suite &"Aristo: storages data dump from {fileInfo}{listMode}":

    test &"Merge {stoLst.len} proof & slots lists to database":
      check noisy.testTxMergeProofAndKvpList(
        stoLst, dbDir, resetDb, fileInfo, oops)

    test &"Compare {stoLst.len} slot lists on database backends":
      if cmpBackends:
        check noisy.testBackendConsistency(stoLst, dbDir, resetDb)
      else:
        skip()

    test &"Delete storage database, successively {stoLst.len} entries":
      check noisy.testTxMergeAndDelete(stoLst, dbDir)

    test &"Distributed backend access {stoLst.len} entries":
      check noisy.testDistributedAccess(stoLst, dbDir)

# ------------------------------------------------------------------------------
# Main function(s)
# ------------------------------------------------------------------------------

proc aristoMain*(noisy = defined(debug)) =
  noisy.miscRunner()
  noisy.transcodeRunner()
  noisy.accountsRunner()
  noisy.storagesRunner()

when isMainModule:
  const
    noisy = defined(debug) or true

  setErrorLevel()

  when true: # and false:
    noisy.miscRunner()

  # Borrowed from `test_sync_snap.nim`
  when true: # and false:
    for n,sam in snapTestList:
      noisy.transcodeRunner(sam)
    for n,sam in snapTestStorageList:
      noisy.transcodeRunner(sam)

  # This one uses dumps from the external `nimbus-eth1-blob` repo
  when true and false:
    import ./test_sync_snap/snap_other_xx
    noisy.showElapsed("@snap_other_xx"):
      for n,sam in snapOtherList:
        noisy.accountsRunner(sam, resetDb=true)

  # This one usues dumps from the external `nimbus-eth1-blob` repo
  when true and false:
    import ./test_sync_snap/snap_storage_xx
    let knownFailures: KnownHasherFailure = @[
      ("storages3__18__25_dump#12.27367",(3,HashifyExistingHashMismatch)),
      ("storages4__26__33_dump#12.23924",(6,HashifyExistingHashMismatch)),
      ("storages5__34__41_dump#10.20512",(1,HashifyRootHashMismatch)),
      ("storagesB__84__92_dump#7.9709",  (7,HashifyExistingHashMismatch)),
      ("storagesD_102_109_dump#18.28287",(9,HashifyExistingHashMismatch)),
    ]
    noisy.showElapsed("@snap_storage_xx"):
      for n,sam in snapStorageList:
        noisy.accountsRunner(sam, resetDb=true)
        noisy.storagesRunner(sam, resetDb=true, oops=knownFailures)

  when true: # and false:
    for n,sam in snapTestList:
      noisy.accountsRunner(sam)
    for n,sam in snapTestStorageList:
      noisy.accountsRunner(sam)
      noisy.storagesRunner(sam)

# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------