nimbus-eth1/tests/test_aristo/test_backend.nim

# Nimbus
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
#    http://www.apache.org/licenses/LICENSE-2.0)
#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
#    http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or
# distributed except according to those terms.

## Aristo (aka Patricia) DB records merge test

import
  std/[algorithm, hashes, sequtils, sets, strutils, tables],
  eth/common,
  results,
  unittest2,
  stew/endians2,
  ../../nimbus/sync/protocol,
  ../../nimbus/db/aristo,
  ../../nimbus/db/aristo/[
    aristo_debug,
    aristo_desc,
    aristo_desc/desc_backend,
    aristo_hashify,
    aristo_init/memory_db,
    aristo_init/rocks_db,
    aristo_persistent,
    aristo_blobify,
    aristo_vid],
  ../replay/xcheck,
  ./test_helpers

const
  BlindHash = EmptyBlob.hash

# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------

func hash(filter: FilterRef): Hash =
  ## Unique hash/filter -- cannot use de/blobify as the expressions
  ## `filter.blobify` and `filter.blobify.value.deblobify.value.blobify` are
  ## not necessarily the same binaries due to unsorted tables.
  ##
  var h = BlindHash
  if not filter.isNil:
    h = h !& filter.src.hash
    h = h !& filter.trg.hash

    for w in filter.vGen.vidReorg:
      h = h !& w.uint64.hash

    for w in filter.sTab.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID):
      let data = filter.sTab.getOrVoid(w).blobify.get(otherwise = EmptyBlob)
      h = h !& (w.uint64.toBytesBE.toSeq & data).hash

    for w in filter.kMap.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID):
      let data = @(filter.kMap.getOrVoid(w))
      h = h !& (w.uint64.toBytesBE.toSeq & data).hash

  !$h

# ------------------------------------------------------------------------------
# Private functions
# ------------------------------------------------------------------------------

proc mergeData(
    db: AristoDbRef;
    rootKey: Hash256;
    rootVid: VertexID;
    proof: openArray[SnapProof];
    leafs: openArray[LeafTiePayload];
    noisy: bool;
      ): bool =
  ## Simplified loop body of `test_mergeProofAndKvpList()`
  if 0 < proof.len:
    let rc = db.merge(rootKey, rootVid)
    xCheckRc rc.error == 0

    let proved = db.merge(proof, rc.value)
    xCheck proved.error in {AristoError(0),MergeHashKeyCachedAlready}

  let merged = db.merge leafs
  xCheck merged.error in {AristoError(0), MergeLeafPathCachedAlready}

  block:
    let rc = db.hashify # (noisy, true)
    xCheckRc rc.error == (0,0):
      noisy.say "***", "dataMerge(9)",
        " nLeafs=", leafs.len,
        "\n    cache dump\n    ", db.pp,
        "\n    backend dump\n    ", db.backend.pp(db)

  true

proc verify(
    ly: LayerRef;                            # Database layer
    be: MemBackendRef|RdbBackendRef;         # Backend
    noisy: bool;
      ): bool =
  ## ..

  let
    beSTab = be.walkVtx.toSeq.mapIt((it[1],it[2])).toTable
    beKMap = be.walkKey.toSeq.mapIt((it[1],it[2])).toTable

  for vid in beSTab.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID):
    let
      nVtx = ly.delta.sTab.getOrVoid vid
      mVtx = beSTab.getOrVoid vid

    xCheck (nVtx != VertexRef(nil))
    xCheck (mVtx != VertexRef(nil))
    xCheck nVtx == mVtx:
      noisy.say "***", "verify",
        " beType=", be.typeof,
        " vid=", vid.pp,
        " nVtx=", nVtx.pp,
        " mVtx=", mVtx.pp

    xCheck beSTab.len == ly.delta.sTab.len
    xCheck beKMap.len == ly.delta.kMap.len

  true

# -----------

proc collectFilter(
    db: AristoDbRef;
    filter: FilterRef;
    tab: var Table[QueueID,Hash];
    noisy: bool;
      ): bool =
  ## Store filter on permanent BE and register digest
  if not filter.isNil:
    let
      fid = QueueID(7 * (tab.len + 1)) # just some number
      be = db.backend
      tx = be.putBegFn()

    be.putFilFn(tx, @[(fid,filter)])
    let rc = be.putEndFn tx
    xCheckRc rc.error == 0

    tab[fid] = filter.hash

  true

proc verifyFiltersImpl[T](
    be: T;
    tab: Table[QueueID,Hash];
    noisy: bool;
      ): bool =
  ## Compare stored filters against registered ones
  var n = 0
  for (_,fid,filter) in be.walkFilBe:
    let
      filterHash = filter.hash
      registered = tab.getOrDefault(fid, BlindHash)

    xCheck (registered != BlindHash)
    xCheck registered == filterHash:
      noisy.say "***", "verifyFiltersImpl",
        " n=", n+1,
        " fid=", fid.pp,
        " filterHash=", filterHash.int.toHex,
        " registered=", registered.int.toHex

    n.inc

  xCheck n == tab.len
  true

proc verifyFilters(
    db: AristoDbRef;
    tab: Table[QueueID,Hash];
    noisy: bool;
      ): bool =
  ## Wrapper
  let be = db.backend
  case be.kind:
  of BackendMemory:
    return be.MemBackendRef.verifyFiltersImpl(tab, noisy)
  of BackendRocksDB:
    return be.RdbBackendRef.verifyFiltersImpl(tab, noisy)
  else:
    discard
  check db.backend.kind == BackendMemory or db.backend.kind == BackendRocksDB

# ------------------------------------------------------------------------------
# Public test function
# ------------------------------------------------------------------------------

proc testBackendConsistency*(
    noisy: bool;
    list: openArray[ProofTrieData];          # Test data
    rdbPath: string;                         # Rocks DB storage directory
    resetDb = false;
    doRdbOk = true;
      ): bool =
  ## Import accounts
  var
    filTab: Table[QueueID,Hash]              # Filter register
    ndb = AristoDbRef()                      # Reference cache
    mdb = AristoDbRef()                      # Memory backend database
    rdb = AristoDbRef()                      # Rocks DB backend database
    rootKey = Hash256()                      # Root key
    count = 0

  defer:
    rdb.finish(flush=true)

  xCheck rdbPath != ""

  for n,w in list:
    if w.root != rootKey or resetDB:
      rootKey = w.root
      count = 0
      ndb = AristoDbRef.init()
      mdb = AristoDbRef.init MemBackendRef

      if doRdbOk:
        if not rdb.backend.isNil: # ignore bootstrap
          let verifyFiltersOk = rdb.verifyFilters(filTab, noisy)
          xCheck verifyFiltersOk
          filTab.clear
        rdb.finish(flush=true)
        let rc = AristoDbRef.init(RdbBackendRef, rdbPath)
        xCheckRc rc.error == 0
        rdb = rc.value

        # Disable automated filter management, still allow filter table access
        # for low level read/write testing.
        rdb.backend.filters = QidSchedRef(nil)
    count.inc

    xCheck ndb.backend.isNil
    xCheck not mdb.backend.isNil
    xCheck doRdbOk or not rdb.backend.isNil

    when true and false:
      noisy.say "***", "beCon(1) <", n, "/", list.len-1, ">", " groups=", count

    block:
      let
        rootVid = VertexID(1)
        leafs = w.kvpLst.mapRootVid VertexID(1) # for merging it into main trie

      block:
        let ndbOk = ndb.mergeData(
          rootKey, rootVid, w.proof, leafs, noisy=false)
        xCheck ndbOk
      block:
        let mdbOk = mdb.mergeData(
          rootKey, rootVid, w.proof, leafs, noisy=false)
        xCheck mdbOk
      if doRdbOk: # optional
        let rdbOk = rdb.mergeData(
          rootKey, rootVid, w.proof, leafs, noisy=false)
        xCheck rdbOk

      when true and false:
        noisy.say "***", "beCon(2) <", n, "/", list.len-1, ">",
          " groups=", count,
          "\n    cache dump\n    ", ndb.pp,
          "\n    backend dump\n    ", ndb.backend.pp(ndb),
          "\n    -------------",
          "\n    mdb cache\n    ", mdb.pp,
          "\n    mdb backend\n    ", mdb.backend.pp(ndb),
          "\n    -------------",
          "\n    rdb cache\n    ", rdb.pp,
          "\n    rdb backend\n    ", rdb.backend.pp(ndb),
          "\n    -------------"

    when true and false:
      noisy.say "***", "beCon(4) <", n, "/", list.len-1, ">", " groups=", count

    var
      mdbPreSaveCache, mdbPreSaveBackend: string
      rdbPreSaveCache, rdbPreSaveBackend: string
    when true: # and false:
      #mdbPreSaveCache = mdb.pp
      #mdbPreSaveBackend = mdb.backend.pp(mdb)
      rdbPreSaveCache = rdb.pp
      rdbPreSaveBackend = rdb.backend.pp(rdb)


    # Provide filter, store filter on permanent BE, and register filter digest
    block:
      let rc = mdb.stow(persistent=false, chunkedMpt=true)
      xCheckRc rc.error == 0
      let collectFilterOk = rdb.collectFilter(mdb.roFilter, filTab, noisy)
      xCheck collectFilterOk

    # Store onto backend database
    block:
      #noisy.say "***", "db-dump\n    ", mdb.pp
      let rc = mdb.stow(persistent=true, chunkedMpt=true)
      xCheckRc rc.error == 0

    if doRdbOk:
      let rc = rdb.stow(persistent=true, chunkedMpt=true)
      xCheckRc rc.error == 0

    block:
      let mdbVerifyOk = ndb.top.verify(mdb.backend.MemBackendRef, noisy)
      xCheck mdbVerifyOk:
        when true and false:
          noisy.say "***", "beCon(4) <", n, "/", list.len-1, ">",
            " groups=", count,
            "\n    ndb cache\n    ", ndb.pp,
            "\n    ndb backend=", ndb.backend.isNil.not,
            #"\n    -------------",
            #"\n    mdb pre-save cache\n    ", mdbPreSaveCache,
            #"\n    mdb pre-save backend\n    ", mdbPreSaveBackend,
            "\n    -------------",
            "\n    mdb cache\n    ", mdb.pp,
            "\n    mdb backend\n    ", mdb.backend.pp(ndb),
            "\n    -------------"

    if doRdbOk:
      let rdbVerifyOk = ndb.top.verify(rdb.backend.RdbBackendRef, noisy)
      xCheck rdbVerifyOk:
        when true and false:
          noisy.say "***", "beCon(4) <", n, "/", list.len-1, ">",
            " groups=", count,
            "\n    ndb cache\n    ", ndb.pp,
            "\n    ndb backend=", ndb.backend.isNil.not,
            "\n    -------------",
            "\n    rdb pre-save cache\n    ", rdbPreSaveCache,
            "\n    rdb pre-save backend\n    ", rdbPreSaveBackend,
            "\n    -------------",
            "\n    rdb cache\n    ", rdb.pp,
            "\n    rdb backend\n    ", rdb.backend.pp(ndb),
            #"\n    -------------",
            #"\n    mdb cache\n    ", mdb.pp,
            #"\n    mdb backend\n    ", mdb.backend.pp(ndb),
            "\n    -------------"

    when true and false:
      noisy.say "***", "beCon(9) <", n, "/", list.len-1, ">", " groups=", count

  # Finally ...
  block:
    let verifyFiltersOk = rdb.verifyFilters(filTab, noisy)
    xCheck verifyFiltersOk

  true

# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------