Aristo lazily delete larger subtrees (#2560)

* Extract sub-tree deletion functions into separate sub-modules

* Move/rename `aristo_desc.accLruSize` => `aristo_constants.ACC_LRU_SIZE`

* Lazily delete sub-trees

why:
  This gives some control of the memory used to keep the deleted vertices
  in the cached layers. For larger sub-trees, keys and vertices might be
  on the persistent backend to a large extend. This would pull an amount
  of extra information from the backend into the cached layer.

  For lazy deleting it is enough to remember sub-trees by a small set of
  (at most 16) sub-roots to be processed when storing persistent data.
  Marking the tree root deleted immediately allows to let most of the code
  base work as before.

* Comments and cosmetics

* No need to import all for `Aristo` here

* Kludge to make `chronicle` usage in sub-modules work with `fluffy`

why:
  That `fluffy` would not run with any logging in `core_deb` is a problem
  I have known for a while. Up to now, logging was only used for debugging.

  With the current `Aristo` PR, there are cases where logging might be
  wanted but this works only if `chronicles` runs without the
  `json[dynamic]` sinks.

  So this should be re-visited.

* More of a kludge

fluffy/fluffy.nim.cfg

@@ -1,4 +1,11 @@
--d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
+#
+# For some reason `json[dynamic]` causes problems with subsequent modules from
+# `Aristo` when compiling `fluffy`. There might be a `chronicles` inport missing
+# but it is not obvious where. -- jordan
+#
+#-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
+
+-d:"chronicles_sinks=textlines[dynamic]"
 -d:"chronicles_runtime_filtering=on"
 -d:"chronicles_disable_thread_id"
 

fluffy/tools/beacon_lc_bridge/nim.cfg

@@ -1,4 +1,6 @@
 # Use only `secp256k1` public key cryptography as an identity in LibP2P.
 -d:"libp2p_pki_schemes=secp256k1"
 
--d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
+# See `fluffy.nim.cfg`
+#-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
+-d:"chronicles_sinks=textlines[dynamic]"

fluffy/tools/portal_bridge/nim.cfg

@@ -1 +1,3 @@
--d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
+# See `fluffy.nim.cfg`
+#-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
+-d:"chronicles_sinks=textlines[dynamic]"

nimbus/db/aristo/TODO.md

@@ -1,9 +1,4 @@
-* Re-visit `delTree()`. Suggestion is deleting small trees on the memory later,
+* Some comletions might be needed for the `aristo_part` module which is a
-  otherwise only deleting the root vertex (so it becomes inaccessible) and
-  remember the follow up vertices which can travel through the tx-layers
-  to be picked up by the backend store.
-
-* Some comletions migh be needed for the `aristo_part` module which is a
   re-implementation of the module supporting *proof-mode*/partial trees.
   + Complete `partMergeStorageData()`. This function might not be needed at
     all unless *snap-sync* is really revived.

nimbus/db/aristo/aristo_constants.nim

@@ -38,6 +38,19 @@ const
     ## functions with fixed assignments of the type of a state root (e.g. for
     ## a receipt or a transaction root.)
 
+  ACC_LRU_SIZE* = 1024 * 1024
+    ## LRU cache size for accounts that have storage, see `.accLeaves` and
+    ## `.stoLeaves` fields of the main descriptor.
+
+  DELETE_SUBTREE_VERTICES_MAX* = 25
+    ## Maximum number of vertices for a tree to be deleted instantly. If the
+    ## tree is larger, only the sub-tree root will be deleted immediately and
+    ## subsequent entries will be deleted not until the cache layers are saved
+    ## to the backend.
+    ##
+    ## Set to zero to disable in which case all sub-trees are deleted
+    ## immediately.
+
 static:
   # must stay away from `VertexID(1)` and `VertexID(2)`
   doAssert 2 < LEAST_FREE_VID

nimbus/db/aristo/aristo_debug.nim

@@ -195,7 +195,7 @@ proc ppPayload(p: LeafPayload, db: AristoDbRef): string =
   of AccountData:
     result = "(" & p.account.ppAriAccount() & "," & p.stoID.ppVid & ")"
   of StoData:
-    result = $p.stoData
+    result = ($p.stoData).squeeze
 
 proc ppVtx(nd: VertexRef, db: AristoDbRef, rvid: RootedVertexID): string =
   if not nd.isValid:

nimbus/db/aristo/aristo_delete.nim

@@ -11,7 +11,6 @@
 ## Aristo DB -- Patricia Trie delete funcionality
 ## ==============================================
 ##
-## Delete by `Hike` type chain of vertices.
 
 {.push raises: [].}
 
@@ -19,6 +18,7 @@ import
   std/typetraits,
   eth/common,
   results,
+  ./aristo_delete/[delete_helpers, delete_subtree],
   "."/[aristo_desc, aristo_fetch, aristo_get, aristo_hike, aristo_layers,
        aristo_utils]
 
@@ -39,79 +39,10 @@ proc branchStillNeeded(vtx: VertexRef): Result[int,void] =
   # Oops, degenerated branch node
   err()
 
-# -----------
-
-proc disposeOfVtx(
-    db: AristoDbRef;                   # Database, top layer
-    rvid: RootedVertexID;              # Vertex ID to clear
-      ) =
-  # Remove entry
-  db.layersResVtx(rvid)
-  db.layersResKey(rvid)
-
 # ------------------------------------------------------------------------------
 # Private functions
 # ------------------------------------------------------------------------------
 
-proc delSubTreeImpl(
-    db: AristoDbRef;                   # Database, top layer
-    root: VertexID;                    # Root vertex
-      ): Result[void,AristoError] =
-  ## Implementation of *delete* sub-trie.
-  var
-    dispose = @[root]
-    (rootVtx, _) = db.getVtxRc((root, root)).valueOr:
-      if error == GetVtxNotFound:
-        return ok()
-      return err(error)
-    follow = @[rootVtx]
-
-  # Collect list of nodes to delete
-  while 0 < follow.len:
-    var redo: seq[VertexRef]
-    for vtx in follow:
-      for vid in vtx.subVids:
-        # Exiting here leaves the tree as-is
-        let vtx = (? db.getVtxRc((root, vid)))[0]
-        redo.add vtx
-        dispose.add vid
-    redo.swap follow
-
-  # Mark collected vertices to be deleted
-  for vid in dispose:
-    db.disposeOfVtx((root, vid))
-
-  ok()
-
-proc delStoTreeImpl(
-    db: AristoDbRef;                   # Database, top layer
-    rvid: RootedVertexID;                    # Root vertex
-    accPath: Hash256;
-    stoPath: NibblesBuf;
-      ): Result[void,AristoError] =
-  ## Implementation of *delete* sub-trie.
-
-  let (vtx, _) = db.getVtxRc(rvid).valueOr:
-    if error == GetVtxNotFound:
-      return ok()
-    return err(error)
-
-  case vtx.vType
-  of Branch:
-    for i in 0..15:
-      if vtx.bVid[i].isValid:
-        ? db.delStoTreeImpl(
-          (rvid.root, vtx.bVid[i]), accPath,
-          stoPath & vtx.ePfx & NibblesBuf.nibble(byte i))
-
-  of Leaf:
-    let stoPath = Hash256(data: (stoPath & vtx.lPfx).getBytes())
-    db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), nil)
-
-  db.disposeOfVtx(rvid)
-
-  ok()
-
 proc deleteImpl(
     db: AristoDbRef;                   # Database, top layer
     hike: Hike;                        # Fully expanded path
@@ -199,7 +130,7 @@ proc deleteAccountRecord*(
 
   # Delete storage tree if present
   if stoID.isValid:
-    ? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath, NibblesBuf())
+    ? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath)
 
   ?db.deleteImpl(hike)
 
@@ -322,7 +253,7 @@ proc deleteStorageTree*(
   # Mark account path Merkle keys for update
   db.updateAccountForHasher accHike
 
-  ? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath, NibblesBuf())
+  ? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath)
 
   # De-register the deleted storage tree from the accounts record
   let leaf = wpAcc.vtx.dup             # Dup on modify

nimbus/db/aristo/aristo_delete/delete_debug.nim

@@ -0,0 +1,127 @@
+# nimbus-eth1
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+{.push raises: [].}
+
+import
+  std/[math, strformat, times],
+  chronicles,
+  ".."/[aristo_desc, aristo_get, aristo_profile]
+
+export
+  aristo_profile.toStr
+
+type
+  SubTreeStats* = tuple
+    nVtxs: int                         ## Number of vertices in sub-tree
+    nLeafs: int                        ## Number of leafs in sub-tree
+    depthMax: int                      ## Maximal vertex path length
+    nStoCache: int                     ## Size of storage leafs cache
+    elapsed: Duration                  ## Time spent analysing
+
+  SubTreeStatsAccu* = tuple
+    count: int                         ## Number of entries
+    sVtxs, qVtxs: float                ## Sum and square sum of `.nVtxs`
+    sLeafs, qLeafs: float              ## Sum and square sum of `.nLeafs`
+    sDepth, qDepth: float              ## Sum and square sum of `.depthMax`
+    sElapsed: Duration                 ## Sum of `.elapsed`
+
+  SubTreeDist* = tuple
+    count: int                         ## Number of entries
+    mVtxs, dVtxs: float                ## Mean and std deviation of `.nVtxs`
+    mLeafs, dLeafs: float              ## Mean and std deviation of `.nLeafs`
+    mDepth, dDepth: float              ## Mean and std deviation of `.depthMax`
+
+# ------------------------------------------------------------------------------
+# Prival helper
+# ------------------------------------------------------------------------------
+
+proc analyseSubTreeImpl(
+    db: AristoDbRef;                   # Database, top layer
+    rvid: RootedVertexID;              # Root vertex
+    depth: int;                        # Recursion depth
+    stats: var SubTreeStats;           # Statistics
+      ) =
+  let (vtx, _) = db.getVtxRc(rvid).valueOr:
+    return
+
+  stats.nVtxs.inc
+
+  if stats.depthMax < depth:
+    stats.depthMax = depth
+
+  case vtx.vType:
+  of Branch:
+    for n in 0..15:
+      if vtx.bVid[n].isValid:
+        db.analyseSubTreeImpl((rvid.root,vtx.bVid[n]), depth+1, stats)
+  of Leaf:
+    stats.nLeafs.inc
+
+
+func evalDist(count: int; sum, sqSum: float): tuple[mean, stdDev: float] =
+  result.mean = sum / count.float
+
+  let
+    sqMean = sqSum / count.float
+    meanSq = result.mean * result.mean
+
+    # Mathematically, `meanSq <= sqMean` but there might be rounding errors
+    # if `meanSq` and `sqMean` are approximately the same.
+    sigma = sqMean - min(meanSq,sqMean)
+    
+  result.stdDev = sigma.sqrt
+
+# ------------------------------------------------------------------------------
+# Public analysis tools
+# ------------------------------------------------------------------------------
+
+proc analyseSubTree*(
+    db: AristoDbRef;                   # Database, top layer
+    rvid: RootedVertexID;              # Root vertex
+    minVtxs: int;                      # Accumulate if `minVtxs` <= `.nVtxs`
+    accu: var SubTreeStatsAccu;        # For accumulated statistics
+      ): SubTreeStats =
+  let start = getTime()
+  db.analyseSubTreeImpl(rvid, 1, result)
+  result.nStoCache = db.stoLeaves.len
+
+  if minVtxs <= result.nVtxs:
+    accu.count.inc
+    accu.sVtxs += result.nVtxs.float
+    accu.qVtxs += (result.nVtxs * result.nVtxs).float
+    accu.sLeafs += result.nLeafs.float
+    accu.qLeafs += (result.nLeafs * result.nLeafs).float
+    accu.sDepth += result.depthMax.float
+    accu.qDepth += (result.depthMax * result.depthMax).float
+
+  result.elapsed = getTime() - start
+  accu.sElapsed += result.elapsed      # Unconditionally collecrd
+
+
+func stats*(a: SubTreeStatsAccu): SubTreeDist =
+  result.count = a.count
+  (result.mVtxs, result.dVtxs) = evalDist(a.count, a.sVtxs, a.qVtxs)
+  (result.mLeafs, result.dLeafs) = evalDist(a.count, a.sLeafs, a.qLeafs)
+  (result.mDepth, result.dDepth) = evalDist(a.count, a.sDepth, a.qDepth)
+
+func strStats*(
+    a: SubTreeStatsAccu;
+      ): tuple[count, vtxs, leafs, depth, elapsed: string] =
+  let w = a.stats()
+  result.count = $w.count
+  result.elapsed = a.sElapsed.toStr
+  result.vtxs = &"{w.mVtxs:.1f}[{w.dVtxs:.1f}]"
+  result.leafs = &"{w.mLeafs:.1f}[{w.dLeafs:.1f}]"
+  result.depth = &"{w.mDepth:.1f}[{w.dDepth:.1f}]"
+
+# ------------------------------------------------------------------------------
+# End
+# ------------------------------------------------------------------------------

nimbus/db/aristo/aristo_delete/delete_helpers.nim

@@ -0,0 +1,25 @@
+# nimbus-eth1
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+{.push raises: [].}
+
+import
+  ".."/[aristo_desc, aristo_layers]
+
+
+proc disposeOfVtx*(
+    db: AristoDbRef;                   # Database, top layer
+    rvid: RootedVertexID;              # Vertex ID to clear
+      ) =
+  # Remove entry
+  db.layersResVtx(rvid)
+  db.layersResKey(rvid)
+
+# End

nimbus/db/aristo/aristo_delete/delete_subtree.nim

@@ -0,0 +1,20 @@
+# nimbus-eth1
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import ../aristo_constants
+
+when DELETE_SUBTREE_VERTICES_MAX == 0:
+  import ./delete_subtree_now as del_sub
+else:
+  import ./delete_subtree_lazy as del_sub
+
+export del_sub
+
+# End

nimbus/db/aristo/aristo_delete/delete_subtree_lazy.nim

@@ -0,0 +1,252 @@
+# nimbus-eth1
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+{.push raises: [].}
+
+import
+  std/sets,
+  eth/common,
+  results,
+  ".."/[aristo_desc, aristo_get, aristo_layers],
+  ./delete_helpers
+
+const
+  extraDebuggingMessages = false # or true
+    ## Enable additional logging noise. Note that this might slow down the
+    ## system performance but will not be too significant. When importing the
+    ## first 5m blocks from `era1` on some Debian system,
+    ## * loading time was ~5h
+    ## * overhead of accumulated analysis times was ~1.2s
+
+type
+  VidCollect = tuple
+    data: array[DELETE_SUBTREE_VERTICES_MAX,VertexID]
+    top: int                         # Next free slot if smaller `.data.len`
+
+
+when extraDebuggingMessages:
+  import
+    std/times,
+    chronicles,
+    ./delete_debug
+
+  const
+    allStatsFrequency = 20
+      ## Print accumutated statistics every `allStatsFrequency` visits of
+      ## the analysis tool.
+
+    minVtxsForLogging = 1000
+      ## Suppress detailed logging for smaller sub-trees
+
+  var stats: SubTreeStatsAccu
+    ## Accumulated statistics
+
+  func `$`(ela: Duration): string =
+    ela.toStr
+
+  template debugLog(info: static[string]; args: varargs[untyped]) =
+    ## Statistics message via `chronicles` logger, makes it easy to
+    ## change priority and format.
+    notice info, args
+
+# ------------------------------------------------------------------------------
+# Private heplers
+# ------------------------------------------------------------------------------
+
+func capa(T: type VidCollect): int =
+  ## Syntactic sugar
+  T.default.data.len
+
+
+proc collectSubTreeLazily(
+  db: AristoDbRef;                     # Database, top layer
+  rvid: RootedVertexID;                # Root vertex
+  vids: var VidCollect;                # Accumulating vertex IDs for deletion
+    ): Result[void,AristoError]  =
+  ## Collect vids for a small sub-tree
+  let (vtx, _) = db.getVtxRc(rvid).valueOr:
+    if error == GetVtxNotFound:
+      return ok()
+    return err(error)
+
+  if vids.top < vids.data.len:
+    vids.data[vids.top] = rvid.vid
+    vids.top.inc                       # Max value of `.top`: `vid.data.len + 1`
+
+    if vtx.vType == Branch:
+      for n in 0..15:
+        if vtx.bVid[n].isValid:
+          ? db.collectSubTreeLazily((rvid.root,vtx.bVid[n]), vids)
+
+  elif vids.top <= vids.data.len:
+    vids.top.inc                       # Terminates here
+
+  ok()
+
+
+proc collectStoTreeLazily(
+  db: AristoDbRef;                     # Database, top layer
+  rvid: RootedVertexID;                # Root vertex
+  accPath: Hash256;                    # Accounts cache designator
+  stoPath: NibblesBuf;                 # Current storage path
+  vids: var VidCollect;                # Accumulating vertex IDs for deletion
+    ): Result[void,AristoError] =
+  ## Collect vertex/vid and delete cache entries.
+  let (vtx, _) = db.getVtxRc(rvid).valueOr:
+    if error == GetVtxNotFound:
+      return ok()
+    return err(error)
+
+  case vtx.vType
+  of Branch:
+    for i in 0..15:
+      if vtx.bVid[i].isValid:
+        ? db.collectStoTreeLazily(
+          (rvid.root, vtx.bVid[i]), accPath,
+          stoPath & vtx.ePfx & NibblesBuf.nibble(byte i),
+          vids)
+
+  of Leaf:
+    let stoPath = Hash256(data: (stoPath & vtx.lPfx).getBytes())
+    db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), nil)
+
+  # There is no useful approach avoiding to walk the whole tree for updating
+  # the storage data access cache.
+  #
+  # The alternative of stopping here and clearing the whole cache did degrade
+  # performance significantly in some tests on mainnet when importing `era1`.
+  #
+  # When not clearing the cache it was seen
+  # * filled up to maximum size most of the time
+  # * at the same time having no `stoPath` hit at all (so there was nothing
+  #   to be cleared.)
+  #
+  if vids.top <= vids.data.len:
+    if vids.top < vids.data.len:
+      vids.data[vids.top] = rvid.vid
+    vids.top.inc                       # Max value of `.top`: `vid.data.len + 1`
+
+  ok()
+
+
+proc disposeOfSubTree(
+    db: AristoDbRef;                   # Database, top layer
+    rvid: RootedVertexID;              # Root vertex
+    vids: var VidCollect;              # Accumulated vertex IDs for disposal
+      ) =
+  ## Evaluate results from `collectSubTreeLazyImpl()` or ftom
+  ## `collectStoTreeLazyImpl)`.
+  ##
+  if vids.top <= typeof(vids).capa:
+    # Delete small tree
+    for n in 0 ..< vids.top:
+      db.disposeOfVtx((rvid.root, vids.data[n]))
+
+  else:
+    # Mark the sub-trees disabled to be deleted not until the layer is
+    # finally stored onto the backend.
+    let vtx = db.getVtxRc(rvid).value[0]
+    for n in 0..15:
+      if vtx.bVid[n].isValid:
+        db.top.delTree.add (rvid.root,vtx.bVid[n])
+
+    # Delete top of tree now.
+    db.disposeOfVtx(rvid)
+
+# ------------------------------------------------------------------------------
+# Public functions
+# ------------------------------------------------------------------------------
+
+proc delSubTreeImpl*(
+    db: AristoDbRef;                   # Database, top layer
+    root: VertexID;                    # Root vertex
+      ): Result[void,AristoError] =
+  ## Delete all the `subRoots`if there are a few, only. Otherwise
+  ## mark it for deleting later.
+  discard db.getVtxRc((root,root)).valueOr:
+    if error == GetVtxNotFound:
+      return ok()
+    return err(error)
+
+  when extraDebuggingMessages:
+    let
+      ana = db.analyseSubTree((root,root), VidCollect.capa+1, stats)
+      start = getTime()
+
+  var dispose: VidCollect
+  ? db.collectSubTreeLazily((root,root), dispose)
+
+  db.disposeOfSubTree((root,root), dispose)
+
+  when extraDebuggingMessages:
+    if typeof(dispose).capa < dispose.top:
+
+      if minVtxsForLogging < ana.nVtxs:
+        debugLog("Generic sub-tree analysis",
+          nVtxs      = ana.nVtxs,
+          nLeafs     = ana.nLeafs,
+          depthMax   = ana.depthMax,
+          nDelTree   = db.top.delTree.len,
+          elaCollect = getTime() - start)
+
+      if (stats.count mod allStatsFrequency) == 0:
+        let
+          start = getTime()
+          (count, vtxs, leafs, depth, elapsed) = stats.strStats
+        debugLog("Sub-tree analysis stats", count, vtxs, leafs, depth, elapsed)
+        stats.sElapsed += getTime() - start
+  ok()
+
+
+proc delStoTreeImpl*(
+    db: AristoDbRef;                   # Database, top layer
+    rvid: RootedVertexID;              # Root vertex
+    accPath: Hash256;
+      ): Result[void,AristoError] =
+  ## Collect vertex/vid and cache entry.
+  discard db.getVtxRc(rvid).valueOr:
+    if error == GetVtxNotFound:
+      return ok()
+    return err(error)
+
+  when extraDebuggingMessages:
+    let
+      ana = db.analyseSubTree(rvid, VidCollect.capa+1, stats)
+      start = getTime()
+
+  var dispose: VidCollect              # Accumulating vertices for deletion
+  ? db.collectStoTreeLazily(rvid, accPath, NibblesBuf(), dispose)
+
+  db.disposeOfSubTree(rvid, dispose)
+
+  when extraDebuggingMessages:
+    if typeof(dispose).capa < dispose.top:
+
+      if minVtxsForLogging < ana.nVtxs or db.stoLeaves.len < ana.nStoCache:
+        debugLog("Storage sub-tree analysis",
+          nVtxs          = ana.nVtxs,
+          nLeafs         = ana.nLeafs,
+          depthMax       = ana.depthMax,
+          nStoCache      = ana.nStoCache,
+          nStoCacheDelta = ana.nStoCache - db.stoLeaves.len,
+          nDelTree       = db.top.delTree.len,
+          elaCollect     = getTime() - start)
+
+      if (stats.count mod allStatsFrequency) == 0:
+        let
+          start = getTime()
+          (count, vtxs, leafs, depth, elapsed) = stats.strStats
+        debugLog("Sub-tree analysis stats", count, vtxs, leafs, depth, elapsed)
+        stats.sElapsed += getTime() - start
+  ok()
+
+# ------------------------------------------------------------------------------
+# End
+# ------------------------------------------------------------------------------

nimbus/db/aristo/aristo_delete/delete_subtree_now.nim

@@ -0,0 +1,101 @@
+# nimbus-eth1
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+{.push raises: [].}
+
+import
+  eth/common,
+  ".."/[aristo_desc, aristo_get, aristo_layers],
+  ./delete_helpers
+
+# ------------------------------------------------------------------------------
+# Private heplers
+# ------------------------------------------------------------------------------
+
+proc delSubTreeNow(
+    db: AristoDbRef;
+    rvid: RootedVertexID;
+      ): Result[void,AristoError] =
+  ## Delete sub-tree now
+  let (vtx, _) = db.getVtxRc(rvid).valueOr:
+    if error == GetVtxNotFound:
+      return ok()
+    return err(error)
+
+  if vtx.vType == Branch:
+    for n in 0..15:
+      if vtx.bVid[n].isValid:
+        ? db.delSubTreeNow((rvid.root,vtx.bVid[n]))
+
+  db.disposeOfVtx(rvid)
+
+  ok()
+
+
+proc delStoTreeNow(
+  db: AristoDbRef;                   # Database, top layer
+  rvid: RootedVertexID;              # Root vertex
+  accPath: Hash256;                  # Accounts cache designator
+  stoPath: NibblesBuf;               # Current storage path
+    ): Result[void,AristoError] =
+  ## Implementation of *delete* sub-trie.
+
+  let (vtx, _) = db.getVtxRc(rvid).valueOr:
+    if error == GetVtxNotFound:
+      return ok()
+    return err(error)
+
+  case vtx.vType
+  of Branch:
+    for i in 0..15:
+      if vtx.bVid[i].isValid:
+        ? db.delStoTreeNow(
+          (rvid.root, vtx.bVid[i]), accPath,
+          stoPath & vtx.ePfx & NibblesBuf.nibble(byte i))
+
+  of Leaf:
+    let stoPath = Hash256(data: (stoPath & vtx.lPfx).getBytes())
+    db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), nil)
+
+  db.disposeOfVtx(rvid)
+
+  ok()
+
+# ------------------------------------------------------------------------------
+# Public functions
+# ------------------------------------------------------------------------------
+
+proc delSubTreeImpl*(
+    db: AristoDbRef;
+    root: VertexID;
+      ): Result[void,AristoError] =
+  discard db.getVtxRc((root, root)).valueOr:
+    if error == GetVtxNotFound:
+      return ok()
+    return err(error)
+  db.delSubTreeNow (root,root)
+
+
+proc delStoTreeImpl*(
+    db: AristoDbRef;                   # Database, top layer
+    rvid: RootedVertexID;              # Root vertex
+    accPath: Hash256;
+      ): Result[void,AristoError] =
+  ## Implementation of *delete* sub-trie.
+  discard db.getVtxRc(rvid).valueOr:
+    if error == GetVtxNotFound:
+      return ok()
+    return err(error)
+  db.delStoTreeNow(rvid, accPath, NibblesBuf())
+
+# ------------------------------------------------------------------------------
+# End
+# ------------------------------------------------------------------------------
+

nimbus/db/aristo/aristo_delta.nim

@@ -14,11 +14,32 @@
 
 import
   std/tables,
+  chronicles,
   eth/common,
   results,
-  "."/[aristo_desc, aristo_layers],
+  ./aristo_delta/[delta_merge, delta_reverse],
   ./aristo_desc/desc_backend,
-  ./aristo_delta/[delta_merge, delta_reverse]
+  "."/[aristo_desc, aristo_get, aristo_layers, aristo_utils]
+
+logScope:
+  topics = "aristo-delta"
+
+# ------------------------------------------------------------------------------
+# Private functions
+# ------------------------------------------------------------------------------
+
+proc delSubTree(db: AristoDbRef; writer: PutHdlRef; rvid: RootedVertexID) =
+  ## Collect subtrees marked for deletion
+  let (vtx,_) = db.getVtxRc(rvid).valueOr:
+    notice "Descending for deletion stopped", rvid, error
+    return
+  for vid in vtx.subVids:
+    db.delSubTree(writer, (rvid.root, vid))
+  db.backend.putVtxFn(writer, rvid, VertexRef(nil))
+  db.backend.putKeyFn(writer, rvid, VOID_HASH_KEY)
+  # Make sure the `rvid` is not mentioned here, anymore for furter update.
+  db.balancer.sTab.del rvid
+  db.balancer.kMap.del rvid
 
 # ------------------------------------------------------------------------------
 # Public functions, save to backend
@@ -85,6 +106,12 @@ proc deltaPersistent*(
 
   # Store structural single trie entries
   let writeBatch = ? be.putBegFn()
+  # This one must come first in order to avoid duplicate `sTree[]` or
+  # `kMap[]` instructions, in the worst case overwiting previously deleted
+  # entries.
+  for rvid in db.balancer.delTree:
+    db.delSubTree(writeBatch, rvid)
+  # Now the standard `sTree[]` and `kMap[]` instructions.
   for rvid, vtx in db.balancer.sTab:
     be.putVtxFn(writeBatch, rvid, vtx)
   for rvid, key in db.balancer.kMap:
@@ -97,12 +124,12 @@ proc deltaPersistent*(
   for accPath, vtx in db.balancer.accLeaves:
     let accKey = accPath.to(AccountKey)
     if not db.accLeaves.lruUpdate(accKey, vtx):
-      discard db.accLeaves.lruAppend(accKey, vtx, accLruSize)
+      discard db.accLeaves.lruAppend(accKey, vtx, ACC_LRU_SIZE)
 
   for mixPath, vtx in db.balancer.stoLeaves:
     let mixKey = mixPath.to(AccountKey)
     if not db.stoLeaves.lruUpdate(mixKey, vtx):
-      discard db.stoLeaves.lruAppend(mixKey, vtx, accLruSize)
+      discard db.stoLeaves.lruAppend(mixKey, vtx, ACC_LRU_SIZE)
 
   # Done with balancer, all saved to backend
   db.balancer = LayerRef(nil)

nimbus/db/aristo/aristo_delta/delta_merge.nim

@@ -48,13 +48,14 @@ proc deltaMerge*(
     result = LayerRef(
       sTab:      lower.sTab, # shallow copy (entries will not be modified)
       kMap:      lower.kMap,
+      delTree:   lower.delTree,
       accLeaves: lower.accLeaves,
       stoLeaves: lower.stoLeaves,
       vTop:      upper.vTop)
     layersMergeOnto(upper, result[])
 
   else:
-    # Otherwise avoid copying some tables by modifyinh `upper`. This is not
+    # Otherwise avoid copying some tables by modifying `upper`. This is not
     # completely free as the merge direction changes to merging the `lower`
     # layer up into the higher prioritised `upper` layer (note that the `lower`
     # argument filter is read-only.) Here again, the `upper` argument must not
@@ -67,6 +68,9 @@ proc deltaMerge*(
       if not upper.kMap.hasKey(rvid):
         upper.kMap[rvid] = key
 
+    for rvid in lower.delTree:
+      upper.delTree.add rvid
+
     for (accPath,leafVtx) in lower.accLeaves.pairs:
       if not upper.accLeaves.hasKey(accPath):
         upper.accLeaves[accPath] = leafVtx

nimbus/db/aristo/aristo_delta/delta_reverse.nim

@@ -12,7 +12,46 @@ import
   std/tables,
   eth/common,
   results,
-  ".."/[aristo_desc, aristo_get]
+  ".."/[aristo_desc, aristo_get, aristo_utils]
+
+# ------------------------------------------------------------------------------
+# Private functions
+# ------------------------------------------------------------------------------
+
+proc revSubTree(
+    db: AristoDbRef;
+    rev: LayerRef;
+    rvid: RootedVertexID;
+      ): Result[void,(VertexID,AristoError)] =
+  ## Collect subtrees marked for deletion
+  let
+    vtx = block:
+      let rc = db.getVtxUbe rvid
+      if rc.isOk:
+        rc.value
+      elif rc.error == GetVtxNotFound:
+        VertexRef(nil)
+      else:
+        return err((rvid.vid,rc.error))
+
+    key = block:
+      let rc = db.getKeyUbe rvid
+      if rc.isOk:
+        rc.value
+      elif rc.error == GetKeyNotFound:
+        VOID_HASH_KEY
+      else:
+        return err((rvid.vid,rc.error))
+
+  if vtx.isValid:
+    for vid in vtx.subVids:
+      ? db.revSubTree(rev, (rvid.root,vid))
+    rev.sTab[rvid] = vtx
+
+  if key.isValid:
+    rev.kMap[rvid] = key
+
+  ok()
 
 # ------------------------------------------------------------------------------
 # Public functions
@@ -48,7 +87,7 @@ proc revFilter*(
     else:
       return err((rvid.vid,rc.error))
 
-  # Calculate reverse changes for the `kMap` sequence.
+  # Calculate reverse changes for the `kMap[]` structural table.
   for rvid in filter.kMap.keys:
     let rc = db.getKeyUbe rvid
     if rc.isOk:
@@ -58,6 +97,10 @@ proc revFilter*(
     else:
       return err((rvid.vid,rc.error))
 
+  # Reverse changes for `delTree[]` list.
+  for rvid in filter.delTree:
+    ? db.revSubTree(rev, rvid)
+
   ok(rev)
 
 # ------------------------------------------------------------------------------

nimbus/db/aristo/aristo_desc.nim

@@ -37,10 +37,6 @@ export
   tables, aristo_constants, desc_error, desc_identifiers, desc_nibbles,
   desc_structural, keyed_queue
 
-const
-  accLruSize* = 1024 * 1024
-    # LRU cache size for accounts that have storage
-
 type
   AristoTxRef* = ref object
     ## Transaction descriptor

nimbus/db/aristo/aristo_desc/desc_structural.nim

@@ -124,6 +124,8 @@ type
     kMap*: Table[RootedVertexID,HashKey]   ## Merkle hash key mapping
     vTop*: VertexID                        ## Last used vertex ID
 
+    delTree*: seq[RootedVertexID]          ## Not yet fully deleted sub-trees
+
     accLeaves*: Table[Hash256, VertexRef]  ## Account path -> VertexRef
     stoLeaves*: Table[Hash256, VertexRef]  ## Storage path -> VertexRef
 

nimbus/db/aristo/aristo_fetch.nim

@@ -78,7 +78,7 @@ proc retrieveAccountPayload(
         return err(FetchPathNotFound)
       return err(error)
 
-  ok db.accLeaves.lruAppend(accKey, leafVtx, accLruSize).lData
+  ok db.accLeaves.lruAppend(accKey, leafVtx, ACC_LRU_SIZE).lData
 
 proc retrieveMerkleHash(
     db: AristoDbRef;
@@ -188,7 +188,7 @@ proc retrieveStoragePayload(
     leafVtx = db.retrieveLeaf(? db.fetchStorageID(accPath), stoPath.data).valueOr:
       return err(error)
 
-  ok db.stoLeaves.lruAppend(mixKey, leafVtx, accLruSize).lData.stoData
+  ok db.stoLeaves.lruAppend(mixKey, leafVtx, ACC_LRU_SIZE).lData.stoData
 
 proc hasStoragePayload(
     db: AristoDbRef;

nimbus/db/aristo/aristo_layers.nim

@@ -171,6 +171,7 @@ func isEmpty*(ly: LayerRef): bool =
   ## tables are empty. The field `txUid` is ignored, here.
   ly.sTab.len == 0 and
   ly.kMap.len == 0 and
+  ly.delTree.len == 0 and
   ly.accLeaves.len == 0 and
   ly.stoLeaves.len == 0
 
@@ -186,6 +187,8 @@ func layersMergeOnto*(src: LayerRef; trg: var LayerObj) =
   for (vid,key) in src.kMap.pairs:
     trg.kMap[vid] = key
   trg.vTop = src.vTop
+  for rvid in src.delTree:
+    trg.delTree.add rvid
   for (accPath,leafVtx) in src.accLeaves.pairs:
     trg.accLeaves[accPath] = leafVtx
   for (mixPath,leafVtx) in src.stoLeaves.pairs:
@@ -204,6 +207,7 @@ func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
     sTab: layers[0].sTab.dup,          # explicit dup for ref values
     kMap: layers[0].kMap,
     vTop: layers[^1].vTop,
+    delTree: layers[0].delTree,
     accLeaves: layers[0].accLeaves,
     stoLeaves: layers[0].stoLeaves)
 
@@ -213,6 +217,8 @@ func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
       result.sTab[vid] = vtx
     for (vid,key) in layers[n].kMap.pairs:
       result.kMap[vid] = key
+    for rvid in layers[n].delTree:
+      result.delTree.add rvid
     for (accPath,vtx) in layers[n].accLeaves.pairs:
       result.accLeaves[accPath] = vtx
     for (mixPath,vtx) in layers[n].stoLeaves.pairs:

nimbus/utils/utils.nim

@@ -16,7 +16,7 @@ import
   stew/byteutils,
   nimcrypto,
   results,
-  ../db/aristo,
+  ../db/aristo/aristo_sign,
   ../constants
 
 export eth_types_rlp

tests/test_coredb/coredb_test_xx.nim

@@ -124,21 +124,21 @@ let
   mainTest6r* = mainSampleEx
     .cloneWith(
       name      = "-ex-ar-some",
-      numBlocks = 257_400,
+      numBlocks = 1_000_000,
       dbType    = AristoDbRocks,
       dbName    = "main-open") # for resuming on the same persistent DB
 
   mainTest7r* = mainSampleEx
     .cloneWith(
       name      = "-ex-ar-more",
-      numBlocks = 1_460_700,   # failure at 1,460,736
+      numBlocks = 1_500_000,
       dbType    = AristoDbRocks,
       dbName    = "main-open") # for resuming on the same persistent DB
 
   mainTest8r* = mainSampleEx
     .cloneWith(
       name      = "-ex-ar-more2",
-      numBlocks = 1_460_735,   # failure at 1,460,736
+      numBlocks = 2_000_000,
       dbType    = AristoDbRocks,
       dbName    = "main-open") # for resuming on the same persistent DB