Aristo lazily delete larger subtrees (#2560)

* Extract sub-tree deletion functions into separate sub-modules

* Move/rename `aristo_desc.accLruSize` => `aristo_constants.ACC_LRU_SIZE`

* Lazily delete sub-trees

why:
  This gives some control of the memory used to keep the deleted vertices
  in the cached layers. For larger sub-trees, keys and vertices might be
  on the persistent backend to a large extend. This would pull an amount
  of extra information from the backend into the cached layer.

  For lazy deleting it is enough to remember sub-trees by a small set of
  (at most 16) sub-roots to be processed when storing persistent data.
  Marking the tree root deleted immediately allows to let most of the code
  base work as before.

* Comments and cosmetics

* No need to import all for `Aristo` here

* Kludge to make `chronicle` usage in sub-modules work with `fluffy`

why:
  That `fluffy` would not run with any logging in `core_deb` is a problem
  I have known for a while. Up to now, logging was only used for debugging.

  With the current `Aristo` PR, there are cases where logging might be
  wanted but this works only if `chronicles` runs without the
  `json[dynamic]` sinks.

  So this should be re-visited.

* More of a kludge
This commit is contained in:
Jordan Hrycaj 2024-08-14 08:54:44 +00:00 committed by GitHub
parent e3908a7b0d
commit ce713d95fc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 655 additions and 102 deletions

View File

@ -1,4 +1,11 @@
-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
#
# For some reason `json[dynamic]` causes problems with subsequent modules from
# `Aristo` when compiling `fluffy`. There might be a `chronicles` inport missing
# but it is not obvious where. -- jordan
#
#-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
-d:"chronicles_sinks=textlines[dynamic]"
-d:"chronicles_runtime_filtering=on"
-d:"chronicles_disable_thread_id"

View File

@ -1,4 +1,6 @@
# Use only `secp256k1` public key cryptography as an identity in LibP2P.
-d:"libp2p_pki_schemes=secp256k1"
-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
# See `fluffy.nim.cfg`
#-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
-d:"chronicles_sinks=textlines[dynamic]"

View File

@ -1 +1,3 @@
-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
# See `fluffy.nim.cfg`
#-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
-d:"chronicles_sinks=textlines[dynamic]"

View File

@ -1,9 +1,4 @@
* Re-visit `delTree()`. Suggestion is deleting small trees on the memory later,
otherwise only deleting the root vertex (so it becomes inaccessible) and
remember the follow up vertices which can travel through the tx-layers
to be picked up by the backend store.
* Some comletions migh be needed for the `aristo_part` module which is a
* Some comletions might be needed for the `aristo_part` module which is a
re-implementation of the module supporting *proof-mode*/partial trees.
+ Complete `partMergeStorageData()`. This function might not be needed at
all unless *snap-sync* is really revived.

View File

@ -38,6 +38,19 @@ const
## functions with fixed assignments of the type of a state root (e.g. for
## a receipt or a transaction root.)
ACC_LRU_SIZE* = 1024 * 1024
## LRU cache size for accounts that have storage, see `.accLeaves` and
## `.stoLeaves` fields of the main descriptor.
DELETE_SUBTREE_VERTICES_MAX* = 25
## Maximum number of vertices for a tree to be deleted instantly. If the
## tree is larger, only the sub-tree root will be deleted immediately and
## subsequent entries will be deleted not until the cache layers are saved
## to the backend.
##
## Set to zero to disable in which case all sub-trees are deleted
## immediately.
static:
# must stay away from `VertexID(1)` and `VertexID(2)`
doAssert 2 < LEAST_FREE_VID

View File

@ -195,7 +195,7 @@ proc ppPayload(p: LeafPayload, db: AristoDbRef): string =
of AccountData:
result = "(" & p.account.ppAriAccount() & "," & p.stoID.ppVid & ")"
of StoData:
result = $p.stoData
result = ($p.stoData).squeeze
proc ppVtx(nd: VertexRef, db: AristoDbRef, rvid: RootedVertexID): string =
if not nd.isValid:

View File

@ -11,7 +11,6 @@
## Aristo DB -- Patricia Trie delete funcionality
## ==============================================
##
## Delete by `Hike` type chain of vertices.
{.push raises: [].}
@ -19,6 +18,7 @@ import
std/typetraits,
eth/common,
results,
./aristo_delete/[delete_helpers, delete_subtree],
"."/[aristo_desc, aristo_fetch, aristo_get, aristo_hike, aristo_layers,
aristo_utils]
@ -39,79 +39,10 @@ proc branchStillNeeded(vtx: VertexRef): Result[int,void] =
# Oops, degenerated branch node
err()
# -----------
proc disposeOfVtx(
db: AristoDbRef; # Database, top layer
rvid: RootedVertexID; # Vertex ID to clear
) =
# Remove entry
db.layersResVtx(rvid)
db.layersResKey(rvid)
# ------------------------------------------------------------------------------
# Private functions
# ------------------------------------------------------------------------------
proc delSubTreeImpl(
db: AristoDbRef; # Database, top layer
root: VertexID; # Root vertex
): Result[void,AristoError] =
## Implementation of *delete* sub-trie.
var
dispose = @[root]
(rootVtx, _) = db.getVtxRc((root, root)).valueOr:
if error == GetVtxNotFound:
return ok()
return err(error)
follow = @[rootVtx]
# Collect list of nodes to delete
while 0 < follow.len:
var redo: seq[VertexRef]
for vtx in follow:
for vid in vtx.subVids:
# Exiting here leaves the tree as-is
let vtx = (? db.getVtxRc((root, vid)))[0]
redo.add vtx
dispose.add vid
redo.swap follow
# Mark collected vertices to be deleted
for vid in dispose:
db.disposeOfVtx((root, vid))
ok()
proc delStoTreeImpl(
db: AristoDbRef; # Database, top layer
rvid: RootedVertexID; # Root vertex
accPath: Hash256;
stoPath: NibblesBuf;
): Result[void,AristoError] =
## Implementation of *delete* sub-trie.
let (vtx, _) = db.getVtxRc(rvid).valueOr:
if error == GetVtxNotFound:
return ok()
return err(error)
case vtx.vType
of Branch:
for i in 0..15:
if vtx.bVid[i].isValid:
? db.delStoTreeImpl(
(rvid.root, vtx.bVid[i]), accPath,
stoPath & vtx.ePfx & NibblesBuf.nibble(byte i))
of Leaf:
let stoPath = Hash256(data: (stoPath & vtx.lPfx).getBytes())
db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), nil)
db.disposeOfVtx(rvid)
ok()
proc deleteImpl(
db: AristoDbRef; # Database, top layer
hike: Hike; # Fully expanded path
@ -199,7 +130,7 @@ proc deleteAccountRecord*(
# Delete storage tree if present
if stoID.isValid:
? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath, NibblesBuf())
? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath)
?db.deleteImpl(hike)
@ -322,7 +253,7 @@ proc deleteStorageTree*(
# Mark account path Merkle keys for update
db.updateAccountForHasher accHike
? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath, NibblesBuf())
? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath)
# De-register the deleted storage tree from the accounts record
let leaf = wpAcc.vtx.dup # Dup on modify

View File

@ -0,0 +1,127 @@
# nimbus-eth1
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
{.push raises: [].}
import
std/[math, strformat, times],
chronicles,
".."/[aristo_desc, aristo_get, aristo_profile]
export
aristo_profile.toStr
type
SubTreeStats* = tuple
nVtxs: int ## Number of vertices in sub-tree
nLeafs: int ## Number of leafs in sub-tree
depthMax: int ## Maximal vertex path length
nStoCache: int ## Size of storage leafs cache
elapsed: Duration ## Time spent analysing
SubTreeStatsAccu* = tuple
count: int ## Number of entries
sVtxs, qVtxs: float ## Sum and square sum of `.nVtxs`
sLeafs, qLeafs: float ## Sum and square sum of `.nLeafs`
sDepth, qDepth: float ## Sum and square sum of `.depthMax`
sElapsed: Duration ## Sum of `.elapsed`
SubTreeDist* = tuple
count: int ## Number of entries
mVtxs, dVtxs: float ## Mean and std deviation of `.nVtxs`
mLeafs, dLeafs: float ## Mean and std deviation of `.nLeafs`
mDepth, dDepth: float ## Mean and std deviation of `.depthMax`
# ------------------------------------------------------------------------------
# Prival helper
# ------------------------------------------------------------------------------
proc analyseSubTreeImpl(
db: AristoDbRef; # Database, top layer
rvid: RootedVertexID; # Root vertex
depth: int; # Recursion depth
stats: var SubTreeStats; # Statistics
) =
let (vtx, _) = db.getVtxRc(rvid).valueOr:
return
stats.nVtxs.inc
if stats.depthMax < depth:
stats.depthMax = depth
case vtx.vType:
of Branch:
for n in 0..15:
if vtx.bVid[n].isValid:
db.analyseSubTreeImpl((rvid.root,vtx.bVid[n]), depth+1, stats)
of Leaf:
stats.nLeafs.inc
func evalDist(count: int; sum, sqSum: float): tuple[mean, stdDev: float] =
result.mean = sum / count.float
let
sqMean = sqSum / count.float
meanSq = result.mean * result.mean
# Mathematically, `meanSq <= sqMean` but there might be rounding errors
# if `meanSq` and `sqMean` are approximately the same.
sigma = sqMean - min(meanSq,sqMean)
result.stdDev = sigma.sqrt
# ------------------------------------------------------------------------------
# Public analysis tools
# ------------------------------------------------------------------------------
proc analyseSubTree*(
db: AristoDbRef; # Database, top layer
rvid: RootedVertexID; # Root vertex
minVtxs: int; # Accumulate if `minVtxs` <= `.nVtxs`
accu: var SubTreeStatsAccu; # For accumulated statistics
): SubTreeStats =
let start = getTime()
db.analyseSubTreeImpl(rvid, 1, result)
result.nStoCache = db.stoLeaves.len
if minVtxs <= result.nVtxs:
accu.count.inc
accu.sVtxs += result.nVtxs.float
accu.qVtxs += (result.nVtxs * result.nVtxs).float
accu.sLeafs += result.nLeafs.float
accu.qLeafs += (result.nLeafs * result.nLeafs).float
accu.sDepth += result.depthMax.float
accu.qDepth += (result.depthMax * result.depthMax).float
result.elapsed = getTime() - start
accu.sElapsed += result.elapsed # Unconditionally collecrd
func stats*(a: SubTreeStatsAccu): SubTreeDist =
result.count = a.count
(result.mVtxs, result.dVtxs) = evalDist(a.count, a.sVtxs, a.qVtxs)
(result.mLeafs, result.dLeafs) = evalDist(a.count, a.sLeafs, a.qLeafs)
(result.mDepth, result.dDepth) = evalDist(a.count, a.sDepth, a.qDepth)
func strStats*(
a: SubTreeStatsAccu;
): tuple[count, vtxs, leafs, depth, elapsed: string] =
let w = a.stats()
result.count = $w.count
result.elapsed = a.sElapsed.toStr
result.vtxs = &"{w.mVtxs:.1f}[{w.dVtxs:.1f}]"
result.leafs = &"{w.mLeafs:.1f}[{w.dLeafs:.1f}]"
result.depth = &"{w.mDepth:.1f}[{w.dDepth:.1f}]"
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

View File

@ -0,0 +1,25 @@
# nimbus-eth1
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
{.push raises: [].}
import
".."/[aristo_desc, aristo_layers]
proc disposeOfVtx*(
db: AristoDbRef; # Database, top layer
rvid: RootedVertexID; # Vertex ID to clear
) =
# Remove entry
db.layersResVtx(rvid)
db.layersResKey(rvid)
# End

View File

@ -0,0 +1,20 @@
# nimbus-eth1
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
import ../aristo_constants
when DELETE_SUBTREE_VERTICES_MAX == 0:
import ./delete_subtree_now as del_sub
else:
import ./delete_subtree_lazy as del_sub
export del_sub
# End

View File

@ -0,0 +1,252 @@
# nimbus-eth1
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
{.push raises: [].}
import
std/sets,
eth/common,
results,
".."/[aristo_desc, aristo_get, aristo_layers],
./delete_helpers
const
extraDebuggingMessages = false # or true
## Enable additional logging noise. Note that this might slow down the
## system performance but will not be too significant. When importing the
## first 5m blocks from `era1` on some Debian system,
## * loading time was ~5h
## * overhead of accumulated analysis times was ~1.2s
type
VidCollect = tuple
data: array[DELETE_SUBTREE_VERTICES_MAX,VertexID]
top: int # Next free slot if smaller `.data.len`
when extraDebuggingMessages:
import
std/times,
chronicles,
./delete_debug
const
allStatsFrequency = 20
## Print accumutated statistics every `allStatsFrequency` visits of
## the analysis tool.
minVtxsForLogging = 1000
## Suppress detailed logging for smaller sub-trees
var stats: SubTreeStatsAccu
## Accumulated statistics
func `$`(ela: Duration): string =
ela.toStr
template debugLog(info: static[string]; args: varargs[untyped]) =
## Statistics message via `chronicles` logger, makes it easy to
## change priority and format.
notice info, args
# ------------------------------------------------------------------------------
# Private heplers
# ------------------------------------------------------------------------------
func capa(T: type VidCollect): int =
## Syntactic sugar
T.default.data.len
proc collectSubTreeLazily(
db: AristoDbRef; # Database, top layer
rvid: RootedVertexID; # Root vertex
vids: var VidCollect; # Accumulating vertex IDs for deletion
): Result[void,AristoError] =
## Collect vids for a small sub-tree
let (vtx, _) = db.getVtxRc(rvid).valueOr:
if error == GetVtxNotFound:
return ok()
return err(error)
if vids.top < vids.data.len:
vids.data[vids.top] = rvid.vid
vids.top.inc # Max value of `.top`: `vid.data.len + 1`
if vtx.vType == Branch:
for n in 0..15:
if vtx.bVid[n].isValid:
? db.collectSubTreeLazily((rvid.root,vtx.bVid[n]), vids)
elif vids.top <= vids.data.len:
vids.top.inc # Terminates here
ok()
proc collectStoTreeLazily(
db: AristoDbRef; # Database, top layer
rvid: RootedVertexID; # Root vertex
accPath: Hash256; # Accounts cache designator
stoPath: NibblesBuf; # Current storage path
vids: var VidCollect; # Accumulating vertex IDs for deletion
): Result[void,AristoError] =
## Collect vertex/vid and delete cache entries.
let (vtx, _) = db.getVtxRc(rvid).valueOr:
if error == GetVtxNotFound:
return ok()
return err(error)
case vtx.vType
of Branch:
for i in 0..15:
if vtx.bVid[i].isValid:
? db.collectStoTreeLazily(
(rvid.root, vtx.bVid[i]), accPath,
stoPath & vtx.ePfx & NibblesBuf.nibble(byte i),
vids)
of Leaf:
let stoPath = Hash256(data: (stoPath & vtx.lPfx).getBytes())
db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), nil)
# There is no useful approach avoiding to walk the whole tree for updating
# the storage data access cache.
#
# The alternative of stopping here and clearing the whole cache did degrade
# performance significantly in some tests on mainnet when importing `era1`.
#
# When not clearing the cache it was seen
# * filled up to maximum size most of the time
# * at the same time having no `stoPath` hit at all (so there was nothing
# to be cleared.)
#
if vids.top <= vids.data.len:
if vids.top < vids.data.len:
vids.data[vids.top] = rvid.vid
vids.top.inc # Max value of `.top`: `vid.data.len + 1`
ok()
proc disposeOfSubTree(
db: AristoDbRef; # Database, top layer
rvid: RootedVertexID; # Root vertex
vids: var VidCollect; # Accumulated vertex IDs for disposal
) =
## Evaluate results from `collectSubTreeLazyImpl()` or ftom
## `collectStoTreeLazyImpl)`.
##
if vids.top <= typeof(vids).capa:
# Delete small tree
for n in 0 ..< vids.top:
db.disposeOfVtx((rvid.root, vids.data[n]))
else:
# Mark the sub-trees disabled to be deleted not until the layer is
# finally stored onto the backend.
let vtx = db.getVtxRc(rvid).value[0]
for n in 0..15:
if vtx.bVid[n].isValid:
db.top.delTree.add (rvid.root,vtx.bVid[n])
# Delete top of tree now.
db.disposeOfVtx(rvid)
# ------------------------------------------------------------------------------
# Public functions
# ------------------------------------------------------------------------------
proc delSubTreeImpl*(
db: AristoDbRef; # Database, top layer
root: VertexID; # Root vertex
): Result[void,AristoError] =
## Delete all the `subRoots`if there are a few, only. Otherwise
## mark it for deleting later.
discard db.getVtxRc((root,root)).valueOr:
if error == GetVtxNotFound:
return ok()
return err(error)
when extraDebuggingMessages:
let
ana = db.analyseSubTree((root,root), VidCollect.capa+1, stats)
start = getTime()
var dispose: VidCollect
? db.collectSubTreeLazily((root,root), dispose)
db.disposeOfSubTree((root,root), dispose)
when extraDebuggingMessages:
if typeof(dispose).capa < dispose.top:
if minVtxsForLogging < ana.nVtxs:
debugLog("Generic sub-tree analysis",
nVtxs = ana.nVtxs,
nLeafs = ana.nLeafs,
depthMax = ana.depthMax,
nDelTree = db.top.delTree.len,
elaCollect = getTime() - start)
if (stats.count mod allStatsFrequency) == 0:
let
start = getTime()
(count, vtxs, leafs, depth, elapsed) = stats.strStats
debugLog("Sub-tree analysis stats", count, vtxs, leafs, depth, elapsed)
stats.sElapsed += getTime() - start
ok()
proc delStoTreeImpl*(
db: AristoDbRef; # Database, top layer
rvid: RootedVertexID; # Root vertex
accPath: Hash256;
): Result[void,AristoError] =
## Collect vertex/vid and cache entry.
discard db.getVtxRc(rvid).valueOr:
if error == GetVtxNotFound:
return ok()
return err(error)
when extraDebuggingMessages:
let
ana = db.analyseSubTree(rvid, VidCollect.capa+1, stats)
start = getTime()
var dispose: VidCollect # Accumulating vertices for deletion
? db.collectStoTreeLazily(rvid, accPath, NibblesBuf(), dispose)
db.disposeOfSubTree(rvid, dispose)
when extraDebuggingMessages:
if typeof(dispose).capa < dispose.top:
if minVtxsForLogging < ana.nVtxs or db.stoLeaves.len < ana.nStoCache:
debugLog("Storage sub-tree analysis",
nVtxs = ana.nVtxs,
nLeafs = ana.nLeafs,
depthMax = ana.depthMax,
nStoCache = ana.nStoCache,
nStoCacheDelta = ana.nStoCache - db.stoLeaves.len,
nDelTree = db.top.delTree.len,
elaCollect = getTime() - start)
if (stats.count mod allStatsFrequency) == 0:
let
start = getTime()
(count, vtxs, leafs, depth, elapsed) = stats.strStats
debugLog("Sub-tree analysis stats", count, vtxs, leafs, depth, elapsed)
stats.sElapsed += getTime() - start
ok()
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

View File

@ -0,0 +1,101 @@
# nimbus-eth1
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
{.push raises: [].}
import
eth/common,
".."/[aristo_desc, aristo_get, aristo_layers],
./delete_helpers
# ------------------------------------------------------------------------------
# Private heplers
# ------------------------------------------------------------------------------
proc delSubTreeNow(
db: AristoDbRef;
rvid: RootedVertexID;
): Result[void,AristoError] =
## Delete sub-tree now
let (vtx, _) = db.getVtxRc(rvid).valueOr:
if error == GetVtxNotFound:
return ok()
return err(error)
if vtx.vType == Branch:
for n in 0..15:
if vtx.bVid[n].isValid:
? db.delSubTreeNow((rvid.root,vtx.bVid[n]))
db.disposeOfVtx(rvid)
ok()
proc delStoTreeNow(
db: AristoDbRef; # Database, top layer
rvid: RootedVertexID; # Root vertex
accPath: Hash256; # Accounts cache designator
stoPath: NibblesBuf; # Current storage path
): Result[void,AristoError] =
## Implementation of *delete* sub-trie.
let (vtx, _) = db.getVtxRc(rvid).valueOr:
if error == GetVtxNotFound:
return ok()
return err(error)
case vtx.vType
of Branch:
for i in 0..15:
if vtx.bVid[i].isValid:
? db.delStoTreeNow(
(rvid.root, vtx.bVid[i]), accPath,
stoPath & vtx.ePfx & NibblesBuf.nibble(byte i))
of Leaf:
let stoPath = Hash256(data: (stoPath & vtx.lPfx).getBytes())
db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), nil)
db.disposeOfVtx(rvid)
ok()
# ------------------------------------------------------------------------------
# Public functions
# ------------------------------------------------------------------------------
proc delSubTreeImpl*(
db: AristoDbRef;
root: VertexID;
): Result[void,AristoError] =
discard db.getVtxRc((root, root)).valueOr:
if error == GetVtxNotFound:
return ok()
return err(error)
db.delSubTreeNow (root,root)
proc delStoTreeImpl*(
db: AristoDbRef; # Database, top layer
rvid: RootedVertexID; # Root vertex
accPath: Hash256;
): Result[void,AristoError] =
## Implementation of *delete* sub-trie.
discard db.getVtxRc(rvid).valueOr:
if error == GetVtxNotFound:
return ok()
return err(error)
db.delStoTreeNow(rvid, accPath, NibblesBuf())
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

View File

@ -14,11 +14,32 @@
import
std/tables,
chronicles,
eth/common,
results,
"."/[aristo_desc, aristo_layers],
./aristo_delta/[delta_merge, delta_reverse],
./aristo_desc/desc_backend,
./aristo_delta/[delta_merge, delta_reverse]
"."/[aristo_desc, aristo_get, aristo_layers, aristo_utils]
logScope:
topics = "aristo-delta"
# ------------------------------------------------------------------------------
# Private functions
# ------------------------------------------------------------------------------
proc delSubTree(db: AristoDbRef; writer: PutHdlRef; rvid: RootedVertexID) =
## Collect subtrees marked for deletion
let (vtx,_) = db.getVtxRc(rvid).valueOr:
notice "Descending for deletion stopped", rvid, error
return
for vid in vtx.subVids:
db.delSubTree(writer, (rvid.root, vid))
db.backend.putVtxFn(writer, rvid, VertexRef(nil))
db.backend.putKeyFn(writer, rvid, VOID_HASH_KEY)
# Make sure the `rvid` is not mentioned here, anymore for furter update.
db.balancer.sTab.del rvid
db.balancer.kMap.del rvid
# ------------------------------------------------------------------------------
# Public functions, save to backend
@ -85,6 +106,12 @@ proc deltaPersistent*(
# Store structural single trie entries
let writeBatch = ? be.putBegFn()
# This one must come first in order to avoid duplicate `sTree[]` or
# `kMap[]` instructions, in the worst case overwiting previously deleted
# entries.
for rvid in db.balancer.delTree:
db.delSubTree(writeBatch, rvid)
# Now the standard `sTree[]` and `kMap[]` instructions.
for rvid, vtx in db.balancer.sTab:
be.putVtxFn(writeBatch, rvid, vtx)
for rvid, key in db.balancer.kMap:
@ -97,12 +124,12 @@ proc deltaPersistent*(
for accPath, vtx in db.balancer.accLeaves:
let accKey = accPath.to(AccountKey)
if not db.accLeaves.lruUpdate(accKey, vtx):
discard db.accLeaves.lruAppend(accKey, vtx, accLruSize)
discard db.accLeaves.lruAppend(accKey, vtx, ACC_LRU_SIZE)
for mixPath, vtx in db.balancer.stoLeaves:
let mixKey = mixPath.to(AccountKey)
if not db.stoLeaves.lruUpdate(mixKey, vtx):
discard db.stoLeaves.lruAppend(mixKey, vtx, accLruSize)
discard db.stoLeaves.lruAppend(mixKey, vtx, ACC_LRU_SIZE)
# Done with balancer, all saved to backend
db.balancer = LayerRef(nil)

View File

@ -48,13 +48,14 @@ proc deltaMerge*(
result = LayerRef(
sTab: lower.sTab, # shallow copy (entries will not be modified)
kMap: lower.kMap,
delTree: lower.delTree,
accLeaves: lower.accLeaves,
stoLeaves: lower.stoLeaves,
vTop: upper.vTop)
layersMergeOnto(upper, result[])
else:
# Otherwise avoid copying some tables by modifyinh `upper`. This is not
# Otherwise avoid copying some tables by modifying `upper`. This is not
# completely free as the merge direction changes to merging the `lower`
# layer up into the higher prioritised `upper` layer (note that the `lower`
# argument filter is read-only.) Here again, the `upper` argument must not
@ -67,6 +68,9 @@ proc deltaMerge*(
if not upper.kMap.hasKey(rvid):
upper.kMap[rvid] = key
for rvid in lower.delTree:
upper.delTree.add rvid
for (accPath,leafVtx) in lower.accLeaves.pairs:
if not upper.accLeaves.hasKey(accPath):
upper.accLeaves[accPath] = leafVtx

View File

@ -12,7 +12,46 @@ import
std/tables,
eth/common,
results,
".."/[aristo_desc, aristo_get]
".."/[aristo_desc, aristo_get, aristo_utils]
# ------------------------------------------------------------------------------
# Private functions
# ------------------------------------------------------------------------------
proc revSubTree(
db: AristoDbRef;
rev: LayerRef;
rvid: RootedVertexID;
): Result[void,(VertexID,AristoError)] =
## Collect subtrees marked for deletion
let
vtx = block:
let rc = db.getVtxUbe rvid
if rc.isOk:
rc.value
elif rc.error == GetVtxNotFound:
VertexRef(nil)
else:
return err((rvid.vid,rc.error))
key = block:
let rc = db.getKeyUbe rvid
if rc.isOk:
rc.value
elif rc.error == GetKeyNotFound:
VOID_HASH_KEY
else:
return err((rvid.vid,rc.error))
if vtx.isValid:
for vid in vtx.subVids:
? db.revSubTree(rev, (rvid.root,vid))
rev.sTab[rvid] = vtx
if key.isValid:
rev.kMap[rvid] = key
ok()
# ------------------------------------------------------------------------------
# Public functions
@ -48,7 +87,7 @@ proc revFilter*(
else:
return err((rvid.vid,rc.error))
# Calculate reverse changes for the `kMap` sequence.
# Calculate reverse changes for the `kMap[]` structural table.
for rvid in filter.kMap.keys:
let rc = db.getKeyUbe rvid
if rc.isOk:
@ -58,6 +97,10 @@ proc revFilter*(
else:
return err((rvid.vid,rc.error))
# Reverse changes for `delTree[]` list.
for rvid in filter.delTree:
? db.revSubTree(rev, rvid)
ok(rev)
# ------------------------------------------------------------------------------

View File

@ -37,10 +37,6 @@ export
tables, aristo_constants, desc_error, desc_identifiers, desc_nibbles,
desc_structural, keyed_queue
const
accLruSize* = 1024 * 1024
# LRU cache size for accounts that have storage
type
AristoTxRef* = ref object
## Transaction descriptor

View File

@ -124,6 +124,8 @@ type
kMap*: Table[RootedVertexID,HashKey] ## Merkle hash key mapping
vTop*: VertexID ## Last used vertex ID
delTree*: seq[RootedVertexID] ## Not yet fully deleted sub-trees
accLeaves*: Table[Hash256, VertexRef] ## Account path -> VertexRef
stoLeaves*: Table[Hash256, VertexRef] ## Storage path -> VertexRef

View File

@ -78,7 +78,7 @@ proc retrieveAccountPayload(
return err(FetchPathNotFound)
return err(error)
ok db.accLeaves.lruAppend(accKey, leafVtx, accLruSize).lData
ok db.accLeaves.lruAppend(accKey, leafVtx, ACC_LRU_SIZE).lData
proc retrieveMerkleHash(
db: AristoDbRef;
@ -188,7 +188,7 @@ proc retrieveStoragePayload(
leafVtx = db.retrieveLeaf(? db.fetchStorageID(accPath), stoPath.data).valueOr:
return err(error)
ok db.stoLeaves.lruAppend(mixKey, leafVtx, accLruSize).lData.stoData
ok db.stoLeaves.lruAppend(mixKey, leafVtx, ACC_LRU_SIZE).lData.stoData
proc hasStoragePayload(
db: AristoDbRef;

View File

@ -171,6 +171,7 @@ func isEmpty*(ly: LayerRef): bool =
## tables are empty. The field `txUid` is ignored, here.
ly.sTab.len == 0 and
ly.kMap.len == 0 and
ly.delTree.len == 0 and
ly.accLeaves.len == 0 and
ly.stoLeaves.len == 0
@ -186,6 +187,8 @@ func layersMergeOnto*(src: LayerRef; trg: var LayerObj) =
for (vid,key) in src.kMap.pairs:
trg.kMap[vid] = key
trg.vTop = src.vTop
for rvid in src.delTree:
trg.delTree.add rvid
for (accPath,leafVtx) in src.accLeaves.pairs:
trg.accLeaves[accPath] = leafVtx
for (mixPath,leafVtx) in src.stoLeaves.pairs:
@ -204,6 +207,7 @@ func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
sTab: layers[0].sTab.dup, # explicit dup for ref values
kMap: layers[0].kMap,
vTop: layers[^1].vTop,
delTree: layers[0].delTree,
accLeaves: layers[0].accLeaves,
stoLeaves: layers[0].stoLeaves)
@ -213,6 +217,8 @@ func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
result.sTab[vid] = vtx
for (vid,key) in layers[n].kMap.pairs:
result.kMap[vid] = key
for rvid in layers[n].delTree:
result.delTree.add rvid
for (accPath,vtx) in layers[n].accLeaves.pairs:
result.accLeaves[accPath] = vtx
for (mixPath,vtx) in layers[n].stoLeaves.pairs:

View File

@ -16,7 +16,7 @@ import
stew/byteutils,
nimcrypto,
results,
../db/aristo,
../db/aristo/aristo_sign,
../constants
export eth_types_rlp

View File

@ -124,21 +124,21 @@ let
mainTest6r* = mainSampleEx
.cloneWith(
name = "-ex-ar-some",
numBlocks = 257_400,
numBlocks = 1_000_000,
dbType = AristoDbRocks,
dbName = "main-open") # for resuming on the same persistent DB
mainTest7r* = mainSampleEx
.cloneWith(
name = "-ex-ar-more",
numBlocks = 1_460_700, # failure at 1,460,736
numBlocks = 1_500_000,
dbType = AristoDbRocks,
dbName = "main-open") # for resuming on the same persistent DB
mainTest8r* = mainSampleEx
.cloneWith(
name = "-ex-ar-more2",
numBlocks = 1_460_735, # failure at 1,460,736
numBlocks = 2_000_000,
dbType = AristoDbRocks,
dbName = "main-open") # for resuming on the same persistent DB