Aristo lazily delete larger subtrees (#2560)
* Extract sub-tree deletion functions into separate sub-modules * Move/rename `aristo_desc.accLruSize` => `aristo_constants.ACC_LRU_SIZE` * Lazily delete sub-trees why: This gives some control of the memory used to keep the deleted vertices in the cached layers. For larger sub-trees, keys and vertices might be on the persistent backend to a large extend. This would pull an amount of extra information from the backend into the cached layer. For lazy deleting it is enough to remember sub-trees by a small set of (at most 16) sub-roots to be processed when storing persistent data. Marking the tree root deleted immediately allows to let most of the code base work as before. * Comments and cosmetics * No need to import all for `Aristo` here * Kludge to make `chronicle` usage in sub-modules work with `fluffy` why: That `fluffy` would not run with any logging in `core_deb` is a problem I have known for a while. Up to now, logging was only used for debugging. With the current `Aristo` PR, there are cases where logging might be wanted but this works only if `chronicles` runs without the `json[dynamic]` sinks. So this should be re-visited. * More of a kludge
This commit is contained in:
Jordan Hrycaj
GitHub
parent
e3908a7b0d
commit
ce713d95fc
|
|
@ -1,4 +1,11 @@
|
|||
-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
|
||||
#
|
||||
# For some reason `json[dynamic]` causes problems with subsequent modules from
|
||||
# `Aristo` when compiling `fluffy`. There might be a `chronicles` inport missing
|
||||
# but it is not obvious where. -- jordan
|
||||
#
|
||||
#-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
|
||||
|
||||
-d:"chronicles_sinks=textlines[dynamic]"
|
||||
-d:"chronicles_runtime_filtering=on"
|
||||
-d:"chronicles_disable_thread_id"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
# Use only `secp256k1` public key cryptography as an identity in LibP2P.
|
||||
-d:"libp2p_pki_schemes=secp256k1"
|
||||
|
||||
-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
|
||||
# See `fluffy.nim.cfg`
|
||||
#-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
|
||||
-d:"chronicles_sinks=textlines[dynamic]"
|
||||
|
|
|
|||
|
|
@ -1 +1,3 @@
|
|||
-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
|
||||
# See `fluffy.nim.cfg`
|
||||
#-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
|
||||
-d:"chronicles_sinks=textlines[dynamic]"
|
||||
|
|
|
|||
|
|
@ -1,9 +1,4 @@
|
|||
* Re-visit `delTree()`. Suggestion is deleting small trees on the memory later,
|
||||
otherwise only deleting the root vertex (so it becomes inaccessible) and
|
||||
remember the follow up vertices which can travel through the tx-layers
|
||||
to be picked up by the backend store.
|
||||
|
||||
* Some comletions migh be needed for the `aristo_part` module which is a
|
||||
* Some comletions might be needed for the `aristo_part` module which is a
|
||||
re-implementation of the module supporting *proof-mode*/partial trees.
|
||||
+ Complete `partMergeStorageData()`. This function might not be needed at
|
||||
all unless *snap-sync* is really revived.
|
||||
|
|
|
|||
|
|
@ -38,6 +38,19 @@ const
|
|||
## functions with fixed assignments of the type of a state root (e.g. for
|
||||
## a receipt or a transaction root.)
|
||||
|
||||
ACC_LRU_SIZE* = 1024 * 1024
|
||||
## LRU cache size for accounts that have storage, see `.accLeaves` and
|
||||
## `.stoLeaves` fields of the main descriptor.
|
||||
|
||||
DELETE_SUBTREE_VERTICES_MAX* = 25
|
||||
## Maximum number of vertices for a tree to be deleted instantly. If the
|
||||
## tree is larger, only the sub-tree root will be deleted immediately and
|
||||
## subsequent entries will be deleted not until the cache layers are saved
|
||||
## to the backend.
|
||||
##
|
||||
## Set to zero to disable in which case all sub-trees are deleted
|
||||
## immediately.
|
||||
|
||||
static:
|
||||
# must stay away from `VertexID(1)` and `VertexID(2)`
|
||||
doAssert 2 < LEAST_FREE_VID
|
||||
|
|
|
|||
|
|
@ -195,7 +195,7 @@ proc ppPayload(p: LeafPayload, db: AristoDbRef): string =
|
|||
of AccountData:
|
||||
result = "(" & p.account.ppAriAccount() & "," & p.stoID.ppVid & ")"
|
||||
of StoData:
|
||||
result = $p.stoData
|
||||
result = ($p.stoData).squeeze
|
||||
|
||||
proc ppVtx(nd: VertexRef, db: AristoDbRef, rvid: RootedVertexID): string =
|
||||
if not nd.isValid:
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@
|
|||
## Aristo DB -- Patricia Trie delete funcionality
|
||||
## ==============================================
|
||||
##
|
||||
## Delete by `Hike` type chain of vertices.
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
|
|
@ -19,6 +18,7 @@ import
|
|||
std/typetraits,
|
||||
eth/common,
|
||||
results,
|
||||
./aristo_delete/[delete_helpers, delete_subtree],
|
||||
"."/[aristo_desc, aristo_fetch, aristo_get, aristo_hike, aristo_layers,
|
||||
aristo_utils]
|
||||
|
||||
|
|
@ -39,79 +39,10 @@ proc branchStillNeeded(vtx: VertexRef): Result[int,void] =
|
|||
# Oops, degenerated branch node
|
||||
err()
|
||||
|
||||
# -----------
|
||||
|
||||
proc disposeOfVtx(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
rvid: RootedVertexID; # Vertex ID to clear
|
||||
) =
|
||||
# Remove entry
|
||||
db.layersResVtx(rvid)
|
||||
db.layersResKey(rvid)
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Private functions
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc delSubTreeImpl(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
root: VertexID; # Root vertex
|
||||
): Result[void,AristoError] =
|
||||
## Implementation of *delete* sub-trie.
|
||||
var
|
||||
dispose = @[root]
|
||||
(rootVtx, _) = db.getVtxRc((root, root)).valueOr:
|
||||
if error == GetVtxNotFound:
|
||||
return ok()
|
||||
return err(error)
|
||||
follow = @[rootVtx]
|
||||
|
||||
# Collect list of nodes to delete
|
||||
while 0 < follow.len:
|
||||
var redo: seq[VertexRef]
|
||||
for vtx in follow:
|
||||
for vid in vtx.subVids:
|
||||
# Exiting here leaves the tree as-is
|
||||
let vtx = (? db.getVtxRc((root, vid)))[0]
|
||||
redo.add vtx
|
||||
dispose.add vid
|
||||
redo.swap follow
|
||||
|
||||
# Mark collected vertices to be deleted
|
||||
for vid in dispose:
|
||||
db.disposeOfVtx((root, vid))
|
||||
|
||||
ok()
|
||||
|
||||
proc delStoTreeImpl(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
rvid: RootedVertexID; # Root vertex
|
||||
accPath: Hash256;
|
||||
stoPath: NibblesBuf;
|
||||
): Result[void,AristoError] =
|
||||
## Implementation of *delete* sub-trie.
|
||||
|
||||
let (vtx, _) = db.getVtxRc(rvid).valueOr:
|
||||
if error == GetVtxNotFound:
|
||||
return ok()
|
||||
return err(error)
|
||||
|
||||
case vtx.vType
|
||||
of Branch:
|
||||
for i in 0..15:
|
||||
if vtx.bVid[i].isValid:
|
||||
? db.delStoTreeImpl(
|
||||
(rvid.root, vtx.bVid[i]), accPath,
|
||||
stoPath & vtx.ePfx & NibblesBuf.nibble(byte i))
|
||||
|
||||
of Leaf:
|
||||
let stoPath = Hash256(data: (stoPath & vtx.lPfx).getBytes())
|
||||
db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), nil)
|
||||
|
||||
db.disposeOfVtx(rvid)
|
||||
|
||||
ok()
|
||||
|
||||
proc deleteImpl(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
hike: Hike; # Fully expanded path
|
||||
|
|
@ -199,7 +130,7 @@ proc deleteAccountRecord*(
|
|||
|
||||
# Delete storage tree if present
|
||||
if stoID.isValid:
|
||||
? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath, NibblesBuf())
|
||||
? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath)
|
||||
|
||||
?db.deleteImpl(hike)
|
||||
|
||||
|
|
@ -322,7 +253,7 @@ proc deleteStorageTree*(
|
|||
# Mark account path Merkle keys for update
|
||||
db.updateAccountForHasher accHike
|
||||
|
||||
? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath, NibblesBuf())
|
||||
? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath)
|
||||
|
||||
# De-register the deleted storage tree from the accounts record
|
||||
let leaf = wpAcc.vtx.dup # Dup on modify
|
||||
|
|
|
|||
|
|
@ -0,0 +1,127 @@
|
|||
# nimbus-eth1
|
||||
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import
|
||||
std/[math, strformat, times],
|
||||
chronicles,
|
||||
".."/[aristo_desc, aristo_get, aristo_profile]
|
||||
|
||||
export
|
||||
aristo_profile.toStr
|
||||
|
||||
type
|
||||
SubTreeStats* = tuple
|
||||
nVtxs: int ## Number of vertices in sub-tree
|
||||
nLeafs: int ## Number of leafs in sub-tree
|
||||
depthMax: int ## Maximal vertex path length
|
||||
nStoCache: int ## Size of storage leafs cache
|
||||
elapsed: Duration ## Time spent analysing
|
||||
|
||||
SubTreeStatsAccu* = tuple
|
||||
count: int ## Number of entries
|
||||
sVtxs, qVtxs: float ## Sum and square sum of `.nVtxs`
|
||||
sLeafs, qLeafs: float ## Sum and square sum of `.nLeafs`
|
||||
sDepth, qDepth: float ## Sum and square sum of `.depthMax`
|
||||
sElapsed: Duration ## Sum of `.elapsed`
|
||||
|
||||
SubTreeDist* = tuple
|
||||
count: int ## Number of entries
|
||||
mVtxs, dVtxs: float ## Mean and std deviation of `.nVtxs`
|
||||
mLeafs, dLeafs: float ## Mean and std deviation of `.nLeafs`
|
||||
mDepth, dDepth: float ## Mean and std deviation of `.depthMax`
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Prival helper
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc analyseSubTreeImpl(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
rvid: RootedVertexID; # Root vertex
|
||||
depth: int; # Recursion depth
|
||||
stats: var SubTreeStats; # Statistics
|
||||
) =
|
||||
let (vtx, _) = db.getVtxRc(rvid).valueOr:
|
||||
return
|
||||
|
||||
stats.nVtxs.inc
|
||||
|
||||
if stats.depthMax < depth:
|
||||
stats.depthMax = depth
|
||||
|
||||
case vtx.vType:
|
||||
of Branch:
|
||||
for n in 0..15:
|
||||
if vtx.bVid[n].isValid:
|
||||
db.analyseSubTreeImpl((rvid.root,vtx.bVid[n]), depth+1, stats)
|
||||
of Leaf:
|
||||
stats.nLeafs.inc
|
||||
|
||||
|
||||
func evalDist(count: int; sum, sqSum: float): tuple[mean, stdDev: float] =
|
||||
result.mean = sum / count.float
|
||||
|
||||
let
|
||||
sqMean = sqSum / count.float
|
||||
meanSq = result.mean * result.mean
|
||||
|
||||
# Mathematically, `meanSq <= sqMean` but there might be rounding errors
|
||||
# if `meanSq` and `sqMean` are approximately the same.
|
||||
sigma = sqMean - min(meanSq,sqMean)
|
||||
|
||||
result.stdDev = sigma.sqrt
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public analysis tools
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc analyseSubTree*(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
rvid: RootedVertexID; # Root vertex
|
||||
minVtxs: int; # Accumulate if `minVtxs` <= `.nVtxs`
|
||||
accu: var SubTreeStatsAccu; # For accumulated statistics
|
||||
): SubTreeStats =
|
||||
let start = getTime()
|
||||
db.analyseSubTreeImpl(rvid, 1, result)
|
||||
result.nStoCache = db.stoLeaves.len
|
||||
|
||||
if minVtxs <= result.nVtxs:
|
||||
accu.count.inc
|
||||
accu.sVtxs += result.nVtxs.float
|
||||
accu.qVtxs += (result.nVtxs * result.nVtxs).float
|
||||
accu.sLeafs += result.nLeafs.float
|
||||
accu.qLeafs += (result.nLeafs * result.nLeafs).float
|
||||
accu.sDepth += result.depthMax.float
|
||||
accu.qDepth += (result.depthMax * result.depthMax).float
|
||||
|
||||
result.elapsed = getTime() - start
|
||||
accu.sElapsed += result.elapsed # Unconditionally collecrd
|
||||
|
||||
|
||||
func stats*(a: SubTreeStatsAccu): SubTreeDist =
|
||||
result.count = a.count
|
||||
(result.mVtxs, result.dVtxs) = evalDist(a.count, a.sVtxs, a.qVtxs)
|
||||
(result.mLeafs, result.dLeafs) = evalDist(a.count, a.sLeafs, a.qLeafs)
|
||||
(result.mDepth, result.dDepth) = evalDist(a.count, a.sDepth, a.qDepth)
|
||||
|
||||
func strStats*(
|
||||
a: SubTreeStatsAccu;
|
||||
): tuple[count, vtxs, leafs, depth, elapsed: string] =
|
||||
let w = a.stats()
|
||||
result.count = $w.count
|
||||
result.elapsed = a.sElapsed.toStr
|
||||
result.vtxs = &"{w.mVtxs:.1f}[{w.dVtxs:.1f}]"
|
||||
result.leafs = &"{w.mLeafs:.1f}[{w.dLeafs:.1f}]"
|
||||
result.depth = &"{w.mDepth:.1f}[{w.dDepth:.1f}]"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# End
|
||||
# ------------------------------------------------------------------------------
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
# nimbus-eth1
|
||||
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import
|
||||
".."/[aristo_desc, aristo_layers]
|
||||
|
||||
|
||||
proc disposeOfVtx*(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
rvid: RootedVertexID; # Vertex ID to clear
|
||||
) =
|
||||
# Remove entry
|
||||
db.layersResVtx(rvid)
|
||||
db.layersResKey(rvid)
|
||||
|
||||
# End
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
# nimbus-eth1
|
||||
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
import ../aristo_constants
|
||||
|
||||
when DELETE_SUBTREE_VERTICES_MAX == 0:
|
||||
import ./delete_subtree_now as del_sub
|
||||
else:
|
||||
import ./delete_subtree_lazy as del_sub
|
||||
|
||||
export del_sub
|
||||
|
||||
# End
|
||||
|
|
@ -0,0 +1,252 @@
|
|||
# nimbus-eth1
|
||||
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import
|
||||
std/sets,
|
||||
eth/common,
|
||||
results,
|
||||
".."/[aristo_desc, aristo_get, aristo_layers],
|
||||
./delete_helpers
|
||||
|
||||
const
|
||||
extraDebuggingMessages = false # or true
|
||||
## Enable additional logging noise. Note that this might slow down the
|
||||
## system performance but will not be too significant. When importing the
|
||||
## first 5m blocks from `era1` on some Debian system,
|
||||
## * loading time was ~5h
|
||||
## * overhead of accumulated analysis times was ~1.2s
|
||||
|
||||
type
|
||||
VidCollect = tuple
|
||||
data: array[DELETE_SUBTREE_VERTICES_MAX,VertexID]
|
||||
top: int # Next free slot if smaller `.data.len`
|
||||
|
||||
|
||||
when extraDebuggingMessages:
|
||||
import
|
||||
std/times,
|
||||
chronicles,
|
||||
./delete_debug
|
||||
|
||||
const
|
||||
allStatsFrequency = 20
|
||||
## Print accumutated statistics every `allStatsFrequency` visits of
|
||||
## the analysis tool.
|
||||
|
||||
minVtxsForLogging = 1000
|
||||
## Suppress detailed logging for smaller sub-trees
|
||||
|
||||
var stats: SubTreeStatsAccu
|
||||
## Accumulated statistics
|
||||
|
||||
func `$`(ela: Duration): string =
|
||||
ela.toStr
|
||||
|
||||
template debugLog(info: static[string]; args: varargs[untyped]) =
|
||||
## Statistics message via `chronicles` logger, makes it easy to
|
||||
## change priority and format.
|
||||
notice info, args
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Private heplers
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
func capa(T: type VidCollect): int =
|
||||
## Syntactic sugar
|
||||
T.default.data.len
|
||||
|
||||
|
||||
proc collectSubTreeLazily(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
rvid: RootedVertexID; # Root vertex
|
||||
vids: var VidCollect; # Accumulating vertex IDs for deletion
|
||||
): Result[void,AristoError] =
|
||||
## Collect vids for a small sub-tree
|
||||
let (vtx, _) = db.getVtxRc(rvid).valueOr:
|
||||
if error == GetVtxNotFound:
|
||||
return ok()
|
||||
return err(error)
|
||||
|
||||
if vids.top < vids.data.len:
|
||||
vids.data[vids.top] = rvid.vid
|
||||
vids.top.inc # Max value of `.top`: `vid.data.len + 1`
|
||||
|
||||
if vtx.vType == Branch:
|
||||
for n in 0..15:
|
||||
if vtx.bVid[n].isValid:
|
||||
? db.collectSubTreeLazily((rvid.root,vtx.bVid[n]), vids)
|
||||
|
||||
elif vids.top <= vids.data.len:
|
||||
vids.top.inc # Terminates here
|
||||
|
||||
ok()
|
||||
|
||||
|
||||
proc collectStoTreeLazily(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
rvid: RootedVertexID; # Root vertex
|
||||
accPath: Hash256; # Accounts cache designator
|
||||
stoPath: NibblesBuf; # Current storage path
|
||||
vids: var VidCollect; # Accumulating vertex IDs for deletion
|
||||
): Result[void,AristoError] =
|
||||
## Collect vertex/vid and delete cache entries.
|
||||
let (vtx, _) = db.getVtxRc(rvid).valueOr:
|
||||
if error == GetVtxNotFound:
|
||||
return ok()
|
||||
return err(error)
|
||||
|
||||
case vtx.vType
|
||||
of Branch:
|
||||
for i in 0..15:
|
||||
if vtx.bVid[i].isValid:
|
||||
? db.collectStoTreeLazily(
|
||||
(rvid.root, vtx.bVid[i]), accPath,
|
||||
stoPath & vtx.ePfx & NibblesBuf.nibble(byte i),
|
||||
vids)
|
||||
|
||||
of Leaf:
|
||||
let stoPath = Hash256(data: (stoPath & vtx.lPfx).getBytes())
|
||||
db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), nil)
|
||||
|
||||
# There is no useful approach avoiding to walk the whole tree for updating
|
||||
# the storage data access cache.
|
||||
#
|
||||
# The alternative of stopping here and clearing the whole cache did degrade
|
||||
# performance significantly in some tests on mainnet when importing `era1`.
|
||||
#
|
||||
# When not clearing the cache it was seen
|
||||
# * filled up to maximum size most of the time
|
||||
# * at the same time having no `stoPath` hit at all (so there was nothing
|
||||
# to be cleared.)
|
||||
#
|
||||
if vids.top <= vids.data.len:
|
||||
if vids.top < vids.data.len:
|
||||
vids.data[vids.top] = rvid.vid
|
||||
vids.top.inc # Max value of `.top`: `vid.data.len + 1`
|
||||
|
||||
ok()
|
||||
|
||||
|
||||
proc disposeOfSubTree(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
rvid: RootedVertexID; # Root vertex
|
||||
vids: var VidCollect; # Accumulated vertex IDs for disposal
|
||||
) =
|
||||
## Evaluate results from `collectSubTreeLazyImpl()` or ftom
|
||||
## `collectStoTreeLazyImpl)`.
|
||||
##
|
||||
if vids.top <= typeof(vids).capa:
|
||||
# Delete small tree
|
||||
for n in 0 ..< vids.top:
|
||||
db.disposeOfVtx((rvid.root, vids.data[n]))
|
||||
|
||||
else:
|
||||
# Mark the sub-trees disabled to be deleted not until the layer is
|
||||
# finally stored onto the backend.
|
||||
let vtx = db.getVtxRc(rvid).value[0]
|
||||
for n in 0..15:
|
||||
if vtx.bVid[n].isValid:
|
||||
db.top.delTree.add (rvid.root,vtx.bVid[n])
|
||||
|
||||
# Delete top of tree now.
|
||||
db.disposeOfVtx(rvid)
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public functions
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc delSubTreeImpl*(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
root: VertexID; # Root vertex
|
||||
): Result[void,AristoError] =
|
||||
## Delete all the `subRoots`if there are a few, only. Otherwise
|
||||
## mark it for deleting later.
|
||||
discard db.getVtxRc((root,root)).valueOr:
|
||||
if error == GetVtxNotFound:
|
||||
return ok()
|
||||
return err(error)
|
||||
|
||||
when extraDebuggingMessages:
|
||||
let
|
||||
ana = db.analyseSubTree((root,root), VidCollect.capa+1, stats)
|
||||
start = getTime()
|
||||
|
||||
var dispose: VidCollect
|
||||
? db.collectSubTreeLazily((root,root), dispose)
|
||||
|
||||
db.disposeOfSubTree((root,root), dispose)
|
||||
|
||||
when extraDebuggingMessages:
|
||||
if typeof(dispose).capa < dispose.top:
|
||||
|
||||
if minVtxsForLogging < ana.nVtxs:
|
||||
debugLog("Generic sub-tree analysis",
|
||||
nVtxs = ana.nVtxs,
|
||||
nLeafs = ana.nLeafs,
|
||||
depthMax = ana.depthMax,
|
||||
nDelTree = db.top.delTree.len,
|
||||
elaCollect = getTime() - start)
|
||||
|
||||
if (stats.count mod allStatsFrequency) == 0:
|
||||
let
|
||||
start = getTime()
|
||||
(count, vtxs, leafs, depth, elapsed) = stats.strStats
|
||||
debugLog("Sub-tree analysis stats", count, vtxs, leafs, depth, elapsed)
|
||||
stats.sElapsed += getTime() - start
|
||||
ok()
|
||||
|
||||
|
||||
proc delStoTreeImpl*(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
rvid: RootedVertexID; # Root vertex
|
||||
accPath: Hash256;
|
||||
): Result[void,AristoError] =
|
||||
## Collect vertex/vid and cache entry.
|
||||
discard db.getVtxRc(rvid).valueOr:
|
||||
if error == GetVtxNotFound:
|
||||
return ok()
|
||||
return err(error)
|
||||
|
||||
when extraDebuggingMessages:
|
||||
let
|
||||
ana = db.analyseSubTree(rvid, VidCollect.capa+1, stats)
|
||||
start = getTime()
|
||||
|
||||
var dispose: VidCollect # Accumulating vertices for deletion
|
||||
? db.collectStoTreeLazily(rvid, accPath, NibblesBuf(), dispose)
|
||||
|
||||
db.disposeOfSubTree(rvid, dispose)
|
||||
|
||||
when extraDebuggingMessages:
|
||||
if typeof(dispose).capa < dispose.top:
|
||||
|
||||
if minVtxsForLogging < ana.nVtxs or db.stoLeaves.len < ana.nStoCache:
|
||||
debugLog("Storage sub-tree analysis",
|
||||
nVtxs = ana.nVtxs,
|
||||
nLeafs = ana.nLeafs,
|
||||
depthMax = ana.depthMax,
|
||||
nStoCache = ana.nStoCache,
|
||||
nStoCacheDelta = ana.nStoCache - db.stoLeaves.len,
|
||||
nDelTree = db.top.delTree.len,
|
||||
elaCollect = getTime() - start)
|
||||
|
||||
if (stats.count mod allStatsFrequency) == 0:
|
||||
let
|
||||
start = getTime()
|
||||
(count, vtxs, leafs, depth, elapsed) = stats.strStats
|
||||
debugLog("Sub-tree analysis stats", count, vtxs, leafs, depth, elapsed)
|
||||
stats.sElapsed += getTime() - start
|
||||
ok()
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# End
|
||||
# ------------------------------------------------------------------------------
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
# nimbus-eth1
|
||||
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import
|
||||
eth/common,
|
||||
".."/[aristo_desc, aristo_get, aristo_layers],
|
||||
./delete_helpers
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Private heplers
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc delSubTreeNow(
|
||||
db: AristoDbRef;
|
||||
rvid: RootedVertexID;
|
||||
): Result[void,AristoError] =
|
||||
## Delete sub-tree now
|
||||
let (vtx, _) = db.getVtxRc(rvid).valueOr:
|
||||
if error == GetVtxNotFound:
|
||||
return ok()
|
||||
return err(error)
|
||||
|
||||
if vtx.vType == Branch:
|
||||
for n in 0..15:
|
||||
if vtx.bVid[n].isValid:
|
||||
? db.delSubTreeNow((rvid.root,vtx.bVid[n]))
|
||||
|
||||
db.disposeOfVtx(rvid)
|
||||
|
||||
ok()
|
||||
|
||||
|
||||
proc delStoTreeNow(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
rvid: RootedVertexID; # Root vertex
|
||||
accPath: Hash256; # Accounts cache designator
|
||||
stoPath: NibblesBuf; # Current storage path
|
||||
): Result[void,AristoError] =
|
||||
## Implementation of *delete* sub-trie.
|
||||
|
||||
let (vtx, _) = db.getVtxRc(rvid).valueOr:
|
||||
if error == GetVtxNotFound:
|
||||
return ok()
|
||||
return err(error)
|
||||
|
||||
case vtx.vType
|
||||
of Branch:
|
||||
for i in 0..15:
|
||||
if vtx.bVid[i].isValid:
|
||||
? db.delStoTreeNow(
|
||||
(rvid.root, vtx.bVid[i]), accPath,
|
||||
stoPath & vtx.ePfx & NibblesBuf.nibble(byte i))
|
||||
|
||||
of Leaf:
|
||||
let stoPath = Hash256(data: (stoPath & vtx.lPfx).getBytes())
|
||||
db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), nil)
|
||||
|
||||
db.disposeOfVtx(rvid)
|
||||
|
||||
ok()
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public functions
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc delSubTreeImpl*(
|
||||
db: AristoDbRef;
|
||||
root: VertexID;
|
||||
): Result[void,AristoError] =
|
||||
discard db.getVtxRc((root, root)).valueOr:
|
||||
if error == GetVtxNotFound:
|
||||
return ok()
|
||||
return err(error)
|
||||
db.delSubTreeNow (root,root)
|
||||
|
||||
|
||||
proc delStoTreeImpl*(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
rvid: RootedVertexID; # Root vertex
|
||||
accPath: Hash256;
|
||||
): Result[void,AristoError] =
|
||||
## Implementation of *delete* sub-trie.
|
||||
discard db.getVtxRc(rvid).valueOr:
|
||||
if error == GetVtxNotFound:
|
||||
return ok()
|
||||
return err(error)
|
||||
db.delStoTreeNow(rvid, accPath, NibblesBuf())
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# End
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
|
|
@ -14,11 +14,32 @@
|
|||
|
||||
import
|
||||
std/tables,
|
||||
chronicles,
|
||||
eth/common,
|
||||
results,
|
||||
"."/[aristo_desc, aristo_layers],
|
||||
./aristo_delta/[delta_merge, delta_reverse],
|
||||
./aristo_desc/desc_backend,
|
||||
./aristo_delta/[delta_merge, delta_reverse]
|
||||
"."/[aristo_desc, aristo_get, aristo_layers, aristo_utils]
|
||||
|
||||
logScope:
|
||||
topics = "aristo-delta"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Private functions
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc delSubTree(db: AristoDbRef; writer: PutHdlRef; rvid: RootedVertexID) =
|
||||
## Collect subtrees marked for deletion
|
||||
let (vtx,_) = db.getVtxRc(rvid).valueOr:
|
||||
notice "Descending for deletion stopped", rvid, error
|
||||
return
|
||||
for vid in vtx.subVids:
|
||||
db.delSubTree(writer, (rvid.root, vid))
|
||||
db.backend.putVtxFn(writer, rvid, VertexRef(nil))
|
||||
db.backend.putKeyFn(writer, rvid, VOID_HASH_KEY)
|
||||
# Make sure the `rvid` is not mentioned here, anymore for furter update.
|
||||
db.balancer.sTab.del rvid
|
||||
db.balancer.kMap.del rvid
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public functions, save to backend
|
||||
|
|
@ -85,6 +106,12 @@ proc deltaPersistent*(
|
|||
|
||||
# Store structural single trie entries
|
||||
let writeBatch = ? be.putBegFn()
|
||||
# This one must come first in order to avoid duplicate `sTree[]` or
|
||||
# `kMap[]` instructions, in the worst case overwiting previously deleted
|
||||
# entries.
|
||||
for rvid in db.balancer.delTree:
|
||||
db.delSubTree(writeBatch, rvid)
|
||||
# Now the standard `sTree[]` and `kMap[]` instructions.
|
||||
for rvid, vtx in db.balancer.sTab:
|
||||
be.putVtxFn(writeBatch, rvid, vtx)
|
||||
for rvid, key in db.balancer.kMap:
|
||||
|
|
@ -97,12 +124,12 @@ proc deltaPersistent*(
|
|||
for accPath, vtx in db.balancer.accLeaves:
|
||||
let accKey = accPath.to(AccountKey)
|
||||
if not db.accLeaves.lruUpdate(accKey, vtx):
|
||||
discard db.accLeaves.lruAppend(accKey, vtx, accLruSize)
|
||||
discard db.accLeaves.lruAppend(accKey, vtx, ACC_LRU_SIZE)
|
||||
|
||||
for mixPath, vtx in db.balancer.stoLeaves:
|
||||
let mixKey = mixPath.to(AccountKey)
|
||||
if not db.stoLeaves.lruUpdate(mixKey, vtx):
|
||||
discard db.stoLeaves.lruAppend(mixKey, vtx, accLruSize)
|
||||
discard db.stoLeaves.lruAppend(mixKey, vtx, ACC_LRU_SIZE)
|
||||
|
||||
# Done with balancer, all saved to backend
|
||||
db.balancer = LayerRef(nil)
|
||||
|
|
|
|||
|
|
@ -48,13 +48,14 @@ proc deltaMerge*(
|
|||
result = LayerRef(
|
||||
sTab: lower.sTab, # shallow copy (entries will not be modified)
|
||||
kMap: lower.kMap,
|
||||
delTree: lower.delTree,
|
||||
accLeaves: lower.accLeaves,
|
||||
stoLeaves: lower.stoLeaves,
|
||||
vTop: upper.vTop)
|
||||
layersMergeOnto(upper, result[])
|
||||
|
||||
else:
|
||||
# Otherwise avoid copying some tables by modifyinh `upper`. This is not
|
||||
# Otherwise avoid copying some tables by modifying `upper`. This is not
|
||||
# completely free as the merge direction changes to merging the `lower`
|
||||
# layer up into the higher prioritised `upper` layer (note that the `lower`
|
||||
# argument filter is read-only.) Here again, the `upper` argument must not
|
||||
|
|
@ -67,6 +68,9 @@ proc deltaMerge*(
|
|||
if not upper.kMap.hasKey(rvid):
|
||||
upper.kMap[rvid] = key
|
||||
|
||||
for rvid in lower.delTree:
|
||||
upper.delTree.add rvid
|
||||
|
||||
for (accPath,leafVtx) in lower.accLeaves.pairs:
|
||||
if not upper.accLeaves.hasKey(accPath):
|
||||
upper.accLeaves[accPath] = leafVtx
|
||||
|
|
|
|||
|
|
@ -12,7 +12,46 @@ import
|
|||
std/tables,
|
||||
eth/common,
|
||||
results,
|
||||
".."/[aristo_desc, aristo_get]
|
||||
".."/[aristo_desc, aristo_get, aristo_utils]
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Private functions
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc revSubTree(
|
||||
db: AristoDbRef;
|
||||
rev: LayerRef;
|
||||
rvid: RootedVertexID;
|
||||
): Result[void,(VertexID,AristoError)] =
|
||||
## Collect subtrees marked for deletion
|
||||
let
|
||||
vtx = block:
|
||||
let rc = db.getVtxUbe rvid
|
||||
if rc.isOk:
|
||||
rc.value
|
||||
elif rc.error == GetVtxNotFound:
|
||||
VertexRef(nil)
|
||||
else:
|
||||
return err((rvid.vid,rc.error))
|
||||
|
||||
key = block:
|
||||
let rc = db.getKeyUbe rvid
|
||||
if rc.isOk:
|
||||
rc.value
|
||||
elif rc.error == GetKeyNotFound:
|
||||
VOID_HASH_KEY
|
||||
else:
|
||||
return err((rvid.vid,rc.error))
|
||||
|
||||
if vtx.isValid:
|
||||
for vid in vtx.subVids:
|
||||
? db.revSubTree(rev, (rvid.root,vid))
|
||||
rev.sTab[rvid] = vtx
|
||||
|
||||
if key.isValid:
|
||||
rev.kMap[rvid] = key
|
||||
|
||||
ok()
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public functions
|
||||
|
|
@ -48,7 +87,7 @@ proc revFilter*(
|
|||
else:
|
||||
return err((rvid.vid,rc.error))
|
||||
|
||||
# Calculate reverse changes for the `kMap` sequence.
|
||||
# Calculate reverse changes for the `kMap[]` structural table.
|
||||
for rvid in filter.kMap.keys:
|
||||
let rc = db.getKeyUbe rvid
|
||||
if rc.isOk:
|
||||
|
|
@ -58,6 +97,10 @@ proc revFilter*(
|
|||
else:
|
||||
return err((rvid.vid,rc.error))
|
||||
|
||||
# Reverse changes for `delTree[]` list.
|
||||
for rvid in filter.delTree:
|
||||
? db.revSubTree(rev, rvid)
|
||||
|
||||
ok(rev)
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -37,10 +37,6 @@ export
|
|||
tables, aristo_constants, desc_error, desc_identifiers, desc_nibbles,
|
||||
desc_structural, keyed_queue
|
||||
|
||||
const
|
||||
accLruSize* = 1024 * 1024
|
||||
# LRU cache size for accounts that have storage
|
||||
|
||||
type
|
||||
AristoTxRef* = ref object
|
||||
## Transaction descriptor
|
||||
|
|
|
|||
|
|
@ -124,6 +124,8 @@ type
|
|||
kMap*: Table[RootedVertexID,HashKey] ## Merkle hash key mapping
|
||||
vTop*: VertexID ## Last used vertex ID
|
||||
|
||||
delTree*: seq[RootedVertexID] ## Not yet fully deleted sub-trees
|
||||
|
||||
accLeaves*: Table[Hash256, VertexRef] ## Account path -> VertexRef
|
||||
stoLeaves*: Table[Hash256, VertexRef] ## Storage path -> VertexRef
|
||||
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ proc retrieveAccountPayload(
|
|||
return err(FetchPathNotFound)
|
||||
return err(error)
|
||||
|
||||
ok db.accLeaves.lruAppend(accKey, leafVtx, accLruSize).lData
|
||||
ok db.accLeaves.lruAppend(accKey, leafVtx, ACC_LRU_SIZE).lData
|
||||
|
||||
proc retrieveMerkleHash(
|
||||
db: AristoDbRef;
|
||||
|
|
@ -188,7 +188,7 @@ proc retrieveStoragePayload(
|
|||
leafVtx = db.retrieveLeaf(? db.fetchStorageID(accPath), stoPath.data).valueOr:
|
||||
return err(error)
|
||||
|
||||
ok db.stoLeaves.lruAppend(mixKey, leafVtx, accLruSize).lData.stoData
|
||||
ok db.stoLeaves.lruAppend(mixKey, leafVtx, ACC_LRU_SIZE).lData.stoData
|
||||
|
||||
proc hasStoragePayload(
|
||||
db: AristoDbRef;
|
||||
|
|
|
|||
|
|
@ -171,6 +171,7 @@ func isEmpty*(ly: LayerRef): bool =
|
|||
## tables are empty. The field `txUid` is ignored, here.
|
||||
ly.sTab.len == 0 and
|
||||
ly.kMap.len == 0 and
|
||||
ly.delTree.len == 0 and
|
||||
ly.accLeaves.len == 0 and
|
||||
ly.stoLeaves.len == 0
|
||||
|
||||
|
|
@ -186,6 +187,8 @@ func layersMergeOnto*(src: LayerRef; trg: var LayerObj) =
|
|||
for (vid,key) in src.kMap.pairs:
|
||||
trg.kMap[vid] = key
|
||||
trg.vTop = src.vTop
|
||||
for rvid in src.delTree:
|
||||
trg.delTree.add rvid
|
||||
for (accPath,leafVtx) in src.accLeaves.pairs:
|
||||
trg.accLeaves[accPath] = leafVtx
|
||||
for (mixPath,leafVtx) in src.stoLeaves.pairs:
|
||||
|
|
@ -204,6 +207,7 @@ func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
|
|||
sTab: layers[0].sTab.dup, # explicit dup for ref values
|
||||
kMap: layers[0].kMap,
|
||||
vTop: layers[^1].vTop,
|
||||
delTree: layers[0].delTree,
|
||||
accLeaves: layers[0].accLeaves,
|
||||
stoLeaves: layers[0].stoLeaves)
|
||||
|
||||
|
|
@ -213,6 +217,8 @@ func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
|
|||
result.sTab[vid] = vtx
|
||||
for (vid,key) in layers[n].kMap.pairs:
|
||||
result.kMap[vid] = key
|
||||
for rvid in layers[n].delTree:
|
||||
result.delTree.add rvid
|
||||
for (accPath,vtx) in layers[n].accLeaves.pairs:
|
||||
result.accLeaves[accPath] = vtx
|
||||
for (mixPath,vtx) in layers[n].stoLeaves.pairs:
|
||||
|
|
|
|||
|
|
@ -96,9 +96,9 @@ proc mergeGenericData*(
|
|||
|
||||
proc mergeStorageData*(
|
||||
db: AristoDbRef; # Database, top layer
|
||||
accPath: Hash256; # Needed for accounts payload
|
||||
stoPath: Hash256; # Storage data path (aka key)
|
||||
stoData: UInt256; # Storage data payload value
|
||||
accPath: Hash256; # Needed for accounts payload
|
||||
stoPath: Hash256; # Storage data path (aka key)
|
||||
stoData: UInt256; # Storage data payload value
|
||||
): Result[void,AristoError] =
|
||||
## Store the `stoData` data argument on the storage area addressed by
|
||||
## `(accPath,stoPath)` where `accPath` is the account key (into the MPT)
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import
|
|||
stew/byteutils,
|
||||
nimcrypto,
|
||||
results,
|
||||
../db/aristo,
|
||||
../db/aristo/aristo_sign,
|
||||
../constants
|
||||
|
||||
export eth_types_rlp
|
||||
|
|
|
|||
|
|
@ -124,21 +124,21 @@ let
|
|||
mainTest6r* = mainSampleEx
|
||||
.cloneWith(
|
||||
name = "-ex-ar-some",
|
||||
numBlocks = 257_400,
|
||||
numBlocks = 1_000_000,
|
||||
dbType = AristoDbRocks,
|
||||
dbName = "main-open") # for resuming on the same persistent DB
|
||||
|
||||
mainTest7r* = mainSampleEx
|
||||
.cloneWith(
|
||||
name = "-ex-ar-more",
|
||||
numBlocks = 1_460_700, # failure at 1,460,736
|
||||
numBlocks = 1_500_000,
|
||||
dbType = AristoDbRocks,
|
||||
dbName = "main-open") # for resuming on the same persistent DB
|
||||
|
||||
mainTest8r* = mainSampleEx
|
||||
.cloneWith(
|
||||
name = "-ex-ar-more2",
|
||||
numBlocks = 1_460_735, # failure at 1,460,736
|
||||
numBlocks = 2_000_000,
|
||||
dbType = AristoDbRocks,
|
||||
dbName = "main-open") # for resuming on the same persistent DB
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue