Aristo lazily delete larger subtrees (#2560)
* Extract sub-tree deletion functions into separate sub-modules * Move/rename `aristo_desc.accLruSize` => `aristo_constants.ACC_LRU_SIZE` * Lazily delete sub-trees why: This gives some control of the memory used to keep the deleted vertices in the cached layers. For larger sub-trees, keys and vertices might be on the persistent backend to a large extend. This would pull an amount of extra information from the backend into the cached layer. For lazy deleting it is enough to remember sub-trees by a small set of (at most 16) sub-roots to be processed when storing persistent data. Marking the tree root deleted immediately allows to let most of the code base work as before. * Comments and cosmetics * No need to import all for `Aristo` here * Kludge to make `chronicle` usage in sub-modules work with `fluffy` why: That `fluffy` would not run with any logging in `core_deb` is a problem I have known for a while. Up to now, logging was only used for debugging. With the current `Aristo` PR, there are cases where logging might be wanted but this works only if `chronicles` runs without the `json[dynamic]` sinks. So this should be re-visited. * More of a kludge
This commit is contained in:
parent
e3908a7b0d
commit
ce713d95fc
|
@ -1,4 +1,11 @@
|
||||||
-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
|
#
|
||||||
|
# For some reason `json[dynamic]` causes problems with subsequent modules from
|
||||||
|
# `Aristo` when compiling `fluffy`. There might be a `chronicles` inport missing
|
||||||
|
# but it is not obvious where. -- jordan
|
||||||
|
#
|
||||||
|
#-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
|
||||||
|
|
||||||
|
-d:"chronicles_sinks=textlines[dynamic]"
|
||||||
-d:"chronicles_runtime_filtering=on"
|
-d:"chronicles_runtime_filtering=on"
|
||||||
-d:"chronicles_disable_thread_id"
|
-d:"chronicles_disable_thread_id"
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,6 @@
|
||||||
# Use only `secp256k1` public key cryptography as an identity in LibP2P.
|
# Use only `secp256k1` public key cryptography as an identity in LibP2P.
|
||||||
-d:"libp2p_pki_schemes=secp256k1"
|
-d:"libp2p_pki_schemes=secp256k1"
|
||||||
|
|
||||||
-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
|
# See `fluffy.nim.cfg`
|
||||||
|
#-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
|
||||||
|
-d:"chronicles_sinks=textlines[dynamic]"
|
||||||
|
|
|
@ -1 +1,3 @@
|
||||||
-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
|
# See `fluffy.nim.cfg`
|
||||||
|
#-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"
|
||||||
|
-d:"chronicles_sinks=textlines[dynamic]"
|
||||||
|
|
|
@ -1,9 +1,4 @@
|
||||||
* Re-visit `delTree()`. Suggestion is deleting small trees on the memory later,
|
* Some comletions might be needed for the `aristo_part` module which is a
|
||||||
otherwise only deleting the root vertex (so it becomes inaccessible) and
|
|
||||||
remember the follow up vertices which can travel through the tx-layers
|
|
||||||
to be picked up by the backend store.
|
|
||||||
|
|
||||||
* Some comletions migh be needed for the `aristo_part` module which is a
|
|
||||||
re-implementation of the module supporting *proof-mode*/partial trees.
|
re-implementation of the module supporting *proof-mode*/partial trees.
|
||||||
+ Complete `partMergeStorageData()`. This function might not be needed at
|
+ Complete `partMergeStorageData()`. This function might not be needed at
|
||||||
all unless *snap-sync* is really revived.
|
all unless *snap-sync* is really revived.
|
||||||
|
|
|
@ -38,6 +38,19 @@ const
|
||||||
## functions with fixed assignments of the type of a state root (e.g. for
|
## functions with fixed assignments of the type of a state root (e.g. for
|
||||||
## a receipt or a transaction root.)
|
## a receipt or a transaction root.)
|
||||||
|
|
||||||
|
ACC_LRU_SIZE* = 1024 * 1024
|
||||||
|
## LRU cache size for accounts that have storage, see `.accLeaves` and
|
||||||
|
## `.stoLeaves` fields of the main descriptor.
|
||||||
|
|
||||||
|
DELETE_SUBTREE_VERTICES_MAX* = 25
|
||||||
|
## Maximum number of vertices for a tree to be deleted instantly. If the
|
||||||
|
## tree is larger, only the sub-tree root will be deleted immediately and
|
||||||
|
## subsequent entries will be deleted not until the cache layers are saved
|
||||||
|
## to the backend.
|
||||||
|
##
|
||||||
|
## Set to zero to disable in which case all sub-trees are deleted
|
||||||
|
## immediately.
|
||||||
|
|
||||||
static:
|
static:
|
||||||
# must stay away from `VertexID(1)` and `VertexID(2)`
|
# must stay away from `VertexID(1)` and `VertexID(2)`
|
||||||
doAssert 2 < LEAST_FREE_VID
|
doAssert 2 < LEAST_FREE_VID
|
||||||
|
|
|
@ -195,7 +195,7 @@ proc ppPayload(p: LeafPayload, db: AristoDbRef): string =
|
||||||
of AccountData:
|
of AccountData:
|
||||||
result = "(" & p.account.ppAriAccount() & "," & p.stoID.ppVid & ")"
|
result = "(" & p.account.ppAriAccount() & "," & p.stoID.ppVid & ")"
|
||||||
of StoData:
|
of StoData:
|
||||||
result = $p.stoData
|
result = ($p.stoData).squeeze
|
||||||
|
|
||||||
proc ppVtx(nd: VertexRef, db: AristoDbRef, rvid: RootedVertexID): string =
|
proc ppVtx(nd: VertexRef, db: AristoDbRef, rvid: RootedVertexID): string =
|
||||||
if not nd.isValid:
|
if not nd.isValid:
|
||||||
|
|
|
@ -11,7 +11,6 @@
|
||||||
## Aristo DB -- Patricia Trie delete funcionality
|
## Aristo DB -- Patricia Trie delete funcionality
|
||||||
## ==============================================
|
## ==============================================
|
||||||
##
|
##
|
||||||
## Delete by `Hike` type chain of vertices.
|
|
||||||
|
|
||||||
{.push raises: [].}
|
{.push raises: [].}
|
||||||
|
|
||||||
|
@ -19,6 +18,7 @@ import
|
||||||
std/typetraits,
|
std/typetraits,
|
||||||
eth/common,
|
eth/common,
|
||||||
results,
|
results,
|
||||||
|
./aristo_delete/[delete_helpers, delete_subtree],
|
||||||
"."/[aristo_desc, aristo_fetch, aristo_get, aristo_hike, aristo_layers,
|
"."/[aristo_desc, aristo_fetch, aristo_get, aristo_hike, aristo_layers,
|
||||||
aristo_utils]
|
aristo_utils]
|
||||||
|
|
||||||
|
@ -39,79 +39,10 @@ proc branchStillNeeded(vtx: VertexRef): Result[int,void] =
|
||||||
# Oops, degenerated branch node
|
# Oops, degenerated branch node
|
||||||
err()
|
err()
|
||||||
|
|
||||||
# -----------
|
|
||||||
|
|
||||||
proc disposeOfVtx(
|
|
||||||
db: AristoDbRef; # Database, top layer
|
|
||||||
rvid: RootedVertexID; # Vertex ID to clear
|
|
||||||
) =
|
|
||||||
# Remove entry
|
|
||||||
db.layersResVtx(rvid)
|
|
||||||
db.layersResKey(rvid)
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Private functions
|
# Private functions
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
proc delSubTreeImpl(
|
|
||||||
db: AristoDbRef; # Database, top layer
|
|
||||||
root: VertexID; # Root vertex
|
|
||||||
): Result[void,AristoError] =
|
|
||||||
## Implementation of *delete* sub-trie.
|
|
||||||
var
|
|
||||||
dispose = @[root]
|
|
||||||
(rootVtx, _) = db.getVtxRc((root, root)).valueOr:
|
|
||||||
if error == GetVtxNotFound:
|
|
||||||
return ok()
|
|
||||||
return err(error)
|
|
||||||
follow = @[rootVtx]
|
|
||||||
|
|
||||||
# Collect list of nodes to delete
|
|
||||||
while 0 < follow.len:
|
|
||||||
var redo: seq[VertexRef]
|
|
||||||
for vtx in follow:
|
|
||||||
for vid in vtx.subVids:
|
|
||||||
# Exiting here leaves the tree as-is
|
|
||||||
let vtx = (? db.getVtxRc((root, vid)))[0]
|
|
||||||
redo.add vtx
|
|
||||||
dispose.add vid
|
|
||||||
redo.swap follow
|
|
||||||
|
|
||||||
# Mark collected vertices to be deleted
|
|
||||||
for vid in dispose:
|
|
||||||
db.disposeOfVtx((root, vid))
|
|
||||||
|
|
||||||
ok()
|
|
||||||
|
|
||||||
proc delStoTreeImpl(
|
|
||||||
db: AristoDbRef; # Database, top layer
|
|
||||||
rvid: RootedVertexID; # Root vertex
|
|
||||||
accPath: Hash256;
|
|
||||||
stoPath: NibblesBuf;
|
|
||||||
): Result[void,AristoError] =
|
|
||||||
## Implementation of *delete* sub-trie.
|
|
||||||
|
|
||||||
let (vtx, _) = db.getVtxRc(rvid).valueOr:
|
|
||||||
if error == GetVtxNotFound:
|
|
||||||
return ok()
|
|
||||||
return err(error)
|
|
||||||
|
|
||||||
case vtx.vType
|
|
||||||
of Branch:
|
|
||||||
for i in 0..15:
|
|
||||||
if vtx.bVid[i].isValid:
|
|
||||||
? db.delStoTreeImpl(
|
|
||||||
(rvid.root, vtx.bVid[i]), accPath,
|
|
||||||
stoPath & vtx.ePfx & NibblesBuf.nibble(byte i))
|
|
||||||
|
|
||||||
of Leaf:
|
|
||||||
let stoPath = Hash256(data: (stoPath & vtx.lPfx).getBytes())
|
|
||||||
db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), nil)
|
|
||||||
|
|
||||||
db.disposeOfVtx(rvid)
|
|
||||||
|
|
||||||
ok()
|
|
||||||
|
|
||||||
proc deleteImpl(
|
proc deleteImpl(
|
||||||
db: AristoDbRef; # Database, top layer
|
db: AristoDbRef; # Database, top layer
|
||||||
hike: Hike; # Fully expanded path
|
hike: Hike; # Fully expanded path
|
||||||
|
@ -199,7 +130,7 @@ proc deleteAccountRecord*(
|
||||||
|
|
||||||
# Delete storage tree if present
|
# Delete storage tree if present
|
||||||
if stoID.isValid:
|
if stoID.isValid:
|
||||||
? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath, NibblesBuf())
|
? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath)
|
||||||
|
|
||||||
?db.deleteImpl(hike)
|
?db.deleteImpl(hike)
|
||||||
|
|
||||||
|
@ -322,7 +253,7 @@ proc deleteStorageTree*(
|
||||||
# Mark account path Merkle keys for update
|
# Mark account path Merkle keys for update
|
||||||
db.updateAccountForHasher accHike
|
db.updateAccountForHasher accHike
|
||||||
|
|
||||||
? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath, NibblesBuf())
|
? db.delStoTreeImpl((stoID.vid, stoID.vid), accPath)
|
||||||
|
|
||||||
# De-register the deleted storage tree from the accounts record
|
# De-register the deleted storage tree from the accounts record
|
||||||
let leaf = wpAcc.vtx.dup # Dup on modify
|
let leaf = wpAcc.vtx.dup # Dup on modify
|
||||||
|
|
|
@ -0,0 +1,127 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
std/[math, strformat, times],
|
||||||
|
chronicles,
|
||||||
|
".."/[aristo_desc, aristo_get, aristo_profile]
|
||||||
|
|
||||||
|
export
|
||||||
|
aristo_profile.toStr
|
||||||
|
|
||||||
|
type
|
||||||
|
SubTreeStats* = tuple
|
||||||
|
nVtxs: int ## Number of vertices in sub-tree
|
||||||
|
nLeafs: int ## Number of leafs in sub-tree
|
||||||
|
depthMax: int ## Maximal vertex path length
|
||||||
|
nStoCache: int ## Size of storage leafs cache
|
||||||
|
elapsed: Duration ## Time spent analysing
|
||||||
|
|
||||||
|
SubTreeStatsAccu* = tuple
|
||||||
|
count: int ## Number of entries
|
||||||
|
sVtxs, qVtxs: float ## Sum and square sum of `.nVtxs`
|
||||||
|
sLeafs, qLeafs: float ## Sum and square sum of `.nLeafs`
|
||||||
|
sDepth, qDepth: float ## Sum and square sum of `.depthMax`
|
||||||
|
sElapsed: Duration ## Sum of `.elapsed`
|
||||||
|
|
||||||
|
SubTreeDist* = tuple
|
||||||
|
count: int ## Number of entries
|
||||||
|
mVtxs, dVtxs: float ## Mean and std deviation of `.nVtxs`
|
||||||
|
mLeafs, dLeafs: float ## Mean and std deviation of `.nLeafs`
|
||||||
|
mDepth, dDepth: float ## Mean and std deviation of `.depthMax`
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Prival helper
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc analyseSubTreeImpl(
|
||||||
|
db: AristoDbRef; # Database, top layer
|
||||||
|
rvid: RootedVertexID; # Root vertex
|
||||||
|
depth: int; # Recursion depth
|
||||||
|
stats: var SubTreeStats; # Statistics
|
||||||
|
) =
|
||||||
|
let (vtx, _) = db.getVtxRc(rvid).valueOr:
|
||||||
|
return
|
||||||
|
|
||||||
|
stats.nVtxs.inc
|
||||||
|
|
||||||
|
if stats.depthMax < depth:
|
||||||
|
stats.depthMax = depth
|
||||||
|
|
||||||
|
case vtx.vType:
|
||||||
|
of Branch:
|
||||||
|
for n in 0..15:
|
||||||
|
if vtx.bVid[n].isValid:
|
||||||
|
db.analyseSubTreeImpl((rvid.root,vtx.bVid[n]), depth+1, stats)
|
||||||
|
of Leaf:
|
||||||
|
stats.nLeafs.inc
|
||||||
|
|
||||||
|
|
||||||
|
func evalDist(count: int; sum, sqSum: float): tuple[mean, stdDev: float] =
|
||||||
|
result.mean = sum / count.float
|
||||||
|
|
||||||
|
let
|
||||||
|
sqMean = sqSum / count.float
|
||||||
|
meanSq = result.mean * result.mean
|
||||||
|
|
||||||
|
# Mathematically, `meanSq <= sqMean` but there might be rounding errors
|
||||||
|
# if `meanSq` and `sqMean` are approximately the same.
|
||||||
|
sigma = sqMean - min(meanSq,sqMean)
|
||||||
|
|
||||||
|
result.stdDev = sigma.sqrt
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public analysis tools
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc analyseSubTree*(
|
||||||
|
db: AristoDbRef; # Database, top layer
|
||||||
|
rvid: RootedVertexID; # Root vertex
|
||||||
|
minVtxs: int; # Accumulate if `minVtxs` <= `.nVtxs`
|
||||||
|
accu: var SubTreeStatsAccu; # For accumulated statistics
|
||||||
|
): SubTreeStats =
|
||||||
|
let start = getTime()
|
||||||
|
db.analyseSubTreeImpl(rvid, 1, result)
|
||||||
|
result.nStoCache = db.stoLeaves.len
|
||||||
|
|
||||||
|
if minVtxs <= result.nVtxs:
|
||||||
|
accu.count.inc
|
||||||
|
accu.sVtxs += result.nVtxs.float
|
||||||
|
accu.qVtxs += (result.nVtxs * result.nVtxs).float
|
||||||
|
accu.sLeafs += result.nLeafs.float
|
||||||
|
accu.qLeafs += (result.nLeafs * result.nLeafs).float
|
||||||
|
accu.sDepth += result.depthMax.float
|
||||||
|
accu.qDepth += (result.depthMax * result.depthMax).float
|
||||||
|
|
||||||
|
result.elapsed = getTime() - start
|
||||||
|
accu.sElapsed += result.elapsed # Unconditionally collecrd
|
||||||
|
|
||||||
|
|
||||||
|
func stats*(a: SubTreeStatsAccu): SubTreeDist =
|
||||||
|
result.count = a.count
|
||||||
|
(result.mVtxs, result.dVtxs) = evalDist(a.count, a.sVtxs, a.qVtxs)
|
||||||
|
(result.mLeafs, result.dLeafs) = evalDist(a.count, a.sLeafs, a.qLeafs)
|
||||||
|
(result.mDepth, result.dDepth) = evalDist(a.count, a.sDepth, a.qDepth)
|
||||||
|
|
||||||
|
func strStats*(
|
||||||
|
a: SubTreeStatsAccu;
|
||||||
|
): tuple[count, vtxs, leafs, depth, elapsed: string] =
|
||||||
|
let w = a.stats()
|
||||||
|
result.count = $w.count
|
||||||
|
result.elapsed = a.sElapsed.toStr
|
||||||
|
result.vtxs = &"{w.mVtxs:.1f}[{w.dVtxs:.1f}]"
|
||||||
|
result.leafs = &"{w.mLeafs:.1f}[{w.dLeafs:.1f}]"
|
||||||
|
result.depth = &"{w.mDepth:.1f}[{w.dDepth:.1f}]"
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
|
@ -0,0 +1,25 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
".."/[aristo_desc, aristo_layers]
|
||||||
|
|
||||||
|
|
||||||
|
proc disposeOfVtx*(
|
||||||
|
db: AristoDbRef; # Database, top layer
|
||||||
|
rvid: RootedVertexID; # Vertex ID to clear
|
||||||
|
) =
|
||||||
|
# Remove entry
|
||||||
|
db.layersResVtx(rvid)
|
||||||
|
db.layersResKey(rvid)
|
||||||
|
|
||||||
|
# End
|
|
@ -0,0 +1,20 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
import ../aristo_constants
|
||||||
|
|
||||||
|
when DELETE_SUBTREE_VERTICES_MAX == 0:
|
||||||
|
import ./delete_subtree_now as del_sub
|
||||||
|
else:
|
||||||
|
import ./delete_subtree_lazy as del_sub
|
||||||
|
|
||||||
|
export del_sub
|
||||||
|
|
||||||
|
# End
|
|
@ -0,0 +1,252 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
std/sets,
|
||||||
|
eth/common,
|
||||||
|
results,
|
||||||
|
".."/[aristo_desc, aristo_get, aristo_layers],
|
||||||
|
./delete_helpers
|
||||||
|
|
||||||
|
const
|
||||||
|
extraDebuggingMessages = false # or true
|
||||||
|
## Enable additional logging noise. Note that this might slow down the
|
||||||
|
## system performance but will not be too significant. When importing the
|
||||||
|
## first 5m blocks from `era1` on some Debian system,
|
||||||
|
## * loading time was ~5h
|
||||||
|
## * overhead of accumulated analysis times was ~1.2s
|
||||||
|
|
||||||
|
type
|
||||||
|
VidCollect = tuple
|
||||||
|
data: array[DELETE_SUBTREE_VERTICES_MAX,VertexID]
|
||||||
|
top: int # Next free slot if smaller `.data.len`
|
||||||
|
|
||||||
|
|
||||||
|
when extraDebuggingMessages:
|
||||||
|
import
|
||||||
|
std/times,
|
||||||
|
chronicles,
|
||||||
|
./delete_debug
|
||||||
|
|
||||||
|
const
|
||||||
|
allStatsFrequency = 20
|
||||||
|
## Print accumutated statistics every `allStatsFrequency` visits of
|
||||||
|
## the analysis tool.
|
||||||
|
|
||||||
|
minVtxsForLogging = 1000
|
||||||
|
## Suppress detailed logging for smaller sub-trees
|
||||||
|
|
||||||
|
var stats: SubTreeStatsAccu
|
||||||
|
## Accumulated statistics
|
||||||
|
|
||||||
|
func `$`(ela: Duration): string =
|
||||||
|
ela.toStr
|
||||||
|
|
||||||
|
template debugLog(info: static[string]; args: varargs[untyped]) =
|
||||||
|
## Statistics message via `chronicles` logger, makes it easy to
|
||||||
|
## change priority and format.
|
||||||
|
notice info, args
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Private heplers
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func capa(T: type VidCollect): int =
|
||||||
|
## Syntactic sugar
|
||||||
|
T.default.data.len
|
||||||
|
|
||||||
|
|
||||||
|
proc collectSubTreeLazily(
|
||||||
|
db: AristoDbRef; # Database, top layer
|
||||||
|
rvid: RootedVertexID; # Root vertex
|
||||||
|
vids: var VidCollect; # Accumulating vertex IDs for deletion
|
||||||
|
): Result[void,AristoError] =
|
||||||
|
## Collect vids for a small sub-tree
|
||||||
|
let (vtx, _) = db.getVtxRc(rvid).valueOr:
|
||||||
|
if error == GetVtxNotFound:
|
||||||
|
return ok()
|
||||||
|
return err(error)
|
||||||
|
|
||||||
|
if vids.top < vids.data.len:
|
||||||
|
vids.data[vids.top] = rvid.vid
|
||||||
|
vids.top.inc # Max value of `.top`: `vid.data.len + 1`
|
||||||
|
|
||||||
|
if vtx.vType == Branch:
|
||||||
|
for n in 0..15:
|
||||||
|
if vtx.bVid[n].isValid:
|
||||||
|
? db.collectSubTreeLazily((rvid.root,vtx.bVid[n]), vids)
|
||||||
|
|
||||||
|
elif vids.top <= vids.data.len:
|
||||||
|
vids.top.inc # Terminates here
|
||||||
|
|
||||||
|
ok()
|
||||||
|
|
||||||
|
|
||||||
|
proc collectStoTreeLazily(
|
||||||
|
db: AristoDbRef; # Database, top layer
|
||||||
|
rvid: RootedVertexID; # Root vertex
|
||||||
|
accPath: Hash256; # Accounts cache designator
|
||||||
|
stoPath: NibblesBuf; # Current storage path
|
||||||
|
vids: var VidCollect; # Accumulating vertex IDs for deletion
|
||||||
|
): Result[void,AristoError] =
|
||||||
|
## Collect vertex/vid and delete cache entries.
|
||||||
|
let (vtx, _) = db.getVtxRc(rvid).valueOr:
|
||||||
|
if error == GetVtxNotFound:
|
||||||
|
return ok()
|
||||||
|
return err(error)
|
||||||
|
|
||||||
|
case vtx.vType
|
||||||
|
of Branch:
|
||||||
|
for i in 0..15:
|
||||||
|
if vtx.bVid[i].isValid:
|
||||||
|
? db.collectStoTreeLazily(
|
||||||
|
(rvid.root, vtx.bVid[i]), accPath,
|
||||||
|
stoPath & vtx.ePfx & NibblesBuf.nibble(byte i),
|
||||||
|
vids)
|
||||||
|
|
||||||
|
of Leaf:
|
||||||
|
let stoPath = Hash256(data: (stoPath & vtx.lPfx).getBytes())
|
||||||
|
db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), nil)
|
||||||
|
|
||||||
|
# There is no useful approach avoiding to walk the whole tree for updating
|
||||||
|
# the storage data access cache.
|
||||||
|
#
|
||||||
|
# The alternative of stopping here and clearing the whole cache did degrade
|
||||||
|
# performance significantly in some tests on mainnet when importing `era1`.
|
||||||
|
#
|
||||||
|
# When not clearing the cache it was seen
|
||||||
|
# * filled up to maximum size most of the time
|
||||||
|
# * at the same time having no `stoPath` hit at all (so there was nothing
|
||||||
|
# to be cleared.)
|
||||||
|
#
|
||||||
|
if vids.top <= vids.data.len:
|
||||||
|
if vids.top < vids.data.len:
|
||||||
|
vids.data[vids.top] = rvid.vid
|
||||||
|
vids.top.inc # Max value of `.top`: `vid.data.len + 1`
|
||||||
|
|
||||||
|
ok()
|
||||||
|
|
||||||
|
|
||||||
|
proc disposeOfSubTree(
|
||||||
|
db: AristoDbRef; # Database, top layer
|
||||||
|
rvid: RootedVertexID; # Root vertex
|
||||||
|
vids: var VidCollect; # Accumulated vertex IDs for disposal
|
||||||
|
) =
|
||||||
|
## Evaluate results from `collectSubTreeLazyImpl()` or ftom
|
||||||
|
## `collectStoTreeLazyImpl)`.
|
||||||
|
##
|
||||||
|
if vids.top <= typeof(vids).capa:
|
||||||
|
# Delete small tree
|
||||||
|
for n in 0 ..< vids.top:
|
||||||
|
db.disposeOfVtx((rvid.root, vids.data[n]))
|
||||||
|
|
||||||
|
else:
|
||||||
|
# Mark the sub-trees disabled to be deleted not until the layer is
|
||||||
|
# finally stored onto the backend.
|
||||||
|
let vtx = db.getVtxRc(rvid).value[0]
|
||||||
|
for n in 0..15:
|
||||||
|
if vtx.bVid[n].isValid:
|
||||||
|
db.top.delTree.add (rvid.root,vtx.bVid[n])
|
||||||
|
|
||||||
|
# Delete top of tree now.
|
||||||
|
db.disposeOfVtx(rvid)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public functions
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc delSubTreeImpl*(
|
||||||
|
db: AristoDbRef; # Database, top layer
|
||||||
|
root: VertexID; # Root vertex
|
||||||
|
): Result[void,AristoError] =
|
||||||
|
## Delete all the `subRoots`if there are a few, only. Otherwise
|
||||||
|
## mark it for deleting later.
|
||||||
|
discard db.getVtxRc((root,root)).valueOr:
|
||||||
|
if error == GetVtxNotFound:
|
||||||
|
return ok()
|
||||||
|
return err(error)
|
||||||
|
|
||||||
|
when extraDebuggingMessages:
|
||||||
|
let
|
||||||
|
ana = db.analyseSubTree((root,root), VidCollect.capa+1, stats)
|
||||||
|
start = getTime()
|
||||||
|
|
||||||
|
var dispose: VidCollect
|
||||||
|
? db.collectSubTreeLazily((root,root), dispose)
|
||||||
|
|
||||||
|
db.disposeOfSubTree((root,root), dispose)
|
||||||
|
|
||||||
|
when extraDebuggingMessages:
|
||||||
|
if typeof(dispose).capa < dispose.top:
|
||||||
|
|
||||||
|
if minVtxsForLogging < ana.nVtxs:
|
||||||
|
debugLog("Generic sub-tree analysis",
|
||||||
|
nVtxs = ana.nVtxs,
|
||||||
|
nLeafs = ana.nLeafs,
|
||||||
|
depthMax = ana.depthMax,
|
||||||
|
nDelTree = db.top.delTree.len,
|
||||||
|
elaCollect = getTime() - start)
|
||||||
|
|
||||||
|
if (stats.count mod allStatsFrequency) == 0:
|
||||||
|
let
|
||||||
|
start = getTime()
|
||||||
|
(count, vtxs, leafs, depth, elapsed) = stats.strStats
|
||||||
|
debugLog("Sub-tree analysis stats", count, vtxs, leafs, depth, elapsed)
|
||||||
|
stats.sElapsed += getTime() - start
|
||||||
|
ok()
|
||||||
|
|
||||||
|
|
||||||
|
proc delStoTreeImpl*(
|
||||||
|
db: AristoDbRef; # Database, top layer
|
||||||
|
rvid: RootedVertexID; # Root vertex
|
||||||
|
accPath: Hash256;
|
||||||
|
): Result[void,AristoError] =
|
||||||
|
## Collect vertex/vid and cache entry.
|
||||||
|
discard db.getVtxRc(rvid).valueOr:
|
||||||
|
if error == GetVtxNotFound:
|
||||||
|
return ok()
|
||||||
|
return err(error)
|
||||||
|
|
||||||
|
when extraDebuggingMessages:
|
||||||
|
let
|
||||||
|
ana = db.analyseSubTree(rvid, VidCollect.capa+1, stats)
|
||||||
|
start = getTime()
|
||||||
|
|
||||||
|
var dispose: VidCollect # Accumulating vertices for deletion
|
||||||
|
? db.collectStoTreeLazily(rvid, accPath, NibblesBuf(), dispose)
|
||||||
|
|
||||||
|
db.disposeOfSubTree(rvid, dispose)
|
||||||
|
|
||||||
|
when extraDebuggingMessages:
|
||||||
|
if typeof(dispose).capa < dispose.top:
|
||||||
|
|
||||||
|
if minVtxsForLogging < ana.nVtxs or db.stoLeaves.len < ana.nStoCache:
|
||||||
|
debugLog("Storage sub-tree analysis",
|
||||||
|
nVtxs = ana.nVtxs,
|
||||||
|
nLeafs = ana.nLeafs,
|
||||||
|
depthMax = ana.depthMax,
|
||||||
|
nStoCache = ana.nStoCache,
|
||||||
|
nStoCacheDelta = ana.nStoCache - db.stoLeaves.len,
|
||||||
|
nDelTree = db.top.delTree.len,
|
||||||
|
elaCollect = getTime() - start)
|
||||||
|
|
||||||
|
if (stats.count mod allStatsFrequency) == 0:
|
||||||
|
let
|
||||||
|
start = getTime()
|
||||||
|
(count, vtxs, leafs, depth, elapsed) = stats.strStats
|
||||||
|
debugLog("Sub-tree analysis stats", count, vtxs, leafs, depth, elapsed)
|
||||||
|
stats.sElapsed += getTime() - start
|
||||||
|
ok()
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
|
@ -0,0 +1,101 @@
|
||||||
|
# nimbus-eth1
|
||||||
|
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
||||||
|
# Licensed under either of
|
||||||
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||||
|
# http://opensource.org/licenses/MIT)
|
||||||
|
# at your option. This file may not be copied, modified, or distributed
|
||||||
|
# except according to those terms.
|
||||||
|
|
||||||
|
{.push raises: [].}
|
||||||
|
|
||||||
|
import
|
||||||
|
eth/common,
|
||||||
|
".."/[aristo_desc, aristo_get, aristo_layers],
|
||||||
|
./delete_helpers
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Private heplers
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc delSubTreeNow(
|
||||||
|
db: AristoDbRef;
|
||||||
|
rvid: RootedVertexID;
|
||||||
|
): Result[void,AristoError] =
|
||||||
|
## Delete sub-tree now
|
||||||
|
let (vtx, _) = db.getVtxRc(rvid).valueOr:
|
||||||
|
if error == GetVtxNotFound:
|
||||||
|
return ok()
|
||||||
|
return err(error)
|
||||||
|
|
||||||
|
if vtx.vType == Branch:
|
||||||
|
for n in 0..15:
|
||||||
|
if vtx.bVid[n].isValid:
|
||||||
|
? db.delSubTreeNow((rvid.root,vtx.bVid[n]))
|
||||||
|
|
||||||
|
db.disposeOfVtx(rvid)
|
||||||
|
|
||||||
|
ok()
|
||||||
|
|
||||||
|
|
||||||
|
proc delStoTreeNow(
|
||||||
|
db: AristoDbRef; # Database, top layer
|
||||||
|
rvid: RootedVertexID; # Root vertex
|
||||||
|
accPath: Hash256; # Accounts cache designator
|
||||||
|
stoPath: NibblesBuf; # Current storage path
|
||||||
|
): Result[void,AristoError] =
|
||||||
|
## Implementation of *delete* sub-trie.
|
||||||
|
|
||||||
|
let (vtx, _) = db.getVtxRc(rvid).valueOr:
|
||||||
|
if error == GetVtxNotFound:
|
||||||
|
return ok()
|
||||||
|
return err(error)
|
||||||
|
|
||||||
|
case vtx.vType
|
||||||
|
of Branch:
|
||||||
|
for i in 0..15:
|
||||||
|
if vtx.bVid[i].isValid:
|
||||||
|
? db.delStoTreeNow(
|
||||||
|
(rvid.root, vtx.bVid[i]), accPath,
|
||||||
|
stoPath & vtx.ePfx & NibblesBuf.nibble(byte i))
|
||||||
|
|
||||||
|
of Leaf:
|
||||||
|
let stoPath = Hash256(data: (stoPath & vtx.lPfx).getBytes())
|
||||||
|
db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), nil)
|
||||||
|
|
||||||
|
db.disposeOfVtx(rvid)
|
||||||
|
|
||||||
|
ok()
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Public functions
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc delSubTreeImpl*(
|
||||||
|
db: AristoDbRef;
|
||||||
|
root: VertexID;
|
||||||
|
): Result[void,AristoError] =
|
||||||
|
discard db.getVtxRc((root, root)).valueOr:
|
||||||
|
if error == GetVtxNotFound:
|
||||||
|
return ok()
|
||||||
|
return err(error)
|
||||||
|
db.delSubTreeNow (root,root)
|
||||||
|
|
||||||
|
|
||||||
|
proc delStoTreeImpl*(
|
||||||
|
db: AristoDbRef; # Database, top layer
|
||||||
|
rvid: RootedVertexID; # Root vertex
|
||||||
|
accPath: Hash256;
|
||||||
|
): Result[void,AristoError] =
|
||||||
|
## Implementation of *delete* sub-trie.
|
||||||
|
discard db.getVtxRc(rvid).valueOr:
|
||||||
|
if error == GetVtxNotFound:
|
||||||
|
return ok()
|
||||||
|
return err(error)
|
||||||
|
db.delStoTreeNow(rvid, accPath, NibblesBuf())
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# End
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
|
@ -14,11 +14,32 @@
|
||||||
|
|
||||||
import
|
import
|
||||||
std/tables,
|
std/tables,
|
||||||
|
chronicles,
|
||||||
eth/common,
|
eth/common,
|
||||||
results,
|
results,
|
||||||
"."/[aristo_desc, aristo_layers],
|
./aristo_delta/[delta_merge, delta_reverse],
|
||||||
./aristo_desc/desc_backend,
|
./aristo_desc/desc_backend,
|
||||||
./aristo_delta/[delta_merge, delta_reverse]
|
"."/[aristo_desc, aristo_get, aristo_layers, aristo_utils]
|
||||||
|
|
||||||
|
logScope:
|
||||||
|
topics = "aristo-delta"
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Private functions
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc delSubTree(db: AristoDbRef; writer: PutHdlRef; rvid: RootedVertexID) =
|
||||||
|
## Collect subtrees marked for deletion
|
||||||
|
let (vtx,_) = db.getVtxRc(rvid).valueOr:
|
||||||
|
notice "Descending for deletion stopped", rvid, error
|
||||||
|
return
|
||||||
|
for vid in vtx.subVids:
|
||||||
|
db.delSubTree(writer, (rvid.root, vid))
|
||||||
|
db.backend.putVtxFn(writer, rvid, VertexRef(nil))
|
||||||
|
db.backend.putKeyFn(writer, rvid, VOID_HASH_KEY)
|
||||||
|
# Make sure the `rvid` is not mentioned here, anymore for furter update.
|
||||||
|
db.balancer.sTab.del rvid
|
||||||
|
db.balancer.kMap.del rvid
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Public functions, save to backend
|
# Public functions, save to backend
|
||||||
|
@ -85,6 +106,12 @@ proc deltaPersistent*(
|
||||||
|
|
||||||
# Store structural single trie entries
|
# Store structural single trie entries
|
||||||
let writeBatch = ? be.putBegFn()
|
let writeBatch = ? be.putBegFn()
|
||||||
|
# This one must come first in order to avoid duplicate `sTree[]` or
|
||||||
|
# `kMap[]` instructions, in the worst case overwiting previously deleted
|
||||||
|
# entries.
|
||||||
|
for rvid in db.balancer.delTree:
|
||||||
|
db.delSubTree(writeBatch, rvid)
|
||||||
|
# Now the standard `sTree[]` and `kMap[]` instructions.
|
||||||
for rvid, vtx in db.balancer.sTab:
|
for rvid, vtx in db.balancer.sTab:
|
||||||
be.putVtxFn(writeBatch, rvid, vtx)
|
be.putVtxFn(writeBatch, rvid, vtx)
|
||||||
for rvid, key in db.balancer.kMap:
|
for rvid, key in db.balancer.kMap:
|
||||||
|
@ -97,12 +124,12 @@ proc deltaPersistent*(
|
||||||
for accPath, vtx in db.balancer.accLeaves:
|
for accPath, vtx in db.balancer.accLeaves:
|
||||||
let accKey = accPath.to(AccountKey)
|
let accKey = accPath.to(AccountKey)
|
||||||
if not db.accLeaves.lruUpdate(accKey, vtx):
|
if not db.accLeaves.lruUpdate(accKey, vtx):
|
||||||
discard db.accLeaves.lruAppend(accKey, vtx, accLruSize)
|
discard db.accLeaves.lruAppend(accKey, vtx, ACC_LRU_SIZE)
|
||||||
|
|
||||||
for mixPath, vtx in db.balancer.stoLeaves:
|
for mixPath, vtx in db.balancer.stoLeaves:
|
||||||
let mixKey = mixPath.to(AccountKey)
|
let mixKey = mixPath.to(AccountKey)
|
||||||
if not db.stoLeaves.lruUpdate(mixKey, vtx):
|
if not db.stoLeaves.lruUpdate(mixKey, vtx):
|
||||||
discard db.stoLeaves.lruAppend(mixKey, vtx, accLruSize)
|
discard db.stoLeaves.lruAppend(mixKey, vtx, ACC_LRU_SIZE)
|
||||||
|
|
||||||
# Done with balancer, all saved to backend
|
# Done with balancer, all saved to backend
|
||||||
db.balancer = LayerRef(nil)
|
db.balancer = LayerRef(nil)
|
||||||
|
|
|
@ -48,13 +48,14 @@ proc deltaMerge*(
|
||||||
result = LayerRef(
|
result = LayerRef(
|
||||||
sTab: lower.sTab, # shallow copy (entries will not be modified)
|
sTab: lower.sTab, # shallow copy (entries will not be modified)
|
||||||
kMap: lower.kMap,
|
kMap: lower.kMap,
|
||||||
|
delTree: lower.delTree,
|
||||||
accLeaves: lower.accLeaves,
|
accLeaves: lower.accLeaves,
|
||||||
stoLeaves: lower.stoLeaves,
|
stoLeaves: lower.stoLeaves,
|
||||||
vTop: upper.vTop)
|
vTop: upper.vTop)
|
||||||
layersMergeOnto(upper, result[])
|
layersMergeOnto(upper, result[])
|
||||||
|
|
||||||
else:
|
else:
|
||||||
# Otherwise avoid copying some tables by modifyinh `upper`. This is not
|
# Otherwise avoid copying some tables by modifying `upper`. This is not
|
||||||
# completely free as the merge direction changes to merging the `lower`
|
# completely free as the merge direction changes to merging the `lower`
|
||||||
# layer up into the higher prioritised `upper` layer (note that the `lower`
|
# layer up into the higher prioritised `upper` layer (note that the `lower`
|
||||||
# argument filter is read-only.) Here again, the `upper` argument must not
|
# argument filter is read-only.) Here again, the `upper` argument must not
|
||||||
|
@ -67,6 +68,9 @@ proc deltaMerge*(
|
||||||
if not upper.kMap.hasKey(rvid):
|
if not upper.kMap.hasKey(rvid):
|
||||||
upper.kMap[rvid] = key
|
upper.kMap[rvid] = key
|
||||||
|
|
||||||
|
for rvid in lower.delTree:
|
||||||
|
upper.delTree.add rvid
|
||||||
|
|
||||||
for (accPath,leafVtx) in lower.accLeaves.pairs:
|
for (accPath,leafVtx) in lower.accLeaves.pairs:
|
||||||
if not upper.accLeaves.hasKey(accPath):
|
if not upper.accLeaves.hasKey(accPath):
|
||||||
upper.accLeaves[accPath] = leafVtx
|
upper.accLeaves[accPath] = leafVtx
|
||||||
|
|
|
@ -12,7 +12,46 @@ import
|
||||||
std/tables,
|
std/tables,
|
||||||
eth/common,
|
eth/common,
|
||||||
results,
|
results,
|
||||||
".."/[aristo_desc, aristo_get]
|
".."/[aristo_desc, aristo_get, aristo_utils]
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Private functions
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
proc revSubTree(
|
||||||
|
db: AristoDbRef;
|
||||||
|
rev: LayerRef;
|
||||||
|
rvid: RootedVertexID;
|
||||||
|
): Result[void,(VertexID,AristoError)] =
|
||||||
|
## Collect subtrees marked for deletion
|
||||||
|
let
|
||||||
|
vtx = block:
|
||||||
|
let rc = db.getVtxUbe rvid
|
||||||
|
if rc.isOk:
|
||||||
|
rc.value
|
||||||
|
elif rc.error == GetVtxNotFound:
|
||||||
|
VertexRef(nil)
|
||||||
|
else:
|
||||||
|
return err((rvid.vid,rc.error))
|
||||||
|
|
||||||
|
key = block:
|
||||||
|
let rc = db.getKeyUbe rvid
|
||||||
|
if rc.isOk:
|
||||||
|
rc.value
|
||||||
|
elif rc.error == GetKeyNotFound:
|
||||||
|
VOID_HASH_KEY
|
||||||
|
else:
|
||||||
|
return err((rvid.vid,rc.error))
|
||||||
|
|
||||||
|
if vtx.isValid:
|
||||||
|
for vid in vtx.subVids:
|
||||||
|
? db.revSubTree(rev, (rvid.root,vid))
|
||||||
|
rev.sTab[rvid] = vtx
|
||||||
|
|
||||||
|
if key.isValid:
|
||||||
|
rev.kMap[rvid] = key
|
||||||
|
|
||||||
|
ok()
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Public functions
|
# Public functions
|
||||||
|
@ -48,7 +87,7 @@ proc revFilter*(
|
||||||
else:
|
else:
|
||||||
return err((rvid.vid,rc.error))
|
return err((rvid.vid,rc.error))
|
||||||
|
|
||||||
# Calculate reverse changes for the `kMap` sequence.
|
# Calculate reverse changes for the `kMap[]` structural table.
|
||||||
for rvid in filter.kMap.keys:
|
for rvid in filter.kMap.keys:
|
||||||
let rc = db.getKeyUbe rvid
|
let rc = db.getKeyUbe rvid
|
||||||
if rc.isOk:
|
if rc.isOk:
|
||||||
|
@ -58,6 +97,10 @@ proc revFilter*(
|
||||||
else:
|
else:
|
||||||
return err((rvid.vid,rc.error))
|
return err((rvid.vid,rc.error))
|
||||||
|
|
||||||
|
# Reverse changes for `delTree[]` list.
|
||||||
|
for rvid in filter.delTree:
|
||||||
|
? db.revSubTree(rev, rvid)
|
||||||
|
|
||||||
ok(rev)
|
ok(rev)
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
|
|
|
@ -37,10 +37,6 @@ export
|
||||||
tables, aristo_constants, desc_error, desc_identifiers, desc_nibbles,
|
tables, aristo_constants, desc_error, desc_identifiers, desc_nibbles,
|
||||||
desc_structural, keyed_queue
|
desc_structural, keyed_queue
|
||||||
|
|
||||||
const
|
|
||||||
accLruSize* = 1024 * 1024
|
|
||||||
# LRU cache size for accounts that have storage
|
|
||||||
|
|
||||||
type
|
type
|
||||||
AristoTxRef* = ref object
|
AristoTxRef* = ref object
|
||||||
## Transaction descriptor
|
## Transaction descriptor
|
||||||
|
|
|
@ -124,6 +124,8 @@ type
|
||||||
kMap*: Table[RootedVertexID,HashKey] ## Merkle hash key mapping
|
kMap*: Table[RootedVertexID,HashKey] ## Merkle hash key mapping
|
||||||
vTop*: VertexID ## Last used vertex ID
|
vTop*: VertexID ## Last used vertex ID
|
||||||
|
|
||||||
|
delTree*: seq[RootedVertexID] ## Not yet fully deleted sub-trees
|
||||||
|
|
||||||
accLeaves*: Table[Hash256, VertexRef] ## Account path -> VertexRef
|
accLeaves*: Table[Hash256, VertexRef] ## Account path -> VertexRef
|
||||||
stoLeaves*: Table[Hash256, VertexRef] ## Storage path -> VertexRef
|
stoLeaves*: Table[Hash256, VertexRef] ## Storage path -> VertexRef
|
||||||
|
|
||||||
|
|
|
@ -78,7 +78,7 @@ proc retrieveAccountPayload(
|
||||||
return err(FetchPathNotFound)
|
return err(FetchPathNotFound)
|
||||||
return err(error)
|
return err(error)
|
||||||
|
|
||||||
ok db.accLeaves.lruAppend(accKey, leafVtx, accLruSize).lData
|
ok db.accLeaves.lruAppend(accKey, leafVtx, ACC_LRU_SIZE).lData
|
||||||
|
|
||||||
proc retrieveMerkleHash(
|
proc retrieveMerkleHash(
|
||||||
db: AristoDbRef;
|
db: AristoDbRef;
|
||||||
|
@ -188,7 +188,7 @@ proc retrieveStoragePayload(
|
||||||
leafVtx = db.retrieveLeaf(? db.fetchStorageID(accPath), stoPath.data).valueOr:
|
leafVtx = db.retrieveLeaf(? db.fetchStorageID(accPath), stoPath.data).valueOr:
|
||||||
return err(error)
|
return err(error)
|
||||||
|
|
||||||
ok db.stoLeaves.lruAppend(mixKey, leafVtx, accLruSize).lData.stoData
|
ok db.stoLeaves.lruAppend(mixKey, leafVtx, ACC_LRU_SIZE).lData.stoData
|
||||||
|
|
||||||
proc hasStoragePayload(
|
proc hasStoragePayload(
|
||||||
db: AristoDbRef;
|
db: AristoDbRef;
|
||||||
|
|
|
@ -171,6 +171,7 @@ func isEmpty*(ly: LayerRef): bool =
|
||||||
## tables are empty. The field `txUid` is ignored, here.
|
## tables are empty. The field `txUid` is ignored, here.
|
||||||
ly.sTab.len == 0 and
|
ly.sTab.len == 0 and
|
||||||
ly.kMap.len == 0 and
|
ly.kMap.len == 0 and
|
||||||
|
ly.delTree.len == 0 and
|
||||||
ly.accLeaves.len == 0 and
|
ly.accLeaves.len == 0 and
|
||||||
ly.stoLeaves.len == 0
|
ly.stoLeaves.len == 0
|
||||||
|
|
||||||
|
@ -186,6 +187,8 @@ func layersMergeOnto*(src: LayerRef; trg: var LayerObj) =
|
||||||
for (vid,key) in src.kMap.pairs:
|
for (vid,key) in src.kMap.pairs:
|
||||||
trg.kMap[vid] = key
|
trg.kMap[vid] = key
|
||||||
trg.vTop = src.vTop
|
trg.vTop = src.vTop
|
||||||
|
for rvid in src.delTree:
|
||||||
|
trg.delTree.add rvid
|
||||||
for (accPath,leafVtx) in src.accLeaves.pairs:
|
for (accPath,leafVtx) in src.accLeaves.pairs:
|
||||||
trg.accLeaves[accPath] = leafVtx
|
trg.accLeaves[accPath] = leafVtx
|
||||||
for (mixPath,leafVtx) in src.stoLeaves.pairs:
|
for (mixPath,leafVtx) in src.stoLeaves.pairs:
|
||||||
|
@ -204,6 +207,7 @@ func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
|
||||||
sTab: layers[0].sTab.dup, # explicit dup for ref values
|
sTab: layers[0].sTab.dup, # explicit dup for ref values
|
||||||
kMap: layers[0].kMap,
|
kMap: layers[0].kMap,
|
||||||
vTop: layers[^1].vTop,
|
vTop: layers[^1].vTop,
|
||||||
|
delTree: layers[0].delTree,
|
||||||
accLeaves: layers[0].accLeaves,
|
accLeaves: layers[0].accLeaves,
|
||||||
stoLeaves: layers[0].stoLeaves)
|
stoLeaves: layers[0].stoLeaves)
|
||||||
|
|
||||||
|
@ -213,6 +217,8 @@ func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
|
||||||
result.sTab[vid] = vtx
|
result.sTab[vid] = vtx
|
||||||
for (vid,key) in layers[n].kMap.pairs:
|
for (vid,key) in layers[n].kMap.pairs:
|
||||||
result.kMap[vid] = key
|
result.kMap[vid] = key
|
||||||
|
for rvid in layers[n].delTree:
|
||||||
|
result.delTree.add rvid
|
||||||
for (accPath,vtx) in layers[n].accLeaves.pairs:
|
for (accPath,vtx) in layers[n].accLeaves.pairs:
|
||||||
result.accLeaves[accPath] = vtx
|
result.accLeaves[accPath] = vtx
|
||||||
for (mixPath,vtx) in layers[n].stoLeaves.pairs:
|
for (mixPath,vtx) in layers[n].stoLeaves.pairs:
|
||||||
|
|
|
@ -96,9 +96,9 @@ proc mergeGenericData*(
|
||||||
|
|
||||||
proc mergeStorageData*(
|
proc mergeStorageData*(
|
||||||
db: AristoDbRef; # Database, top layer
|
db: AristoDbRef; # Database, top layer
|
||||||
accPath: Hash256; # Needed for accounts payload
|
accPath: Hash256; # Needed for accounts payload
|
||||||
stoPath: Hash256; # Storage data path (aka key)
|
stoPath: Hash256; # Storage data path (aka key)
|
||||||
stoData: UInt256; # Storage data payload value
|
stoData: UInt256; # Storage data payload value
|
||||||
): Result[void,AristoError] =
|
): Result[void,AristoError] =
|
||||||
## Store the `stoData` data argument on the storage area addressed by
|
## Store the `stoData` data argument on the storage area addressed by
|
||||||
## `(accPath,stoPath)` where `accPath` is the account key (into the MPT)
|
## `(accPath,stoPath)` where `accPath` is the account key (into the MPT)
|
||||||
|
|
|
@ -16,7 +16,7 @@ import
|
||||||
stew/byteutils,
|
stew/byteutils,
|
||||||
nimcrypto,
|
nimcrypto,
|
||||||
results,
|
results,
|
||||||
../db/aristo,
|
../db/aristo/aristo_sign,
|
||||||
../constants
|
../constants
|
||||||
|
|
||||||
export eth_types_rlp
|
export eth_types_rlp
|
||||||
|
|
|
@ -124,21 +124,21 @@ let
|
||||||
mainTest6r* = mainSampleEx
|
mainTest6r* = mainSampleEx
|
||||||
.cloneWith(
|
.cloneWith(
|
||||||
name = "-ex-ar-some",
|
name = "-ex-ar-some",
|
||||||
numBlocks = 257_400,
|
numBlocks = 1_000_000,
|
||||||
dbType = AristoDbRocks,
|
dbType = AristoDbRocks,
|
||||||
dbName = "main-open") # for resuming on the same persistent DB
|
dbName = "main-open") # for resuming on the same persistent DB
|
||||||
|
|
||||||
mainTest7r* = mainSampleEx
|
mainTest7r* = mainSampleEx
|
||||||
.cloneWith(
|
.cloneWith(
|
||||||
name = "-ex-ar-more",
|
name = "-ex-ar-more",
|
||||||
numBlocks = 1_460_700, # failure at 1,460,736
|
numBlocks = 1_500_000,
|
||||||
dbType = AristoDbRocks,
|
dbType = AristoDbRocks,
|
||||||
dbName = "main-open") # for resuming on the same persistent DB
|
dbName = "main-open") # for resuming on the same persistent DB
|
||||||
|
|
||||||
mainTest8r* = mainSampleEx
|
mainTest8r* = mainSampleEx
|
||||||
.cloneWith(
|
.cloneWith(
|
||||||
name = "-ex-ar-more2",
|
name = "-ex-ar-more2",
|
||||||
numBlocks = 1_460_735, # failure at 1,460,736
|
numBlocks = 2_000_000,
|
||||||
dbType = AristoDbRocks,
|
dbType = AristoDbRocks,
|
||||||
dbName = "main-open") # for resuming on the same persistent DB
|
dbName = "main-open") # for resuming on the same persistent DB
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue