status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Aristo and kvt balancer management update (#2504) 

* Aristo: Merge `delta_siblings` module into `deltaPersistent()`

* Aristo: Add `isEmpty()` for canonical checking whether a layer is empty

* Aristo: Merge `LayerDeltaRef` into `LayerObj`

why:
  No need to maintain nested object refs anymore. Previously the
 `LayerDeltaRef` object had a companion `LayerFinalRef` which held
  non-delta layer information.

* Kvt: Merge `LayerDeltaRef` into `LayerRef`

why:
  No need to maintain nested object refs (as with `Aristo`)

* Kvt: Re-write balancer logic similar to `Aristo`

why:
  Although `Kvt` was a cheap copy of `Aristo` it sort of got out of
  sync and the balancer code was wrong.

* Update iterator over forked peers

why:
  Yield additional field `isLast` indicating that the last iteration
  cycle was approached.

* Optimise balancer calculation.

why:
  One can often avoid providing a new object containing the merge of two
  layers for the balancer. This avoids copying tables. In some cases this
  is replaced by `hasKey()` look ups though. One uses one of the two
  to combine and merges the other into the first.

  Of course, this needs some checks for making sure that none of the
  components to merge is eventually shared with something else.

* Fix copyright year
This commit is contained in:
Jordan Hrycaj 2024-07-18 21:32:32 +00:00 committed by GitHub
parent ee323d5ff8
commit 5ac362fe6f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 362 additions and 462 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
nimbus/db/aristo/aristo_debug.nim
View File

@ -291,8 +291,8 @@ proc ppXMap*(
result.setLen(result.len - 1) result.setLen(result.len - 1)
result &= "}" result &= "}"
proc ppFilter( proc ppBalancer(
fl: LayerDeltaRef; fl: LayerRef;
db: AristoDbRef; db: AristoDbRef;
indent: int; indent: int;
): string = ): string =
@ -398,18 +398,18 @@ proc ppLayer(
if 2 < nOKs: if 2 < nOKs:
result &= "<layer>".doPrefix(false) result &= "<layer>".doPrefix(false)
if vTopOk: if vTopOk:
result &= "".doPrefix(true) & "vTop=" & layer.delta.vTop.ppVid result &= "".doPrefix(true) & "vTop=" & layer.vTop.ppVid
if sTabOk: if sTabOk:
let let
tLen = layer.delta.sTab.len tLen = layer.sTab.len
info = "sTab(" & $tLen & ")" info = "sTab(" & $tLen & ")"
result &= info.doPrefix(0 < tLen) & layer.delta.sTab.ppSTab(db,indent+2) result &= info.doPrefix(0 < tLen) & layer.sTab.ppSTab(db,indent+2)
if kMapOk: if kMapOk:
let let
tLen = layer.delta.kMap.len tLen = layer.kMap.len
info = "kMap(" & $tLen & ")" info = "kMap(" & $tLen & ")"
result &= info.doPrefix(0 < tLen) result &= info.doPrefix(0 < tLen)
result &= db.ppXMap(layer.delta.kMap, indent+2) result &= db.ppXMap(layer.kMap, indent+2)
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public functions # Public functions
@ -528,53 +528,17 @@ proc pp*(
layer: LayerRef; layer: LayerRef;
db: AristoDbRef; db: AristoDbRef;
indent = 4; indent = 4;
balancerOk = false;
sTabOk = true,
kMapOk = true,
other = true,
): string = ): string =
if balancerOk:
layer.ppLayer( layer.ppLayer(
db, vTopOk=true, sTabOk=true, kMapOk=true) db.orDefault(), vTopOk=other, sTabOk=sTabOk, kMapOk=kMapOk)
else:
proc pp*(
layer: LayerRef;
db: AristoDbRef;
xTabOk: bool;
indent = 4;
): string =
layer.ppLayer( layer.ppLayer(
db, vTopOk=true, sTabOk=xTabOk, kMapOk=true) db.orDefault(), vTopOk=other, sTabOk=sTabOk, kMapOk=kMapOk)
proc pp*(
layer: LayerRef;
db: AristoDbRef;
xTabOk: bool;
kMapOk: bool;
other = false;
indent = 4;
): string =
layer.ppLayer(
db, vTopOk=other, sTabOk=xTabOk, kMapOk=kMapOk)
proc pp*(
db: AristoDbRef;
xTabOk: bool;
indent = 4;
): string =
db.layersCc.pp(db, xTabOk=xTabOk, indent=indent)
proc pp*(
db: AristoDbRef;
xTabOk: bool;
kMapOk: bool;
other = false;
indent = 4;
): string =
db.layersCc.pp(db, xTabOk=xTabOk, kMapOk=kMapOk, other=other, indent=indent)
proc pp*(
filter: LayerDeltaRef;
db = AristoDbRef(nil);
indent = 4;
): string =
filter.ppFilter(db.orDefault(), indent)
proc pp*( proc pp*(
be: BackendRef; be: BackendRef;
@ -582,7 +546,7 @@ proc pp*(
limit = 100; limit = 100;
indent = 4; indent = 4;
): string = ): string =
result = db.balancer.ppFilter(db, indent+1) & indent.toPfx result = db.balancer.ppBalancer(db, indent+1) & indent.toPfx
case be.kind: case be.kind:
of BackendMemory: of BackendMemory:
result &= be.MemBackendRef.ppBe(db, limit, indent+1) result &= be.MemBackendRef.ppBe(db, limit, indent+1)
@ -599,11 +563,12 @@ proc pp*(
topOk = true; topOk = true;
stackOk = true; stackOk = true;
kMapOk = true; kMapOk = true;
sTabOk = true;
limit = 100; limit = 100;
): string = ): string =
if topOk: if topOk:
result = db.layersCc.pp( result = db.layersCc.pp(
db, xTabOk=true, kMapOk=kMapOk, other=true, indent=indent) db, sTabOk=sTabOk, kMapOk=kMapOk, other=true, indent=indent)
let stackOnlyOk = stackOk and not (topOk or balancerOk or backendOk) let stackOnlyOk = stackOk and not (topOk or balancerOk or backendOk)
if not stackOnlyOk: if not stackOnlyOk:
result &= indent.toPfx & "level=" & $db.stack.len result &= indent.toPfx & "level=" & $db.stack.len
@ -614,15 +579,15 @@ proc pp*(
let let
m = layers.len - n - 1 m = layers.len - n - 1
l = db.layersCc m l = db.layersCc m
a = w.delta.kMap.values.toSeq.filterIt(not it.isValid).len a = w.kMap.values.toSeq.filterIt(not it.isValid).len
c = l.delta.kMap.values.toSeq.filterIt(not it.isValid).len c = l.kMap.values.toSeq.filterIt(not it.isValid).len
result &= "(" & $(w.delta.kMap.len - a) & "," & $a & ")" result &= "(" & $(w.kMap.len - a) & "," & $a & ")"
lStr &= " " & $m & "=(" & $(l.delta.kMap.len - c) & "," & $c & ")" lStr &= " " & $m & "=(" & $(l.kMap.len - c) & "," & $c & ")"
result &= " =>" & lStr result &= " =>" & lStr
if backendOk: if backendOk:
result &= indent.toPfx & db.backend.pp(db, limit=limit, indent) result &= indent.toPfx & db.backend.pp(db, limit=limit, indent)
elif balancerOk: elif balancerOk:
result &= indent.toPfx & db.balancer.ppFilter(db, indent+1) result &= indent.toPfx & db.balancer.ppBalancer(db, indent+1)
proc pp*(sdb: MerkleSignRef; indent = 4): string = proc pp*(sdb: MerkleSignRef; indent = 4): string =
result = "" & result = "" &

42
nimbus/db/aristo/aristo_delta.nim
View File

@ -16,9 +16,9 @@ import
std/tables, std/tables,
eth/common, eth/common,
results, results,
./aristo_desc, "."/[aristo_desc, aristo_layers],
./aristo_desc/desc_backend, ./aristo_desc/desc_backend,
./aristo_delta/delta_siblings ./aristo_delta/[delta_merge, delta_reverse]
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public functions, save to backend # Public functions, save to backend
@ -34,17 +34,16 @@ proc deltaPersistent*(
nxtFid = 0u64; # Next filter ID (if any) nxtFid = 0u64; # Next filter ID (if any)
reCentreOk = false; reCentreOk = false;
): Result[void,AristoError] = ): Result[void,AristoError] =
## Resolve (i.e. move) the backend filter into the physical backend database. ## Resolve (i.e. move) the balancer into the physical backend database.
## ##
## This needs write permission on the backend DB for the argument `db` ## This needs write permission on the backend DB for the descriptor argument
## descriptor (see the function `aristo_desc.isCentre()`.) With the argument ## `db` (see the function `aristo_desc.isCentre()`.) If the argument flag
## flag `reCentreOk` passed `true`, write permission will be temporarily ## `reCentreOk` is passed `true`, write permission will be temporarily
## acquired when needed. ## acquired when needed.
## ##
## When merging the current backend filter, its reverse will be is stored as ## When merging the current backend filter, its reverse will be is stored
## back log on the filter fifos (so the current state can be retrieved.) ## on other non-centre descriptors so there is no visible database change
## Also, other non-centre descriptors are updated so there is no visible ## for these.
## database change for these descriptors.
## ##
let be = db.backend let be = db.backend
if be.isNil: if be.isNil:
@ -64,9 +63,21 @@ proc deltaPersistent*(
# Always re-centre to `parent` (in case `reCentreOk` was set) # Always re-centre to `parent` (in case `reCentreOk` was set)
defer: discard parent.reCentre() defer: discard parent.reCentre()
# Initialise peer filter balancer. # Update forked balancers here do that errors are detected early (if any.)
let updateSiblings = ? UpdateSiblingsRef.init db if 0 < db.nForked:
defer: updateSiblings.rollback() let rev = db.revFilter(db.balancer).valueOr:
return err(error[1])
if not rev.isEmpty: # Can an empty `rev` happen at all?
var unsharedRevOk = true
for w in db.forked:
if not w.db.balancer.isValid:
unsharedRevOk = false
# The `rev` filter can be modified if one can make sure that it is
# not shared (i.e. only previously merged into the w.db.balancer.)
# Note that it is trivially true for a single fork.
let modLowerOk = w.isLast and unsharedRevOk
w.db.balancer = deltaMerge(
w.db.balancer, modUpperOk=false, rev, modLowerOk=modLowerOk)
let lSst = SavedState( let lSst = SavedState(
key: EMPTY_ROOT_HASH, # placeholder for more key: EMPTY_ROOT_HASH, # placeholder for more
@ -93,8 +104,9 @@ proc deltaPersistent*(
if not db.stoLeaves.lruUpdate(mixKey, vtx): if not db.stoLeaves.lruUpdate(mixKey, vtx):
discard db.stoLeaves.lruAppend(mixKey, vtx, accLruSize) discard db.stoLeaves.lruAppend(mixKey, vtx, accLruSize)
# Update dudes and this descriptor # Done with balancer, all saved to backend
? updateSiblings.update().commit() db.balancer = LayerRef(nil)
ok() ok()
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------

90
nimbus/db/aristo/aristo_delta/delta_merge.nim
View File

@ -11,72 +11,70 @@
import import
std/tables, std/tables,
eth/common, eth/common,
results, ".."/[aristo_desc, aristo_layers]
".."/[aristo_desc, aristo_get]
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public functions # Public functions
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
proc deltaMerge*( proc deltaMerge*(
db: AristoDbRef; upper: LayerRef; # Think of `top`, `nil` is ok
upper: LayerDeltaRef; # new filter, `nil` is ok modUpperOk: bool; # May re-use/modify `upper`
lower: LayerDeltaRef; # Trg filter, `nil` is ok lower: LayerRef; # Think of `balancer`, `nil` is ok
): Result[LayerDeltaRef,(VertexID,AristoError)] = modLowerOk: bool; # May re-use/modify `lower`
): LayerRef =
## Merge argument `upper` into the `lower` filter instance. ## Merge argument `upper` into the `lower` filter instance.
## ##
## Note that the namimg `upper` and `lower` indicate that the filters are ## Note that the namimg `upper` and `lower` indicate that the filters are
## stacked and the database access is `upper -> lower -> backend` whereas ## stacked and the database access is `upper -> lower -> backend`.
## the `src/trg` matching logic goes the other way round.
## ##
# Degenerate case: `upper` is void
if lower.isNil: if lower.isNil:
if upper.isNil: # Degenerate case: `upper` is void
# Even more degenerate case when both filters are void result = upper
return ok LayerDeltaRef(nil)
return ok(upper)
# Degenerate case: `upper` is non-trivial and `lower` is void elif upper.isNil:
if upper.isNil: # Degenerate case: `lower` is void
return ok(lower) result = lower
# There is no need to deep copy table vertices as they will not be modified. elif modLowerOk:
let newFilter = LayerDeltaRef( # Can modify `lower` which is the prefered action mode but applies only
sTab: lower.sTab, # in cases where the `lower` argument is not shared.
lower.vTop = upper.vTop
layersMergeOnto(upper, lower[])
result = lower
elif not modUpperOk:
# Cannot modify any argument layers.
result = LayerRef(
sTab: lower.sTab, # shallow copy (entries will not be modified)
kMap: lower.kMap, kMap: lower.kMap,
accLeaves: lower.accLeaves,
stoLeaves: lower.stoLeaves,
vTop: upper.vTop) vTop: upper.vTop)
layersMergeOnto(upper, result[])
for (rvid,vtx) in upper.sTab.pairs:
if vtx.isValid or not newFilter.sTab.hasKey rvid:
newFilter.sTab[rvid] = vtx
elif newFilter.sTab.getOrVoid(rvid).isValid:
let rc = db.getVtxUbe rvid
if rc.isOk:
newFilter.sTab[rvid] = vtx # VertexRef(nil)
elif rc.error == GetVtxNotFound:
newFilter.sTab.del rvid
else: else:
return err((rvid.vid,rc.error)) # Otherwise avoid copying some tables by modifyinh `upper`. This is not
# completely free as the merge direction changes to merging the `lower`
# layer up into the higher prioritised `upper` layer (note that the `lower`
# argument filter is read-only.) Here again, the `upper` argument must not
# be a shared layer/filter.
for (rvid,vtx) in lower.sTab.pairs:
if not upper.sTab.hasKey(rvid):
upper.sTab[rvid] = vtx
for (rvid,key) in upper.kMap.pairs: for (rvid,key) in lower.kMap.pairs:
if key.isValid or not newFilter.kMap.hasKey rvid: if not upper.kMap.hasKey(rvid):
newFilter.kMap[rvid] = key upper.kMap[rvid] = key
elif newFilter.kMap.getOrVoid(rvid).isValid:
let rc = db.getKeyUbe rvid
if rc.isOk:
newFilter.kMap[rvid] = key
elif rc.error == GetKeyNotFound:
newFilter.kMap.del rvid
else:
return err((rvid.vid,rc.error))
for (accPath,leafVtx) in upper.accLeaves.pairs: for (accPath,leafVtx) in lower.accLeaves.pairs:
newFilter.accLeaves[accPath] = leafVtx if not upper.accLeaves.hasKey(accPath):
upper.accLeaves[accPath] = leafVtx
for (mixPath,leafVtx) in upper.stoLeaves.pairs: for (mixPath,leafVtx) in lower.stoLeaves.pairs:
newFilter.stoLeaves[mixPath] = leafVtx if not upper.stoLeaves.hasKey(mixPath):
upper.stoLeaves[mixPath] = leafVtx
ok newFilter result = upper
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# End # End

7
nimbus/db/aristo/aristo_delta/delta_reverse.nim
View File

@ -20,16 +20,15 @@ import
proc revFilter*( proc revFilter*(
db: AristoDbRef; # Database db: AristoDbRef; # Database
filter: LayerDeltaRef; # Filter to revert filter: LayerRef; # Filter to revert
): Result[LayerDeltaRef,(VertexID,AristoError)] = ): Result[LayerRef,(VertexID,AristoError)] =
## Assemble reverse filter for the `filter` argument, i.e. changes to the ## Assemble reverse filter for the `filter` argument, i.e. changes to the
## backend that reverse the effect of applying the this read-only filter. ## backend that reverse the effect of applying the this read-only filter.
## ##
## This read-only filter is calculated against the current unfiltered ## This read-only filter is calculated against the current unfiltered
## backend (excluding optionally installed read-only filter.) ## backend (excluding optionally installed read-only filter.)
## ##
# Register MPT state roots for reverting back let rev = LayerRef()
let rev = LayerDeltaRef()
# Get vid generator state on backend # Get vid generator state on backend
block: block:

120
nimbus/db/aristo/aristo_delta/delta_siblings.nim
View File

@ -1,120 +0,0 @@
# nimbus-eth1
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
import
results,
eth/common,
../aristo_desc,
"."/[delta_merge, delta_reverse]
type
UpdateState = enum
Initial = 0
Updated,
Finished
UpdateSiblingsRef* = ref object
## Update transactional context
state: UpdateState
db: AristoDbRef ## Main database access
balancers: seq[(AristoDbRef,LayerDeltaRef)] ## Rollback data
rev: LayerDeltaRef ## Reverse filter set up
# ------------------------------------------------------------------------------
# Public contructor, commit, rollback
# ------------------------------------------------------------------------------
proc rollback*(ctx: UpdateSiblingsRef) =
## Rollback any changes made by the `update()` function. Subsequent
## `rollback()` or `commit()` calls will be without effect.
if ctx.state == Updated:
for (d,f) in ctx.balancers:
d.balancer = f
ctx.state = Finished
proc commit*(ctx: UpdateSiblingsRef): Result[void,AristoError] =
## Accept updates. Subsequent `rollback()` calls will be without effect.
if ctx.state != Updated:
ctx.rollback()
return err(FilSiblingsCommitUnfinshed)
ctx.db.balancer = LayerDeltaRef(nil)
ctx.state = Finished
ok()
proc commit*(
rc: Result[UpdateSiblingsRef,AristoError];
): Result[void,AristoError] =
## Variant of `commit()` for joining with `update()`
(? rc).commit()
proc init*(
T: type UpdateSiblingsRef; # Context type
db: AristoDbRef; # Main database
): Result[T,AristoError] =
## Set up environment for installing the reverse of the `db` argument current
## read-only filter onto every associated descriptor referring to the same
## database.
if not db.isCentre:
return err(FilBackendRoMode)
if db.nForked == 0:
return ok T(db: db) # No need to do anything
func fromVae(err: (VertexID,AristoError)): AristoError =
err[1]
# Filter rollback context
ok T(
db: db,
rev: ? db.revFilter(db.balancer).mapErr fromVae) # Reverse filter
# ------------------------------------------------------------------------------
# Public functions
# ------------------------------------------------------------------------------
proc update*(ctx: UpdateSiblingsRef): Result[UpdateSiblingsRef,AristoError] =
## This function installs the reverse of the `init()` argument `db` current
## read-only filter onto every associated descriptor referring to the same
## database. If the function fails, a `rollback()` is called automatically.
##
## This function might do some real work so it was detached from `init()` so
## it can be called late but before the physical database is updated.
##
if ctx.state == Initial:
ctx.state = Updated
if not ctx.rev.isNil:
let db = ctx.db
# Update distributed filters. Note that the physical backend database
# must not have been updated, yet. So the new root key for the backend
# will be `db.balancer.kMap[$1]`.
for w in db.forked:
if w.balancer.isNil:
# Sharing the `ctx.rev` ref is safe as it is read-inly
w.balancer = ctx.rev
else:
let rc = db.deltaMerge(w.balancer, ctx.rev)
if rc.isErr:
ctx.rollback()
return err(rc.error[1])
ctx.balancers.add (w, w.balancer)
w.balancer = rc.value
ok(ctx)
proc update*(
rc: Result[UpdateSiblingsRef,AristoError]
): Result[UpdateSiblingsRef,AristoError] =
## Variant of `update()` for joining with `init()`
(? rc).update()
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

28
nimbus/db/aristo/aristo_desc.nim
View File

@ -74,7 +74,7 @@ type
## Three tier database object supporting distributed instances. ## Three tier database object supporting distributed instances.
top*: LayerRef ## Database working layer, mutable top*: LayerRef ## Database working layer, mutable
stack*: seq[LayerRef] ## Stashed immutable parent layers stack*: seq[LayerRef] ## Stashed immutable parent layers
balancer*: LayerDeltaRef ## Baland out concurrent backend access balancer*: LayerRef ## Balance out concurrent backend access
backend*: BackendRef ## Backend database (may well be `nil`) backend*: BackendRef ## Backend database (may well be `nil`)
txRef*: AristoTxRef ## Latest active transaction txRef*: AristoTxRef ## Latest active transaction
@ -152,8 +152,8 @@ func isValid*(nd: NodeRef): bool =
func isValid*(pid: PathID): bool = func isValid*(pid: PathID): bool =
pid != VOID_PATH_ID pid != VOID_PATH_ID
func isValid*(filter: LayerDeltaRef): bool = func isValid*(layer: LayerRef): bool =
filter != LayerDeltaRef(nil) layer != LayerRef(nil)
func isValid*(root: Hash256): bool = func isValid*(root: Hash256): bool =
root != EMPTY_ROOT_HASH root != EMPTY_ROOT_HASH
@ -250,11 +250,11 @@ proc fork*(
if not noTopLayer: if not noTopLayer:
clone.top = LayerRef.init() clone.top = LayerRef.init()
if not db.balancer.isNil: if not db.balancer.isNil:
clone.top.delta.vTop = db.balancer.vTop clone.top.vTop = db.balancer.vTop
else: else:
let rc = clone.backend.getTuvFn() let rc = clone.backend.getTuvFn()
if rc.isOk: if rc.isOk:
clone.top.delta.vTop = rc.value clone.top.vTop = rc.value
elif rc.error != GetTuvNotFound: elif rc.error != GetTuvNotFound:
return err(rc.error) return err(rc.error)
@ -263,13 +263,19 @@ proc fork*(
ok clone ok clone
iterator forked*(db: AristoDbRef): AristoDbRef = iterator forked*(db: AristoDbRef): tuple[db: AristoDbRef, isLast: bool] =
## Interate over all non centre descriptors (see comments on `reCentre()` ## Interate over all non centre descriptors (see comments on `reCentre()`
## for details.) ## for details.)
##
## The second `isLast` yielded loop entry is `true` if the yielded tuple
## is the last entry in the list.
##
if not db.dudes.isNil: if not db.dudes.isNil:
for dude in db.getCentre.dudes.peers.items: var nLeft = db.dudes.peers.len
for dude in db.dudes.peers.items:
if dude != db.dudes.centre: if dude != db.dudes.centre:
yield dude nLeft.dec
yield (dude, nLeft == 1)
func nForked*(db: AristoDbRef): int = func nForked*(db: AristoDbRef): int =
## Returns the number of non centre descriptors (see comments on `reCentre()` ## Returns the number of non centre descriptors (see comments on `reCentre()`
@ -313,12 +319,12 @@ iterator rstack*(db: AristoDbRef): LayerRef =
for i in 0..<db.stack.len: for i in 0..<db.stack.len:
yield db.stack[db.stack.len - i - 1] yield db.stack[db.stack.len - i - 1]
proc deltaAtLevel*(db: AristoDbRef, level: int): LayerDeltaRef = proc deltaAtLevel*(db: AristoDbRef, level: int): LayerRef =
if level == 0: if level == 0:
db.top.delta db.top
elif level > 0: elif level > 0:
doAssert level <= db.stack.len doAssert level <= db.stack.len
db.stack[^level].delta db.stack[^level]
elif level == -1: elif level == -1:
doAssert db.balancer != nil doAssert db.balancer != nil
db.balancer db.balancer

10
nimbus/db/aristo/aristo_desc/desc_structural.nim
View File

@ -89,7 +89,8 @@ type
key*: Hash256 ## Some state hash (if any) key*: Hash256 ## Some state hash (if any)
serial*: uint64 ## Generic identifier from application serial*: uint64 ## Generic identifier from application
LayerDeltaRef* = ref object LayerRef* = ref LayerObj
LayerObj* = object
## Delta layers are stacked implying a tables hierarchy. Table entries on ## Delta layers are stacked implying a tables hierarchy. Table entries on
## a higher level take precedence over lower layer table entries. So an ## a higher level take precedence over lower layer table entries. So an
## existing key-value table entry of a layer on top supersedes same key ## existing key-value table entry of a layer on top supersedes same key
@ -119,11 +120,6 @@ type
accLeaves*: Table[Hash256, VertexRef] ## Account path -> VertexRef accLeaves*: Table[Hash256, VertexRef] ## Account path -> VertexRef
stoLeaves*: Table[Hash256, VertexRef] ## Storage path -> VertexRef stoLeaves*: Table[Hash256, VertexRef] ## Storage path -> VertexRef
LayerRef* = ref LayerObj
LayerObj* = object
## Hexary trie database layer structures. Any layer holds the full
## change relative to the backend.
delta*: LayerDeltaRef ## Most structural tables held as deltas
txUid*: uint ## Transaction identifier if positive txUid*: uint ## Transaction identifier if positive
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -132,7 +128,7 @@ type
func init*(T: type LayerRef): T = func init*(T: type LayerRef): T =
## Constructor, returns empty layer ## Constructor, returns empty layer
T(delta: LayerDeltaRef()) T()
func hash*(node: NodeRef): Hash = func hash*(node: NodeRef): Hash =
## Table/KeyedQueue/HashSet mixin ## Table/KeyedQueue/HashSet mixin

3
nimbus/db/aristo/aristo_init/persistent.nim
View File

@ -51,8 +51,7 @@ proc newAristoRdbDbRef(
return err(rc.error) return err(rc.error)
rc.value rc.value
ok((AristoDbRef( ok((AristoDbRef(
top: LayerRef( top: LayerRef(vTop: vTop),
delta: LayerDeltaRef(vTop: vTop)),
backend: be), oCfs)) backend: be), oCfs))
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------

93
nimbus/db/aristo/aristo_layers.nim
View File

@ -30,7 +30,7 @@ func dup(sTab: Table[RootedVertexID,VertexRef]): Table[RootedVertexID,VertexRef]
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
func vTop*(db: AristoDbRef): VertexID = func vTop*(db: AristoDbRef): VertexID =
db.top.delta.vTop db.top.vTop
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public getters/helpers # Public getters/helpers
@ -42,7 +42,7 @@ func nLayersVtx*(db: AristoDbRef): int =
## layers as there might be duplicate entries for the same vertex ID on ## layers as there might be duplicate entries for the same vertex ID on
## different layers. ## different layers.
## ##
db.stack.mapIt(it.delta.sTab.len).foldl(a + b, db.top.delta.sTab.len) db.stack.mapIt(it.sTab.len).foldl(a + b, db.top.sTab.len)
func nLayersKey*(db: AristoDbRef): int = func nLayersKey*(db: AristoDbRef): int =
## Number of vertex ID/key entries on the cache layers. This is an upper ## Number of vertex ID/key entries on the cache layers. This is an upper
@ -50,7 +50,7 @@ func nLayersKey*(db: AristoDbRef): int =
## layers as there might be duplicate entries for the same vertex ID on ## layers as there might be duplicate entries for the same vertex ID on
## different layers. ## different layers.
## ##
db.stack.mapIt(it.delta.kMap.len).foldl(a + b, db.top.delta.kMap.len) db.stack.mapIt(it.kMap.len).foldl(a + b, db.top.kMap.len)
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public functions: getter variants # Public functions: getter variants
@ -60,11 +60,11 @@ func layersGetVtx*(db: AristoDbRef; rvid: RootedVertexID): Opt[(VertexRef, int)]
## Find a vertex on the cache layers. An `ok()` result might contain a ## Find a vertex on the cache layers. An `ok()` result might contain a
## `nil` vertex if it is stored on the cache that way. ## `nil` vertex if it is stored on the cache that way.
## ##
db.top.delta.sTab.withValue(rvid, item): db.top.sTab.withValue(rvid, item):
return Opt.some((item[], 0)) return Opt.some((item[], 0))
for i, w in enumerate(db.rstack): for i, w in enumerate(db.rstack):
w.delta.sTab.withValue(rvid, item): w.sTab.withValue(rvid, item):
return Opt.some((item[], i + 1)) return Opt.some((item[], i + 1))
Opt.none((VertexRef, int)) Opt.none((VertexRef, int))
@ -78,11 +78,11 @@ func layersGetKey*(db: AristoDbRef; rvid: RootedVertexID): Opt[(HashKey, int)] =
## Find a hash key on the cache layers. An `ok()` result might contain a void ## Find a hash key on the cache layers. An `ok()` result might contain a void
## hash key if it is stored on the cache that way. ## hash key if it is stored on the cache that way.
## ##
db.top.delta.kMap.withValue(rvid, item): db.top.kMap.withValue(rvid, item):
return Opt.some((item[], 0)) return Opt.some((item[], 0))
for i, w in enumerate(db.rstack): for i, w in enumerate(db.rstack):
w.delta.kMap.withValue(rvid, item): w.kMap.withValue(rvid, item):
return ok((item[], i + 1)) return ok((item[], i + 1))
Opt.none((HashKey, int)) Opt.none((HashKey, int))
@ -92,21 +92,21 @@ func layersGetKeyOrVoid*(db: AristoDbRef; rvid: RootedVertexID): HashKey =
(db.layersGetKey(rvid).valueOr (VOID_HASH_KEY, 0))[0] (db.layersGetKey(rvid).valueOr (VOID_HASH_KEY, 0))[0]
func layersGetAccLeaf*(db: AristoDbRef; accPath: Hash256): Opt[VertexRef] = func layersGetAccLeaf*(db: AristoDbRef; accPath: Hash256): Opt[VertexRef] =
db.top.delta.accLeaves.withValue(accPath, item): db.top.accLeaves.withValue(accPath, item):
return Opt.some(item[]) return Opt.some(item[])
for w in db.rstack: for w in db.rstack:
w.delta.accLeaves.withValue(accPath, item): w.accLeaves.withValue(accPath, item):
return Opt.some(item[]) return Opt.some(item[])
Opt.none(VertexRef) Opt.none(VertexRef)
func layersGetStoLeaf*(db: AristoDbRef; mixPath: Hash256): Opt[VertexRef] = func layersGetStoLeaf*(db: AristoDbRef; mixPath: Hash256): Opt[VertexRef] =
db.top.delta.stoLeaves.withValue(mixPath, item): db.top.stoLeaves.withValue(mixPath, item):
return Opt.some(item[]) return Opt.some(item[])
for w in db.rstack: for w in db.rstack:
w.delta.stoLeaves.withValue(mixPath, item): w.stoLeaves.withValue(mixPath, item):
return Opt.some(item[]) return Opt.some(item[])
Opt.none(VertexRef) Opt.none(VertexRef)
@ -121,7 +121,7 @@ func layersPutVtx*(
vtx: VertexRef; vtx: VertexRef;
) = ) =
## Store a (potentally empty) vertex on the top layer ## Store a (potentally empty) vertex on the top layer
db.top.delta.sTab[rvid] = vtx db.top.sTab[rvid] = vtx
func layersResVtx*( func layersResVtx*(
db: AristoDbRef; db: AristoDbRef;
@ -138,7 +138,7 @@ func layersPutKey*(
key: HashKey; key: HashKey;
) = ) =
## Store a (potentally void) hash key on the top layer ## Store a (potentally void) hash key on the top layer
db.top.delta.kMap[rvid] = key db.top.kMap[rvid] = key
func layersResKey*(db: AristoDbRef; rvid: RootedVertexID) = func layersResKey*(db: AristoDbRef; rvid: RootedVertexID) =
@ -157,30 +157,39 @@ proc layersUpdateVtx*(
func layersPutAccLeaf*(db: AristoDbRef; accPath: Hash256; leafVtx: VertexRef) = func layersPutAccLeaf*(db: AristoDbRef; accPath: Hash256; leafVtx: VertexRef) =
db.top.delta.accLeaves[accPath] = leafVtx db.top.accLeaves[accPath] = leafVtx
func layersPutStoLeaf*(db: AristoDbRef; mixPath: Hash256; leafVtx: VertexRef) = func layersPutStoLeaf*(db: AristoDbRef; mixPath: Hash256; leafVtx: VertexRef) =
db.top.delta.stoLeaves[mixPath] = leafVtx db.top.stoLeaves[mixPath] = leafVtx
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public functions # Public functions
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
func isEmpty*(ly: LayerRef): bool =
## Returns `true` if the layer does not contain any changes, i.e. all the
## tables are empty. The field `txUid` is ignored, here.
ly.sTab.len == 0 and
ly.kMap.len == 0 and
ly.accLeaves.len == 0 and
ly.stoLeaves.len == 0
func layersMergeOnto*(src: LayerRef; trg: var LayerObj) = func layersMergeOnto*(src: LayerRef; trg: var LayerObj) =
## Merges the argument `src` into the argument `trg` and returns `trg`. For ## Merges the argument `src` into the argument `trg` and returns `trg`. For
## the result layer, the `txUid` value set to `0`. ## the result layer, the `txUid` value set to `0`.
## ##
trg.txUid = 0 trg.txUid = 0
for (vid,vtx) in src.delta.sTab.pairs: for (vid,vtx) in src.sTab.pairs:
trg.delta.sTab[vid] = vtx trg.sTab[vid] = vtx
for (vid,key) in src.delta.kMap.pairs: for (vid,key) in src.kMap.pairs:
trg.delta.kMap[vid] = key trg.kMap[vid] = key
trg.delta.vTop = src.delta.vTop trg.vTop = src.vTop
for (accPath,leafVtx) in src.delta.accLeaves.pairs: for (accPath,leafVtx) in src.accLeaves.pairs:
trg.delta.accLeaves[accPath] = leafVtx trg.accLeaves[accPath] = leafVtx
for (mixPath,leafVtx) in src.delta.stoLeaves.pairs: for (mixPath,leafVtx) in src.stoLeaves.pairs:
trg.delta.stoLeaves[mixPath] = leafVtx trg.stoLeaves[mixPath] = leafVtx
func layersCc*(db: AristoDbRef; level = high(int)): LayerRef = func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
## Provide a collapsed copy of layers up to a particular transaction level. ## Provide a collapsed copy of layers up to a particular transaction level.
@ -192,24 +201,22 @@ func layersCc*(db: AristoDbRef; level = high(int)): LayerRef =
# Set up initial layer (bottom layer) # Set up initial layer (bottom layer)
result = LayerRef( result = LayerRef(
delta: LayerDeltaRef( sTab: layers[0].sTab.dup, # explicit dup for ref values
sTab: layers[0].delta.sTab.dup, # explicit dup for ref values kMap: layers[0].kMap,
kMap: layers[0].delta.kMap, vTop: layers[^1].vTop,
vTop: layers[^1].delta.vTop, accLeaves: layers[0].accLeaves,
accLeaves: layers[0].delta.accLeaves, stoLeaves: layers[0].stoLeaves)
stoLeaves: layers[0].delta.stoLeaves,
))
# Consecutively merge other layers on top # Consecutively merge other layers on top
for n in 1 ..< layers.len: for n in 1 ..< layers.len:
for (vid,vtx) in layers[n].delta.sTab.pairs: for (vid,vtx) in layers[n].sTab.pairs:
result.delta.sTab[vid] = vtx result.sTab[vid] = vtx
for (vid,key) in layers[n].delta.kMap.pairs: for (vid,key) in layers[n].kMap.pairs:
result.delta.kMap[vid] = key result.kMap[vid] = key
for (accPath,vtx) in layers[n].delta.accLeaves.pairs: for (accPath,vtx) in layers[n].accLeaves.pairs:
result.delta.accLeaves[accPath] = vtx result.accLeaves[accPath] = vtx
for (mixPath,vtx) in layers[n].delta.stoLeaves.pairs: for (mixPath,vtx) in layers[n].stoLeaves.pairs:
result.delta.stoLeaves[mixPath] = vtx result.stoLeaves[mixPath] = vtx
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public iterators # Public iterators
@ -226,12 +233,12 @@ iterator layersWalkVtx*(
## the one with a zero vertex which are othewise skipped by the iterator. ## the one with a zero vertex which are othewise skipped by the iterator.
## The `seen` argument must not be modified while the iterator is active. ## The `seen` argument must not be modified while the iterator is active.
## ##
for (rvid,vtx) in db.top.delta.sTab.pairs: for (rvid,vtx) in db.top.sTab.pairs:
yield (rvid,vtx) yield (rvid,vtx)
seen.incl rvid.vid seen.incl rvid.vid
for w in db.rstack: for w in db.rstack:
for (rvid,vtx) in w.delta.sTab.pairs: for (rvid,vtx) in w.sTab.pairs:
if rvid.vid notin seen: if rvid.vid notin seen:
yield (rvid,vtx) yield (rvid,vtx)
seen.incl rvid.vid seen.incl rvid.vid
@ -251,12 +258,12 @@ iterator layersWalkKey*(
## Walk over all `(VertexID,HashKey)` pairs on the cache layers. Note that ## Walk over all `(VertexID,HashKey)` pairs on the cache layers. Note that
## entries are unsorted. ## entries are unsorted.
var seen: HashSet[VertexID] var seen: HashSet[VertexID]
for (rvid,key) in db.top.delta.kMap.pairs: for (rvid,key) in db.top.kMap.pairs:
yield (rvid,key) yield (rvid,key)
seen.incl rvid.vid seen.incl rvid.vid
for w in db.rstack: for w in db.rstack:
for (rvid,key) in w.delta.kMap.pairs: for (rvid,key) in w.kMap.pairs:
if rvid.vid notin seen: if rvid.vid notin seen:
yield (rvid,key) yield (rvid,key)
seen.incl rvid.vid seen.incl rvid.vid

6
nimbus/db/aristo/aristo_tx.nim
View File

@ -134,7 +134,7 @@ proc findTx*(
if db.txRef.isNil: if db.txRef.isNil:
# Try `(vid,key)` on top layer # Try `(vid,key)` on top layer
let topKey = db.top.delta.kMap.getOrVoid rvid let topKey = db.top.kMap.getOrVoid rvid
if topKey == key: if topKey == key:
return ok(0) return ok(0)
@ -143,11 +143,11 @@ proc findTx*(
for (n,tx,layer,error) in db.txRef.txFrameWalk: for (n,tx,layer,error) in db.txRef.txFrameWalk:
if error != AristoError(0): if error != AristoError(0):
return err(error) return err(error)
if layer.delta.kMap.getOrVoid(rvid) == key: if layer.kMap.getOrVoid(rvid) == key:
return ok(n) return ok(n)
# Try bottom layer # Try bottom layer
let botKey = db.stack[0].delta.kMap.getOrVoid rvid let botKey = db.stack[0].kMap.getOrVoid rvid
if botKey == key: if botKey == key:
return ok(db.stack.len) return ok(db.stack.len)

3
nimbus/db/aristo/aristo_tx/tx_fork.nim
View File

@ -64,8 +64,7 @@ proc txFork*(
let stackLayer = block: let stackLayer = block:
let rc = db.getTuvBE() let rc = db.getTuvBE()
if rc.isOk: if rc.isOk:
LayerRef( LayerRef(vTop: rc.value)
delta: LayerDeltaRef(vTop: rc.value))
elif rc.error == GetTuvNotFound: elif rc.error == GetTuvNotFound:
LayerRef.init() LayerRef.init()
else: else:

20
nimbus/db/aristo/aristo_tx/tx_frame.nim
View File

@ -14,7 +14,6 @@
{.push raises: [].} {.push raises: [].}
import import
std/tables,
results, results,
".."/[aristo_desc, aristo_layers] ".."/[aristo_desc, aristo_layers]
@ -81,10 +80,10 @@ proc txFrameBegin*(db: AristoDbRef): Result[AristoTxRef,AristoError] =
if db.txFrameLevel != db.stack.len: if db.txFrameLevel != db.stack.len:
return err(TxStackGarbled) return err(TxStackGarbled)
let vTop = db.top.delta.vTop let vTop = db.top.vTop
db.stack.add db.top db.stack.add db.top
db.top = LayerRef( db.top = LayerRef(
delta: LayerDeltaRef(vTop: vTop), vTop: vTop,
txUid: db.getTxUid) txUid: db.getTxUid)
db.txRef = AristoTxRef( db.txRef = AristoTxRef(
@ -123,18 +122,11 @@ proc txFrameCommit*(
let db = ? tx.getDbDescFromTopTx() let db = ? tx.getDbDescFromTopTx()
# Pop layer from stack and merge database top layer onto it # Pop layer from stack and merge database top layer onto it
let merged = block: let merged = db.stack[^1]
if db.top.delta.sTab.len == 0 and
db.top.delta.kMap.len == 0:
# Avoid `layersMergeOnto()`
db.top.delta = db.stack[^1].delta
db.stack.setLen(db.stack.len-1) db.stack.setLen(db.stack.len-1)
db.top if not db.top.isEmpty():
else: # Only call `layersMergeOnto()` if layer is empty
let layer = db.stack[^1] db.top.layersMergeOnto merged[]
db.stack.setLen(db.stack.len-1)
db.top.layersMergeOnto layer[]
layer
# Install `merged` stack top layer and update stack # Install `merged` stack top layer and update stack
db.top = merged db.top = merged

43
nimbus/db/aristo/aristo_tx/tx_stow.nim
View File

@ -14,10 +14,25 @@
{.push raises: [].} {.push raises: [].}
import import
std/tables,
results, results,
../aristo_delta/delta_merge, ../aristo_delta/delta_merge,
".."/[aristo_desc, aristo_delta] ".."/[aristo_desc, aristo_delta, aristo_layers]
# ------------------------------------------------------------------------------
# Private functions
# ------------------------------------------------------------------------------
proc txStowOk*(
db: AristoDbRef; # Database
persistent: bool; # Stage only unless `true`
): Result[void,AristoError] =
if not db.txRef.isNil:
return err(TxPendingTx)
if 0 < db.stack.len:
return err(TxStackGarbled)
if persistent and not db.deltaPersistentOk():
return err(TxBackendNotWritable)
ok()
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public functions # Public functions
@ -30,24 +45,18 @@ proc txStow*(
): Result[void,AristoError] = ): Result[void,AristoError] =
## Worker for `stow()` and `persist()` variants. ## Worker for `stow()` and `persist()` variants.
## ##
if not db.txRef.isNil: ? db.txStowOk persistent
return err(TxPendingTx)
if 0 < db.stack.len:
return err(TxStackGarbled)
if persistent and not db.deltaPersistentOk():
return err(TxBackendNotWritable)
if db.top.delta.sTab.len != 0 or if not db.top.isEmpty():
db.top.delta.kMap.len != 0 or # Note that `deltaMerge()` will return the `db.top` argument if the
db.top.delta.accLeaves.len != 0 or # `db.balancer` is `nil`. Also, the `db.balancer` is read-only. In the
db.top.delta.stoLeaves.len != 0: # case that there are no forked peers one can ignore that restriction as
# no balancer is shared.
# Note that `deltaMerge()` will return the 1st argument if the 2nd is `nil` db.balancer = deltaMerge(
db.balancer = db.deltaMerge(db.top.delta, db.balancer).valueOr: db.top, modUpperOk = true, db.balancer, modLowerOk = db.nForked()==0)
return err(error[1])
# New empty top layer # New empty top layer
db.top = LayerRef(delta: LayerDeltaRef(vTop: db.balancer.vTop)) db.top = LayerRef(vTop: db.balancer.vTop)
if persistent: if persistent:
# Merge/move `balancer` into persistent tables (unless missing) # Merge/move `balancer` into persistent tables (unless missing)

13
nimbus/db/aristo/aristo_vid.nim
View File

@ -24,17 +24,16 @@ import
proc vidFetch*(db: AristoDbRef): VertexID = proc vidFetch*(db: AristoDbRef): VertexID =
## Fetch next vertex ID. ## Fetch next vertex ID.
## ##
if db.top.delta.vTop == 0: if db.top.vTop == 0:
db.top.delta.vTop = VertexID(LEAST_FREE_VID) db.top.vTop = VertexID(LEAST_FREE_VID)
else: else:
db.top.delta.vTop.inc db.top.vTop.inc
db.top.delta.vTop db.top.vTop
proc vidDispose*(db: AristoDbRef; vid: VertexID) = proc vidDispose*(db: AristoDbRef; vid: VertexID) =
## Only top vertexIDs are disposed ## Only top vertexIDs are disposed
if vid == db.top.delta.vTop and if vid == db.top.vTop and LEAST_FREE_VID < db.top.vTop.distinctBase:
LEAST_FREE_VID < db.top.delta.vTop.distinctBase: db.top.vTop.dec
db.top.delta.vTop.dec
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# End # End

8
nimbus/db/aristo/aristo_walk/walk_private.nim
View File

@ -23,12 +23,12 @@ iterator walkVtxBeImpl*[T](
): tuple[rvid: RootedVertexID, vtx: VertexRef] = ): tuple[rvid: RootedVertexID, vtx: VertexRef] =
## Generic iterator ## Generic iterator
when T is VoidBackendRef: when T is VoidBackendRef:
let filter = if db.balancer.isNil: LayerDeltaRef() else: db.balancer let filter = if db.balancer.isNil: LayerRef() else: db.balancer
else: else:
mixin walkVtx mixin walkVtx
let filter = LayerDeltaRef() let filter = LayerRef()
if not db.balancer.isNil: if not db.balancer.isNil:
filter.sTab = db.balancer.sTab # copy table filter.sTab = db.balancer.sTab # copy table
@ -52,12 +52,12 @@ iterator walkKeyBeImpl*[T](
): tuple[rvid: RootedVertexID, key: HashKey] = ): tuple[rvid: RootedVertexID, key: HashKey] =
## Generic iterator ## Generic iterator
when T is VoidBackendRef: when T is VoidBackendRef:
let filter = if db.balancer.isNil: LayerDeltaRef() else: db.balancer let filter = if db.balancer.isNil: LayerRef() else: db.balancer
else: else:
mixin walkKey mixin walkKey
let filter = LayerDeltaRef() let filter = LayerRef()
if not db.balancer.isNil: if not db.balancer.isNil:
filter.kMap = db.balancer.kMap # copy table filter.kMap = db.balancer.kMap # copy table

8
nimbus/db/kvt/kvt_debug.nim
View File

@ -1,5 +1,5 @@
# nimbus-eth1 # nimbus-eth1
# Copyright (c) 2023 Status Research & Development GmbH # Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of # Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0) # http://www.apache.org/licenses/LICENSE-2.0)
@ -134,11 +134,11 @@ proc ppBe[T](be: T; db: KvtDbRef; indent: int): string =
proc ppLayer(layer: LayerRef; db: KvtDbRef; indent = 4): string = proc ppLayer(layer: LayerRef; db: KvtDbRef; indent = 4): string =
let let
tLen = layer.delta.sTab.len tLen = layer.sTab.len
info = "tab(" & $tLen & ")" info = "tab(" & $tLen & ")"
pfx1 = indent.toPfx(1) pfx1 = indent.toPfx(1)
pfx2 = if 0 < tLen: indent.toPfx(2) else: " " pfx2 = if 0 < tLen: indent.toPfx(2) else: " "
"<layer>" & pfx1 & info & pfx2 & layer.delta.sTab.ppTab(db,indent+2) "<layer>" & pfx1 & info & pfx2 & layer.sTab.ppTab(db,indent+2)
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public functions # Public functions
@ -152,7 +152,7 @@ proc pp*(
case be.kind: case be.kind:
of BackendMemory: of BackendMemory:
result &= be.MemBackendRef.ppBe(db, indent) result &= be.MemBackendRef.ppBe(db, indent)
of BackendRocksDB: of BackendRocksDB,BackendRdbTriggered:
result &= be.RdbBackendRef.ppBe(db, indent) result &= be.RdbBackendRef.ppBe(db, indent)
of BackendVoid: of BackendVoid:
result &= "<NoBackend>" result &= "<NoBackend>"

38
nimbus/db/kvt/kvt_delta.nim
View File

@ -23,23 +23,12 @@ import
# Public functions # Public functions
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
proc deltaMerge*( proc deltaPersistentOk*(db: KvtDbRef): bool =
db: KvtDbRef; # Database
delta: LayerDeltaRef; # Filter to apply to database
) =
## Merge the argument `delta` into the balancer filter layer. Note that
## this function has no control of the filter source. Having merged the
## argument `delta`, all the `top` and `stack` layers should be cleared.
##
db.merge(delta, db.balancer)
proc deltaUpdateOk*(db: KvtDbRef): bool =
## Check whether the balancer filter can be merged into the backend ## Check whether the balancer filter can be merged into the backend
not db.backend.isNil and db.isCentre not db.backend.isNil and db.isCentre
proc deltaUpdate*( proc deltaPersistent*(
db: KvtDbRef; # Database db: KvtDbRef; # Database
reCentreOk = false; reCentreOk = false;
): Result[void,KvtError] = ): Result[void,KvtError] =
@ -71,17 +60,30 @@ proc deltaUpdate*(
# Always re-centre to `parent` (in case `reCentreOk` was set) # Always re-centre to `parent` (in case `reCentreOk` was set)
defer: discard parent.reCentre() defer: discard parent.reCentre()
# Update forked balancers here do that errors are detected early (if any.)
if 0 < db.nForked:
let rev = db.revFilter(db.balancer).valueOr:
return err(error[1])
if 0 < rev.sTab.len: # Can an empty `rev` happen at all?
var unsharedRevOk = true
for w in db.forked:
if not w.db.balancer.isValid:
unsharedRevOk = false
# The `rev` filter can be modified if one can make sure that it is
# not shared (i.e. only previously merged into the w.db.balancer.)
# Note that it is trivially true for a single fork.
let modLowerOk = w.isLast and unsharedRevOk
w.db.balancer = deltaMerge(
w.db.balancer, modUpperOk=false, rev, modLowerOk=modLowerOk)
# Store structural single trie entries # Store structural single trie entries
let writeBatch = ? be.putBegFn() let writeBatch = ? be.putBegFn()
be.putKvpFn(writeBatch, db.balancer.sTab.pairs.toSeq) be.putKvpFn(writeBatch, db.balancer.sTab.pairs.toSeq)
? be.putEndFn writeBatch ? be.putEndFn writeBatch
# Update peer filter balance. # Done with balancer, all saved to backend
let rev = db.deltaReverse db.balancer db.balancer = LayerRef(nil)
for w in db.forked:
db.merge(rev, w.balancer)
db.balancer = LayerDeltaRef(nil)
ok() ok()
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------

68
nimbus/db/kvt/kvt_delta/delta_merge.nim
View File

@ -10,37 +10,61 @@
import import
std/tables, std/tables,
results, eth/common,
".."/[kvt_desc, kvt_utils] ../kvt_desc
# ------------------------------------------------------------------------------
# Private functions
# ------------------------------------------------------------------------------
proc layersMergeOnto(src: LayerRef; trg: var LayerObj) =
for (key,val) in src.sTab.pairs:
trg.sTab[key] = val
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public functions # Public functions
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
proc merge*( proc deltaMerge*(
db: KvtDbRef; # Database upper: LayerRef; # Think of `top`, `nil` is ok
upper: LayerDeltaRef; # Filter to apply onto `lower` modUpperOk: bool; # May re-use/modify `upper`
lower: var LayerDeltaRef; # Target filter, will be modified lower: LayerRef; # Think of `balancer`, `nil` is ok
) = modLowerOk: bool; # May re-use/modify `lower`
## Merge the argument filter `upper` onto the argument filter `lower` ): LayerRef =
## relative to the *unfiltered* backend database on `db.backened`. The `lower` ## Merge argument `upper` into the `lower` filter instance.
## filter argument will have been modified.
## ##
## In case that the argument `lower` is `nil`, it will be replaced by `upper`. ## Note that the namimg `upper` and `lower` indicate that the filters are
## stacked and the database access is `upper -> lower -> backend`.
## ##
if lower.isNil: if lower.isNil:
lower = upper # Degenerate case: `upper` is void
elif not upper.isNil: result = upper
for (key,val) in upper.sTab.pairs:
if val.isValid or not lower.sTab.hasKey key: elif upper.isNil:
lower.sTab[key] = val # Degenerate case: `lower` is void
elif lower.sTab.getOrVoid(key).isValid: result = lower
let rc = db.getUbe key
if rc.isOk: elif modLowerOk:
lower.sTab[key] = val # empty blob # Can modify `lower` which is the prefered action mode but applies only
# in cases where the `lower` argument is not shared.
layersMergeOnto(upper, lower[])
result = lower
elif not modUpperOk:
# Cannot modify any argument layers.
result = LayerRef(sTab: lower.sTab)
layersMergeOnto(upper, result[])
else: else:
doAssert rc.error == GetNotFound # Otherwise avoid copying some tables by modifyinh `upper`. This is not
lower.sTab.del key # no need to keep that in merged filter # completely free as the merge direction changes to merging the `lower`
# layer up into the higher prioritised `upper` layer (note that the `lower`
# argument filter is read-only.) Here again, the `upper` argument must not
# be a shared layer/filter.
for (key,val) in lower.sTab.pairs:
if not upper.sTab.hasKey(key):
upper.sTab[key] = val
result = upper
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# End # End

31
nimbus/db/kvt/kvt_delta/delta_reverse.nim
View File

@ -10,6 +10,7 @@
import import
std/tables, std/tables,
eth/common,
results, results,
".."/[kvt_desc, kvt_utils] ".."/[kvt_desc, kvt_utils]
@ -17,27 +18,29 @@ import
# Public functions # Public functions
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
proc deltaReverse*( proc revFilter*(
db: KvtDbRef; # Database db: KvtDbRef; # Database
delta: LayerDeltaRef; # Filter to revert filter: LayerRef; # Filter to revert
): LayerDeltaRef = ): Result[LayerRef,(Blob,KvtError)] =
## Assemble a reverse filter for the `delta` argument, i.e. changes to the ## Assemble reverse filter for the `filter` argument, i.e. changes to the
## backend that reverse the effect of applying this to the balancer filter. ## backend that reverse the effect of applying the this read-only filter.
## The resulting filter is calculated against the current *unfiltered*
## backend (excluding optionally installed balancer filters.)
## ##
## If `delta` is `nil`, the result will be `nil` as well. ## This read-only filter is calculated against the current unfiltered
if not delta.isNil: ## backend (excluding optionally installed read-only filter.)
result = LayerDeltaRef() ##
let rev = LayerRef()
# Calculate reverse changes for the `sTab[]` structural table # Calculate reverse changes for the `sTab[]` structural table
for key in delta.sTab.keys: for key in filter.sTab.keys:
let rc = db.getUbe key let rc = db.getUbe key
if rc.isOk: if rc.isOk:
result.sTab[key] = rc.value rev.sTab[key] = rc.value
elif rc.error == GetNotFound:
rev.sTab[key] = EmptyBlob
else: else:
doAssert rc.error == GetNotFound return err((key,rc.error))
result.sTab[key] = EmptyBlob
ok(rev)
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# End # End

14
nimbus/db/kvt/kvt_desc.nim
View File

@ -46,7 +46,7 @@ type
## Three tier database object supporting distributed instances. ## Three tier database object supporting distributed instances.
top*: LayerRef ## Database working layer, mutable top*: LayerRef ## Database working layer, mutable
stack*: seq[LayerRef] ## Stashed immutable parent layers stack*: seq[LayerRef] ## Stashed immutable parent layers
balancer*: LayerDeltaRef ## Apply read filter (locks writing) balancer*: LayerRef ## Balance out concurrent backend access
backend*: BackendRef ## Backend database (may well be `nil`) backend*: BackendRef ## Backend database (may well be `nil`)
txRef*: KvtTxRef ## Latest active transaction txRef*: KvtTxRef ## Latest active transaction
@ -82,6 +82,9 @@ func getOrVoid*(tab: Table[Blob,Blob]; w: Blob): Blob =
func isValid*(key: Blob): bool = func isValid*(key: Blob): bool =
key != EmptyBlob key != EmptyBlob
func isValid*(layer: LayerRef): bool =
layer != LayerRef(nil)
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public functions, miscellaneous # Public functions, miscellaneous
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -162,13 +165,18 @@ proc fork*(
ok clone ok clone
iterator forked*(db: KvtDbRef): KvtDbRef = iterator forked*(db: KvtDbRef): tuple[db: KvtDbRef, isLast: bool] =
## Interate over all non centre descriptors (see comments on `reCentre()` ## Interate over all non centre descriptors (see comments on `reCentre()`
## for details.) ## for details.)
##
## The second `isLast` yielded loop entry is `true` if the yielded tuple
## is the last entry in the list.
if not db.dudes.isNil: if not db.dudes.isNil:
var nLeft = db.dudes.peers.len
for dude in db.dudes.peers.items: for dude in db.dudes.peers.items:
if dude != db.dudes.centre: if dude != db.dudes.centre:
yield dude nLeft.dec
yield (dude, nLeft == 1)
func nForked*(db: KvtDbRef): int = func nForked*(db: KvtDbRef): int =
## Returns the number of non centre descriptors (see comments on `reCentre()` ## Returns the number of non centre descriptors (see comments on `reCentre()`

15
nimbus/db/kvt/kvt_desc/desc_structural.nim
View File

@ -18,14 +18,11 @@ import
eth/common eth/common
type type
LayerDeltaRef* = ref object LayerRef* = ref LayerObj
## Delta tables relative to previous layer LayerObj* = object
sTab*: Table[Blob,Blob] ## Structural data table
LayerRef* = ref object
## Kvt database layer structures. Any layer holds the full ## Kvt database layer structures. Any layer holds the full
## change relative to the backend. ## change relative to the backend.
delta*: LayerDeltaRef ## Structural tables held as deltas sTab*: Table[Blob,Blob] ## Structural data table
txUid*: uint ## Transaction identifier if positive txUid*: uint ## Transaction identifier if positive
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -34,11 +31,11 @@ type
func init*(T: type LayerRef): T = func init*(T: type LayerRef): T =
## Constructor, returns empty layer ## Constructor, returns empty layer
T(delta: LayerDeltaRef()) T()
func dup*(ly: LayerDeltaRef): LayerDeltaRef = func dup*(ly: LayerRef): LayerRef =
## Duplicate/copy ## Duplicate/copy
LayerDeltaRef(sTab: ly.sTab) LayerRef(sTab: ly.sTab)
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# End # End

22
nimbus/db/kvt/kvt_layers.nim
View File

@ -25,7 +25,7 @@ func nLayersKeys*(db: KvtDbRef): int =
## bound for the number of effective key/value mappings held on the cache ## bound for the number of effective key/value mappings held on the cache
## layers as there might be duplicate entries for the same key on different ## layers as there might be duplicate entries for the same key on different
## layers. ## layers.
db.stack.mapIt(it.delta.sTab.len).foldl(a + b, db.top.delta.sTab.len) db.stack.mapIt(it.sTab.len).foldl(a + b, db.top.sTab.len)
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public functions: get function # Public functions: get function
@ -37,11 +37,11 @@ func layersLen*(db: KvtDbRef; key: openArray[byte]|seq[byte]): Opt[int] =
when key isnot seq[byte]: when key isnot seq[byte]:
let key = @key let key = @key
db.top.delta.sTab.withValue(key, item): db.top.sTab.withValue(key, item):
return Opt.some(item[].len()) return Opt.some(item[].len())
for w in db.rstack: for w in db.rstack:
w.delta.sTab.withValue(key, item): w.sTab.withValue(key, item):
return Opt.some(item[].len()) return Opt.some(item[].len())
Opt.none(int) Opt.none(int)
@ -58,11 +58,11 @@ func layersGet*(db: KvtDbRef; key: openArray[byte]|seq[byte]): Opt[Blob] =
when key isnot seq[byte]: when key isnot seq[byte]:
let key = @key let key = @key
db.top.delta.sTab.withValue(key, item): db.top.sTab.withValue(key, item):
return Opt.some(item[]) return Opt.some(item[])
for w in db.rstack: for w in db.rstack:
w.delta.sTab.withValue(key, item): w.sTab.withValue(key, item):
return Opt.some(item[]) return Opt.some(item[])
Opt.none(Blob) Opt.none(Blob)
@ -73,7 +73,7 @@ func layersGet*(db: KvtDbRef; key: openArray[byte]|seq[byte]): Opt[Blob] =
func layersPut*(db: KvtDbRef; key: openArray[byte]; data: openArray[byte]) = func layersPut*(db: KvtDbRef; key: openArray[byte]; data: openArray[byte]) =
## Store a (potentally empty) value on the top layer ## Store a (potentally empty) value on the top layer
db.top.delta.sTab[@key] = @data db.top.sTab[@key] = @data
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public functions # Public functions
@ -87,12 +87,12 @@ func layersCc*(db: KvtDbRef; level = high(int)): LayerRef =
else: db.stack[0 .. level] else: db.stack[0 .. level]
# Set up initial layer (bottom layer) # Set up initial layer (bottom layer)
result = LayerRef(delta: LayerDeltaRef(sTab: layers[0].delta.sTab)) result = LayerRef(sTab: layers[0].sTab)
# Consecutively merge other layers on top # Consecutively merge other layers on top
for n in 1 ..< layers.len: for n in 1 ..< layers.len:
for (key,val) in layers[n].delta.sTab.pairs: for (key,val) in layers[n].sTab.pairs:
result.delta.sTab[key] = val result.sTab[key] = val
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public iterators # Public iterators
@ -109,12 +109,12 @@ iterator layersWalk*(
## the one with a zero vertex which are othewise skipped by the iterator. ## the one with a zero vertex which are othewise skipped by the iterator.
## The `seen` argument must not be modified while the iterator is active. ## The `seen` argument must not be modified while the iterator is active.
## ##
for (key,val) in db.top.delta.sTab.pairs: for (key,val) in db.top.sTab.pairs:
yield (key,val) yield (key,val)
seen.incl key seen.incl key
for w in db.rstack: for w in db.rstack:
for (key,val) in w.delta.sTab.pairs: for (key,val) in w.sTab.pairs:
if key notin seen: if key notin seen:
yield (key,val) yield (key,val)
seen.incl key seen.incl key

5
nimbus/db/kvt/kvt_tx/tx_frame.nim
View File

@ -83,7 +83,6 @@ proc txFrameBegin*(db: KvtDbRef): Result[KvtTxRef,KvtError] =
db.stack.add db.top db.stack.add db.top
db.top = LayerRef( db.top = LayerRef(
delta: LayerDeltaRef(),
txUid: db.getTxUid) txUid: db.getTxUid)
db.txRef = KvtTxRef( db.txRef = KvtTxRef(
db: db, db: db,
@ -122,8 +121,8 @@ proc txFrameCommit*(
# Replace the top two layers by its merged version # Replace the top two layers by its merged version
let merged = db.stack[^1] let merged = db.stack[^1]
for (key,val) in db.top.delta.sTab.pairs: for (key,val) in db.top.sTab.pairs:
merged.delta.sTab[key] = val merged.sTab[key] = val
# Install `merged` layer # Install `merged` layer
db.top = merged db.top = merged

20
nimbus/db/kvt/kvt_tx/tx_stow.nim
View File

@ -16,6 +16,7 @@
import import
std/tables, std/tables,
results, results,
../kvt_delta/delta_merge,
".."/[kvt_desc, kvt_delta] ".."/[kvt_desc, kvt_delta]
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -31,10 +32,8 @@ proc txStowOk*(
return err(TxPendingTx) return err(TxPendingTx)
if 0 < db.stack.len: if 0 < db.stack.len:
return err(TxStackGarbled) return err(TxStackGarbled)
if persistent and not db.deltaPersistentOk():
if persistent and not db.deltaUpdateOk():
return err(TxBackendNotWritable) return err(TxBackendNotWritable)
ok() ok()
proc txStow*( proc txStow*(
@ -49,13 +48,20 @@ proc txStow*(
## ##
? db.txStowOk persistent ? db.txStowOk persistent
if 0 < db.top.delta.sTab.len: if 0 < db.top.sTab.len:
db.deltaMerge db.top.delta # Note that `deltaMerge()` will return the `db.top` argument if the
db.top.delta = LayerDeltaRef() # `db.balancer` is `nil`. Also, the `db.balancer` is read-only. In the
# case that there are no forked peers one can ignore that restriction as
# no balancer is shared.
db.balancer = deltaMerge(
db.top, modUpperOk = true, db.balancer, modLowerOk = db.nForked()==0)
# New empty top layer
db.top = LayerRef()
if persistent: if persistent:
# Move `balancer` data into persistent tables # Move `balancer` data into persistent tables
? db.deltaUpdate() ? db.deltaPersistent()
ok() ok()

14
tests/test_aristo/test_balancer.nim
View File

@ -149,7 +149,7 @@ proc cleanUp(dx: var DbTriplet) =
# ---------------------- # ----------------------
proc isDbEq(a, b: LayerDeltaRef; db: AristoDbRef; noisy = true): bool = proc isDbEq(a, b: LayerRef; db: AristoDbRef; noisy = true): bool =
## Verify that argument filter `a` has the same effect on the ## Verify that argument filter `a` has the same effect on the
## physical/unfiltered backend of `db` as argument filter `b`. ## physical/unfiltered backend of `db` as argument filter `b`.
if a.isNil: if a.isNil:
@ -255,8 +255,8 @@ proc testBalancer*(
# Resulting clause (11) filters from `aristo/README.md` example # Resulting clause (11) filters from `aristo/README.md` example
# which will be used in the second part of the tests # which will be used in the second part of the tests
var var
c11Filter1 = LayerDeltaRef(nil) c11Filter1 = LayerRef(nil)
c11Filter3 = LayerDeltaRef(nil) c11Filter3 = LayerRef(nil)
# Work through clauses (8)..(11) from `aristo/README.md` example # Work through clauses (8)..(11) from `aristo/README.md` example
block: block:
@ -278,14 +278,14 @@ proc testBalancer*(
block: block:
let rc = db1.persist() let rc = db1.persist()
xCheckRc rc.error == 0 xCheckRc rc.error == 0
xCheck db1.balancer == LayerDeltaRef(nil) xCheck db1.balancer == LayerRef(nil)
xCheck db2.balancer == db3.balancer xCheck db2.balancer == db3.balancer
block: block:
let rc = db2.stow() # non-persistent let rc = db2.stow() # non-persistent
xCheckRc rc.error == 0: xCheckRc rc.error == 0:
noisy.say "*** testDistributedAccess (3)", "n=", n, "db2".dump db2 noisy.say "*** testDistributedAccess (3)", "n=", n, "db2".dump db2
xCheck db1.balancer == LayerDeltaRef(nil) xCheck db1.balancer == LayerRef(nil)
xCheck db2.balancer != db3.balancer xCheck db2.balancer != db3.balancer
# Clause (11) from `aristo/README.md` example # Clause (11) from `aristo/README.md` example
@ -293,7 +293,7 @@ proc testBalancer*(
block: block:
let rc = db2.persist() let rc = db2.persist()
xCheckRc rc.error == 0 xCheckRc rc.error == 0
xCheck db2.balancer == LayerDeltaRef(nil) xCheck db2.balancer == LayerRef(nil)
# Check/verify backends # Check/verify backends
block: block:
@ -326,7 +326,7 @@ proc testBalancer*(
block: block:
let rc = db2.persist() let rc = db2.persist()
xCheckRc rc.error == 0 xCheckRc rc.error == 0
xCheck db2.balancer == LayerDeltaRef(nil) xCheck db2.balancer == LayerRef(nil)
xCheck db1.balancer == db3.balancer xCheck db1.balancer == db3.balancer
# Clause (13) from `aristo/README.md` example # Clause (13) from `aristo/README.md` example