status-im/nimbus-eth1
2
0
Fork 0
mirror of https://github.com/status-im/nimbus-eth1.git synced 2025-02-28 11:50:45 +00:00
Code Issues Projects Releases Wiki Activity

Merge AristoTxRef/KvtTxRef with their LayerRef (#3078)

Browse Source 
* Merge AristoTxRef/KvtTxRef with their LayerRef

also clean up unused files

* rm unused test_rocksdb

* Fix tests

* lint
This commit is contained in:
andri lim 2025-02-17 10:30:43 +07:00 committed by GitHub
parent caca11b30b
commit a1a9c6b027
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
44 changed files with 160 additions and 565 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@ -241,7 +241,6 @@ endif
# builds and runs the nimbus test suite
test: | build deps rocksdb
$(ENV_SCRIPT) nim test_rocksdb $(NIM_PARAMS) nimbus.nims
$(ENV_SCRIPT) nim test $(NIM_PARAMS) nimbus.nims
test_import: nimbus_execution_client

6
execution_chain/db/aristo.nim
View File

@ -1,5 +1,5 @@
# nimbus-eth1
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Copyright (c) 2023-2025 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
@ -19,10 +19,10 @@ export
aristo_api, aristo_constants
import
aristo/aristo_init
aristo/aristo_init/memory_only,
aristo/aristo_init/init_common
export
MemBackendRef,
VoidBackendRef,
finish,
init

7
execution_chain/db/aristo/aristo_api.nim
View File

@ -18,7 +18,9 @@ import
results,
./aristo_desc/desc_backend,
./aristo_init/memory_db,
"."/[aristo_delete, aristo_desc, aristo_fetch, aristo_init, aristo_merge,
./aristo_init/memory_only,
./aristo_init/init_common,
"."/[aristo_delete, aristo_desc, aristo_fetch, aristo_merge,
aristo_part, aristo_path, aristo_persist, aristo_profile, aristo_tx_frame]
export
@ -379,9 +381,6 @@ proc dup(be: BackendRef): BackendRef =
when AristoPersistentBackendOk:
return RdbBackendRef(be).dup
of BackendVoid:
discard
# ------------------------------------------------------------------------------
# Public API constuctors
# ------------------------------------------------------------------------------

5
execution_chain/db/aristo/aristo_check.nim
View File

@ -17,7 +17,7 @@ import
eth/common/hashes,
results,
./aristo_walk/persistent,
"."/[aristo_desc, aristo_get, aristo_init],
"."/[aristo_desc, aristo_get, aristo_init/memory_only],
./aristo_check/[check_be, check_top, check_twig]
# ------------------------------------------------------------------------------
@ -71,9 +71,6 @@ proc checkBE*(
return MemBackendRef.checkBE db
of BackendRocksDB:
return RdbBackendRef.checkBE db
of BackendVoid:
return VoidBackendRef.checkBE db
proc check*(
db: AristoTxRef; # Database

4
execution_chain/db/aristo/aristo_check/check_be.nim
View File

@ -17,13 +17,13 @@ import
stew/interval_set,
../../aristo,
../aristo_walk/persistent,
".."/[aristo_desc, aristo_get, aristo_layers]
".."/[aristo_desc, aristo_get]
# ------------------------------------------------------------------------------
# Public functions
# ------------------------------------------------------------------------------
proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
proc checkBE*[T: RdbBackendRef|MemBackendRef](
_: type T;
db: AristoDbRef; # Database, top layer
): Result[void,(VertexID,AristoError)] =

2
execution_chain/db/aristo/aristo_check/check_top.nim
View File

@ -75,7 +75,7 @@ proc checkTopCommon*(
let
kMapCount = db.layersWalkKey.toSeq.mapIt(it[1]).filterIt(it.isValid).len
kMapNilCount = db.layersWalkKey.toSeq.len - kMapCount
vTop = db.layer.vTop
vTop = db.vTop
var
topVid = VertexID(0)
stoRoots: HashSet[VertexID]

12
execution_chain/db/aristo/aristo_debug.nim
View File

@ -25,7 +25,7 @@ import
# ------------------------------------------------------------------------------
func orDefault(db: AristoTxRef): AristoTxRef =
if db.isNil: AristoTxRef(layer: LayerRef()) else: db
if db.isNil: AristoTxRef() else: db
# --------------------------
@ -359,7 +359,7 @@ proc ppXMap*(
proc ppBalancer(
fl: LayerRef;
fl: AristoTxRef;
db: AristoTxRef;
indent: int;
): string =
@ -434,7 +434,7 @@ proc ppBe[T](be: T; db: AristoTxRef; limit: int; indent: int): string =
result &= "[]"
proc ppLayer(
layer: LayerRef;
layer: AristoTxRef;
db: AristoTxRef;
vTopOk: bool;
sTabOk: bool;
@ -663,7 +663,7 @@ func pp*(wp: VidVtxPair; db: AristoTxRef): string =
proc pp*(
layer: LayerRef;
layer: AristoTxRef;
db: AristoTxRef;
indent = 4;
sTabOk = true,
@ -679,14 +679,12 @@ proc pp*(
limit = 100;
indent = 4;
): string =
result = db.layer.ppBalancer(db, indent+1) & indent.toPfx
result = db.ppBalancer(db, indent+1) & indent.toPfx
case be.kind:
of BackendMemory:
result &= be.MemBackendRef.ppBe(db, limit, indent+1)
of BackendRocksDB:
result &= be.RdbBackendRef.ppBe(db, limit, indent+1)
of BackendVoid:
result &= "<NoBackend>"
proc pp*(
db: AristoTxRef;

46
execution_chain/db/aristo/aristo_desc.nim
View File

@ -41,9 +41,39 @@ export
type
AristoTxRef* = ref object
## Transaction descriptor
##
## Delta layers are stacked implying a tables hierarchy. Table entries on
## a higher level take precedence over lower layer table entries. So an
## existing key-value table entry of a layer on top supersedes same key
## entries on all lower layers. A missing entry on a higher layer indicates
## that the key-value pair might be fond on some lower layer.
##
## A zero value (`nil`, empty hash etc.) is considered am missing key-value
## pair. Tables on the `LayerDelta` may have stray zero key-value pairs for
## missing entries due to repeated transactions while adding and deleting
## entries. There is no need to purge redundant zero entries.
##
## As for `kMap[]` entries, there might be a zero value entriy relating
## (i.e. indexed by the same vertex ID) to an `sMap[]` non-zero value entry
## (of the same layer or a lower layer whatever comes first.) This entry
## is kept as a reminder that the hash value of the `kMap[]` entry needs
## to be re-compiled.
##
## The reasoning behind the above scenario is that every vertex held on the
## `sTab[]` tables must correspond to a hash entry held on the `kMap[]`
## tables. So a corresponding zero value or missing entry produces an
## inconsistent state that must be resolved.
db*: AristoDbRef ## Database descriptor
parent*: AristoTxRef ## Previous transaction
layer*: LayerRef
sTab*: Table[RootedVertexID,VertexRef] ## Structural vertex table
kMap*: Table[RootedVertexID,HashKey] ## Merkle hash key mapping
vTop*: VertexID ## Last used vertex ID
accLeaves*: Table[Hash32, VertexRef] ## Account path -> VertexRef
stoLeaves*: Table[Hash32, VertexRef] ## Storage path -> VertexRef
cTop*: VertexID ## Last committed vertex ID
blockNumber*: Opt[uint64] ## Block number set when freezing the frame
AristoDbRef* = ref object
@ -120,8 +150,8 @@ func isValid*(nd: NodeRef): bool =
func isValid*(pid: PathID): bool =
pid != VOID_PATH_ID
func isValid*(layer: LayerRef): bool =
layer != LayerRef(nil)
func isValid*(tx: AristoTxRef): bool =
tx != AristoTxRef(nil)
func isValid*(root: Hash32): bool =
root != emptyRoot
@ -163,21 +193,21 @@ iterator stack*(tx: AristoTxRef): AristoTxRef =
while frames.len > 0:
yield frames.pop()
iterator rstack*(tx: AristoTxRef): (LayerRef, int) =
iterator rstack*(tx: AristoTxRef): (AristoTxRef, int) =
# Stack in reverse order, ie going from tx to base
var tx = tx
var i = 0
while tx != nil:
let level = if tx.parent == nil: -1 else: i
yield (tx.layer, level)
yield (tx, level)
tx = tx.parent
proc deltaAtLevel*(db: AristoTxRef, level: int): LayerRef =
proc deltaAtLevel*(db: AristoTxRef, level: int): AristoTxRef =
if level == -2:
nil
elif level == -1:
db.db.txRef.layer
db.db.txRef
else:
var
frame = db
@ -187,7 +217,7 @@ proc deltaAtLevel*(db: AristoTxRef, level: int): LayerRef =
frame = frame.parent
level -= 1
frame.layer
frame
# ------------------------------------------------------------------------------
# End

23
execution_chain/db/aristo/aristo_desc/desc_error.nim
View File

@ -12,17 +12,6 @@ type
AristoError* = enum
NothingSerious = 0
# Miscelaneous/unclassified handy helpers
GenericError
# Data record transcoders, `blobify()` from `blobify.nim`
BlobifyBranchMissingRefs
BlobifyExtMissingRefs
BlobifyExtPathOverflow
BlobifyLeafPathOverflow
BlobifyNilVertex
# Cache checker `checkCache()`
CheckAnyVidDeadStorageRoot
@ -79,17 +68,6 @@ type
DelStoRootNotAccepted
DelVidStaleVtx
# Functions from `aristo_desc.nim`
DescMustBeOnCentre
DescNotAllowedOnCentre
DescStaleDescriptor
# Functions from `aristo_delta.nim`
FilBackendRoMode
FilSiblingsCommitUnfinshed
# Fetch functions from `aristo_fetch.nim`
FetchAccInaccessible
FetchAccPathWithoutLeaf
@ -205,7 +183,6 @@ type
PartVtxSlotWasNotModified
# RocksDB backend
RdbBeCantCreateDataDir
RdbBeCantCreateTmpDir
RdbBeDriverDelAdmError
RdbBeDriverDelKeyError

31
execution_chain/db/aristo/aristo_desc/desc_structural.nim
View File

@ -94,37 +94,6 @@ type
key*: Hash32 ## Some state hash (if any)
serial*: uint64 ## Generic identifier from application
LayerRef* = ref Layer
Layer* = object
## Delta layers are stacked implying a tables hierarchy. Table entries on
## a higher level take precedence over lower layer table entries. So an
## existing key-value table entry of a layer on top supersedes same key
## entries on all lower layers. A missing entry on a higher layer indicates
## that the key-value pair might be fond on some lower layer.
##
## A zero value (`nil`, empty hash etc.) is considered am missing key-value
## pair. Tables on the `LayerDelta` may have stray zero key-value pairs for
## missing entries due to repeated transactions while adding and deleting
## entries. There is no need to purge redundant zero entries.
##
## As for `kMap[]` entries, there might be a zero value entriy relating
## (i.e. indexed by the same vertex ID) to an `sMap[]` non-zero value entry
## (of the same layer or a lower layer whatever comes first.) This entry
## is kept as a reminder that the hash value of the `kMap[]` entry needs
## to be re-compiled.
##
## The reasoning behind the above scenario is that every vertex held on the
## `sTab[]` tables must correspond to a hash entry held on the `kMap[]`
## tables. So a corresponding zero value or missing entry produces an
## inconsistent state that must be resolved.
##
sTab*: Table[RootedVertexID,VertexRef] ## Structural vertex table
kMap*: Table[RootedVertexID,HashKey] ## Merkle hash key mapping
vTop*: VertexID ## Last used vertex ID
accLeaves*: Table[Hash32, VertexRef] ## Account path -> VertexRef
stoLeaves*: Table[Hash32, VertexRef] ## Storage path -> VertexRef
GetVtxFlag* = enum
PeekCache
## Peek into, but don't update cache - useful on work loads that are

3
execution_chain/db/aristo/aristo_init/init_common.nim
View File

@ -19,7 +19,6 @@ const
type
BackendType* = enum
BackendVoid = 0 ## For providing backend-less constructor
BackendMemory
BackendRocksDB
@ -100,7 +99,7 @@ proc init*(
let
vTop = if backend == nil: VertexID(0) else: ?backend.getTuvFn()
db = AristoDbRef(
txRef: AristoTxRef(layer: LayerRef(vTop: vTop)),
txRef: AristoTxRef(vTop: vTop),
backend: backend,
accLeaves: LruCache[Hash32, VertexRef].init(ACC_LRU_SIZE),
stoLeaves: LruCache[Hash32, VertexRef].init(ACC_LRU_SIZE),

1
execution_chain/db/aristo/aristo_init/memory_db.nim
View File

@ -15,7 +15,6 @@
## backend access
## ::
## import
## aristo/aristo_init,
## aristo/aristo_init/aristo_memory
##
## let rc = newAristoDbRef(BackendMemory)

27
execution_chain/db/aristo/aristo_init/memory_only.nim
View File

@ -18,12 +18,6 @@ import
../aristo_desc/desc_backend,
"."/[init_common, memory_db]
type
VoidBackendRef* = ref object of TypedBackendRef
## Dummy descriptor type, used as `nil` reference
MemOnlyBackend* = VoidBackendRef|MemBackendRef
export
BackendType,
GuestDbRef,
@ -37,10 +31,7 @@ proc kind*(
be: BackendRef;
): BackendType =
## Retrieves the backend type symbol for a `be` backend database argument
## where `BackendVoid` is returned for the`nil` backend.
if be.isNil:
BackendVoid
else:
doAssert(not be.isNil)
be.TypedBackendRef.beKind
# ------------------------------------------------------------------------------
@ -49,24 +40,14 @@ proc kind*(
proc init*(
T: type AristoDbRef; # Target type
B: type MemOnlyBackend; # Backend type
B: type MemBackendRef; # Backend type
): T =
## Memory backend constructor.
##
when B is VoidBackendRef:
AristoDbRef.init(nil)[]
elif B is MemBackendRef:
AristoDbRef.init(memoryBackend())[]
else:
raiseAssert "Unknown backend"
proc init*(
T: type AristoDbRef; # Target type
): T =
## Shortcut for `AristoDbRef.init(VoidBackendRef)`
AristoDbRef.init VoidBackendRef
proc init*(T: type AristoDbRef): T =
AristoDbRef.init(MemBackendRef)
# ------------------------------------------------------------------------------
# End

16
execution_chain/db/aristo/aristo_layers.nim
View File

@ -72,8 +72,8 @@ func layersPutVtx*(
vtx: VertexRef;
) =
## Store a (potentally empty) vertex on the top layer
db.layer.sTab[rvid] = vtx
db.layer.kMap.del(rvid)
db.sTab[rvid] = vtx
db.kMap.del(rvid)
func layersResVtx*(
db: AristoTxRef;
@ -90,8 +90,8 @@ func layersPutKey*(
key: HashKey;
) =
## Store a (potentally void) hash key on the top layer
db.layer.sTab[rvid] = vtx
db.layer.kMap[rvid] = key
db.sTab[rvid] = vtx
db.kMap[rvid] = key
func layersResKey*(db: AristoTxRef; rvid: RootedVertexID, vtx: VertexRef) =
## Shortcut for `db.layersPutKey(vid, VOID_HASH_KEY)`. It is sort of the
@ -104,16 +104,16 @@ func layersResKeys*(db: AristoTxRef; hike: Hike) =
db.layersResKey((hike.root, hike.legs[^i].wp.vid), hike.legs[^i].wp.vtx)
func layersPutAccLeaf*(db: AristoTxRef; accPath: Hash32; leafVtx: VertexRef) =
db.layer.accLeaves[accPath] = leafVtx
db.accLeaves[accPath] = leafVtx
func layersPutStoLeaf*(db: AristoTxRef; mixPath: Hash32; leafVtx: VertexRef) =
db.layer.stoLeaves[mixPath] = leafVtx
db.stoLeaves[mixPath] = leafVtx
# ------------------------------------------------------------------------------
# Public functions
# ------------------------------------------------------------------------------
func isEmpty*(ly: LayerRef): bool =
func isEmpty*(ly: AristoTxRef): bool =
## Returns `true` if the layer does not contain any changes, i.e. all the
## tables are empty.
ly.sTab.len == 0 and
@ -121,7 +121,7 @@ func isEmpty*(ly: LayerRef): bool =
ly.accLeaves.len == 0 and
ly.stoLeaves.len == 0
proc mergeAndReset*(trg, src: var Layer) =
proc mergeAndReset*(trg, src: AristoTxRef) =
## Merges the argument `src` into the argument `trg` and clears `src`.
trg.vTop = src.vTop

4
execution_chain/db/aristo/aristo_profile.nim
View File

@ -1,5 +1,5 @@
# Nimbus
# Copyright (c) 2024 Status Research & Development GmbH
# Copyright (c) 2024-2025 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
@ -11,7 +11,7 @@
{.push raises: [].}
import
std/[algorithm, math, sequtils, strformat, strutils, tables, times],
std/[algorithm, math, sequtils, strformat, tables, times],
eth/common
type

28
execution_chain/db/aristo/aristo_tx_frame.nim
View File

@ -27,14 +27,10 @@ proc txFrameBegin*(db: AristoDbRef, parent: AristoTxRef): AristoTxRef =
else:
parent
let
vTop = parent.layer.vTop
layer = LayerRef(vTop: vTop)
AristoTxRef(
db: db,
parent: parent,
layer: layer)
vTop: parent.vTop)
proc baseTxFrame*(db: AristoDbRef): AristoTxRef=
db.txRef
@ -56,7 +52,7 @@ proc txFramePersist*(
txFrame: AristoTxRef;
) =
if txFrame == db.txRef and txFrame.layer.sTab.len == 0:
if txFrame == db.txRef and txFrame.sTab.len == 0:
# No changes in frame - no `checkpoint` requirement - nothing to do here
return
@ -76,7 +72,7 @@ proc txFramePersist*(
for frame in txFrame.stack():
if frame == db.txRef:
continue
mergeAndReset(db.txRef.layer[], frame.layer[])
mergeAndReset(db.txRef, frame)
db.txRef.blockNumber = frame.blockNumber
frame.dispose() # This will also dispose `txFrame` itself!
@ -86,13 +82,13 @@ proc txFramePersist*(
db.txRef = txFrame
# Store structural single trie entries
for rvid, vtx in txFrame.layer.sTab:
txFrame.layer.kMap.withValue(rvid, key) do:
for rvid, vtx in db.txRef.sTab:
txFrame.kMap.withValue(rvid, key) do:
be.putVtxFn(batch, rvid, vtx, key[])
do:
be.putVtxFn(batch, rvid, vtx, default(HashKey))
be.putTuvFn(batch, txFrame.layer.vTop)
be.putTuvFn(batch, txFrame.vTop)
be.putLstFn(batch, lSst)
# TODO above, we only prepare the changes to the database but don't actually
@ -101,16 +97,16 @@ proc txFramePersist*(
# in-memory and on-disk state)
# Copy back updated payloads
for accPath, vtx in txFrame.layer.accLeaves:
for accPath, vtx in txFrame.accLeaves:
db.accLeaves.put(accPath, vtx)
for mixPath, vtx in txFrame.layer.stoLeaves:
for mixPath, vtx in txFrame.stoLeaves:
db.stoLeaves.put(mixPath, vtx)
txFrame.layer.sTab.clear()
txFrame.layer.kMap.clear()
txFrame.layer.accLeaves.clear()
txFrame.layer.stoLeaves.clear()
txFrame.sTab.clear()
txFrame.kMap.clear()
txFrame.accLeaves.clear()
txFrame.stoLeaves.clear()
# ------------------------------------------------------------------------------
# End

8
execution_chain/db/aristo/aristo_vid.nim
View File

@ -23,11 +23,11 @@ import
proc vidFetch*(db: AristoTxRef, n = 1): VertexID =
## Fetch next vertex ID.
##
if db.layer.vTop == 0:
db.layer.vTop = VertexID(LEAST_FREE_VID)
var ret = db.layer.vTop
if db.vTop == 0:
db.vTop = VertexID(LEAST_FREE_VID)
var ret = db.vTop
ret.inc
db.layer.vTop.inc(n)
db.vTop.inc(n)
ret
# ------------------------------------------------------------------------------

10
execution_chain/db/aristo/aristo_walk/memory_only.nim
View File

@ -1,6 +1,6 @@
# Nimbus - Types, data structures and shared utilities used in network sync
#
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Copyright (c) 2023-2025 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
@ -14,7 +14,7 @@
##
import
../aristo_init/[memory_db, memory_only],
".."/[aristo_desc, aristo_init],
".."/[aristo_desc],
./walk_private
export
@ -26,7 +26,7 @@ export
# Public iterators (all in one)
# ------------------------------------------------------------------------------
iterator walkVtxBe*[T: MemBackendRef|VoidBackendRef](
iterator walkVtxBe*[T: MemBackendRef](
_: type T;
db: AristoDbRef;
kinds = {Branch, Leaf};
@ -37,7 +37,7 @@ iterator walkVtxBe*[T: MemBackendRef|VoidBackendRef](
for (rvid,vtx) in walkVtxBeImpl[T](db, kinds):
yield (rvid,vtx)
iterator walkKeyBe*[T: MemBackendRef|VoidBackendRef](
iterator walkKeyBe*[T: MemBackendRef](
_: type T;
db: AristoDbRef;
): tuple[rvid: RootedVertexID, key: HashKey] =
@ -47,7 +47,7 @@ iterator walkKeyBe*[T: MemBackendRef|VoidBackendRef](
# -----------
iterator walkPairs*[T: MemBackendRef|VoidBackendRef](
iterator walkPairs*[T: MemBackendRef](
_: type T;
db: AristoDbRef;
): tuple[rvid: RootedVertexID, vtx: VertexRef] =

18
execution_chain/db/aristo/aristo_walk/walk_private.nim
View File

@ -12,7 +12,7 @@
import
std/[algorithm, sequtils, sets, tables],
results,
".."/[aristo_desc, aristo_init, aristo_layers]
".."/[aristo_desc, aristo_layers]
# ------------------------------------------------------------------------------
# Public generic iterators
@ -23,15 +23,11 @@ iterator walkVtxBeImpl*[T](
kinds: set[VertexType];
): tuple[rvid: RootedVertexID, vtx: VertexRef] =
## Generic iterator
when T is VoidBackendRef:
let filter = if db.txRef.isNil: LayerRef() else: db.txRef.layer
else:
mixin walkVtx
let filter = LayerRef()
let filter = AristoTxRef()
if not db.txRef.isNil:
filter.sTab = db.txRef.layer.sTab # copy table
filter.sTab = db.txRef.sTab # copy table
for (rvid,vtx) in db.backend.T.walkVtx(kinds):
if filter.sTab.hasKey rvid:
@ -54,15 +50,11 @@ iterator walkKeyBeImpl*[T](
db: AristoDbRef; # Database with optional backend filter
): tuple[rvid: RootedVertexID, key: HashKey] =
## Generic iterator
when T is VoidBackendRef:
let filter = if db.txRef.isNil: LayerRef() else: db.txRef.layer
else:
mixin walkKey
let filter = LayerRef()
let filter = AristoTxRef()
if not db.txRef.isNil:
filter.kMap = db.txRef.layer.kMap # copy table
filter.kMap = db.txRef.kMap # copy table
for (rvid,key) in db.backend.T.walkKey:
if filter.kMap.hasKey rvid:

5
execution_chain/db/core_db/backend/aristo_db.nim
View File

@ -48,11 +48,6 @@ proc newMemoryCoreDbRef*(): CoreDbRef =
KvtDbRef.init(use_kvt.MemBackendRef),
AristoDbRef.init(use_ari.MemBackendRef))
proc newVoidCoreDbRef*(): CoreDbRef =
AristoDbVoid.create(
KvtDbRef.init(use_kvt.VoidBackendRef),
AristoDbRef.init(use_ari.VoidBackendRef))
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

3
execution_chain/db/core_db/base/base_desc.nim
View File

@ -1,5 +1,5 @@
# Nimbus
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Copyright (c) 2023-2025 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
@ -20,7 +20,6 @@ type
Ooops
AristoDbMemory ## Memory backend emulator
AristoDbRocks ## RocksDB backend
AristoDbVoid ## No backend
const
CoreDbPersistentTypes* = {AristoDbRocks}

3
execution_chain/db/core_db/memory_only.nim
View File

@ -41,9 +41,6 @@ proc newCoreDbRef*(
when dbType == AristoDbMemory:
newMemoryCoreDbRef()
elif dbType == AristoDbVoid:
newVoidCoreDbRef()
else:
{.error: "Unsupported constructor " & $dbType & ".newCoreDbRef()".}

138
execution_chain/db/kvstore_rocksdb.nim
View File

@ -1,138 +0,0 @@
# Nimbus
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
{.push raises: [].}
import
std/[os, sequtils],
results,
rocksdb,
eth/db/kvstore
export kvstore
type
RocksStoreRef* = ref object of RootObj
db: RocksDbReadWriteRef
RocksNamespaceRef* = ref object of RootObj
colFamily: ColFamilyReadWrite
# ------------------------------------------------------------------------------
# RocksStoreRef procs
# ------------------------------------------------------------------------------
proc rocksDb*(store: RocksStoreRef): RocksDbReadWriteRef =
store.db
proc get*(
store: RocksStoreRef,
key: openArray[byte],
onData: kvstore.DataProc): KvResult[bool] =
store.db.get(key, onData)
proc find*(
store: RocksStoreRef,
prefix: openArray[byte],
onFind: kvstore.KeyValueProc): KvResult[int] =
raiseAssert "Unimplemented"
proc put*(store: RocksStoreRef, key, value: openArray[byte]): KvResult[void] =
store.db.put(key, value)
proc contains*(store: RocksStoreRef, key: openArray[byte]): KvResult[bool] =
store.db.keyExists(key)
proc del*(store: RocksStoreRef, key: openArray[byte]): KvResult[bool] =
let rc = store.db.keyExists(key)
if rc.isErr:
return rc
if not rc.value:
return ok(false)
let res = store.db.delete(key)
if res.isErr():
return err(res.error())
ok(true)
proc clear*(store: RocksStoreRef): KvResult[bool] =
raiseAssert "Unimplemented"
proc close*(store: RocksStoreRef) =
store.db.close()
proc init*(
T: type RocksStoreRef,
basePath: string,
name: string,
namespaces = @["default"]): KvResult[T] =
let dataDir = basePath / name / "data"
try:
createDir(dataDir)
except OSError, IOError:
return err("RocksStoreRef: cannot create database directory")
let db = ? openRocksDb(dataDir, columnFamilies = namespaces.mapIt(
initColFamilyDescriptor(it, defaultColFamilyOptions(autoClose = true))))
ok(T(db: db))
# ------------------------------------------------------------------------------
# RocksNamespaceRef procs
# ------------------------------------------------------------------------------
proc name*(store: RocksNamespaceRef): string =
store.colFamily.name
proc get*(
ns: RocksNamespaceRef,
key: openArray[byte],
onData: kvstore.DataProc): KvResult[bool] =
ns.colFamily.get(key, onData)
proc find*(
ns: RocksNamespaceRef,
prefix: openArray[byte],
onFind: kvstore.KeyValueProc): KvResult[int] =
raiseAssert "Unimplemented"
proc put*(ns: RocksNamespaceRef, key, value: openArray[byte]): KvResult[void] =
ns.colFamily.put(key, value)
proc contains*(ns: RocksNamespaceRef, key: openArray[byte]): KvResult[bool] =
ns.colFamily.keyExists(key)
proc del*(ns: RocksNamespaceRef, key: openArray[byte]): KvResult[bool] =
let exists = ? ns.colFamily.keyExists(key)
if not exists:
return ok(false)
let res = ns.colFamily.delete(key)
if res.isErr():
return err(res.error())
ok(true)
proc clear*(ns: RocksNamespaceRef): KvResult[bool] =
raiseAssert "Unimplemented"
proc close*(ns: RocksNamespaceRef) =
# To close the database, call close on RocksStoreRef.
raiseAssert "Unimplemented"
proc openNamespace*(
store: RocksStoreRef,
name: string): KvResult[RocksNamespaceRef] =
doAssert not store.db.isClosed()
ok(RocksNamespaceRef(colFamily: ?store.db.getColFamily(name)))

5
execution_chain/db/kvt.nim
View File

@ -1,5 +1,5 @@
# nimbus-eth1
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Copyright (c) 2023-2025 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
@ -19,10 +19,9 @@ export
kvt_api, kvt_constants
import
kvt/kvt_init
kvt/kvt_init/memory_only
export
MemBackendRef,
VoidBackendRef,
finish,
init

8
execution_chain/db/kvt/kvt_api.nim
View File

@ -17,7 +17,8 @@ import
../aristo/aristo_profile,
./kvt_desc/desc_backend,
./kvt_init/memory_db,
"."/[kvt_desc, kvt_init, kvt_persist, kvt_tx_frame, kvt_utils]
./kvt_init/memory_only,
"."/[kvt_desc, kvt_persist, kvt_tx_frame, kvt_utils]
const
AutoValidateApiHooks = defined(release).not
@ -117,13 +118,10 @@ proc dup(be: BackendRef): BackendRef =
of BackendMemory:
return MemBackendRef(be).dup
of BackendRocksDB, BackendRdbTriggered:
of BackendRocksDB:
when KvtPersistentBackendOk:
return RdbBackendRef(be).dup
of BackendVoid:
discard
# ------------------------------------------------------------------------------
# Public API constuctors
# ------------------------------------------------------------------------------

8
execution_chain/db/kvt/kvt_debug.nim
View File

@ -1,5 +1,5 @@
# nimbus-eth1
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Copyright (c) 2023-2025 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
@ -131,7 +131,7 @@ proc ppBe[T](be: T; db: KvtDbRef; indent: int): string =
spc = if 0 < n: pfx2 else: " "
"<" & $be.kind & ">" & pfx1 & "tab" & spc & "{" & data & "}"
proc ppLayer(layer: LayerRef; db: KvtDbRef; indent = 4): string =
proc ppLayer(layer: AristoTxRef; db: KvtDbRef; indent = 4): string =
let
tLen = layer.sTab.len
info = "tab(" & $tLen & ")"
@ -151,10 +151,8 @@ proc pp*(
case be.kind:
of BackendMemory:
result &= be.MemBackendRef.ppBe(db, indent)
of BackendRocksDB,BackendRdbTriggered:
of BackendRocksDB:
result &= be.RdbBackendRef.ppBe(db, indent)
of BackendVoid:
result &= "<NoBackend>"
proc pp*(
db: KvtDbRef;

14
execution_chain/db/kvt/kvt_desc.nim
View File

@ -16,21 +16,21 @@
import
std/[hashes, tables],
./kvt_constants,
./kvt_desc/[desc_error, desc_structural]
./kvt_desc/[desc_error]
from ./kvt_desc/desc_backend
import BackendRef, PutHdlRef
# Not auto-exporting backend
export
hashes, tables, kvt_constants, desc_error, desc_structural, PutHdlRef
hashes, tables, kvt_constants, desc_error, PutHdlRef
type
KvtTxRef* = ref object
## Transaction descriptor
db*: KvtDbRef ## Database descriptor
parent*: KvtTxRef ## Previous transaction
layer*: LayerRef
sTab*: Table[seq[byte],seq[byte]] ## Structural data table
KvtDbRef* = ref object of RootRef
## Three tier database object supporting distributed instances.
@ -58,8 +58,8 @@ func getOrVoid*(tab: Table[seq[byte],seq[byte]]; w: seq[byte]): seq[byte] =
func isValid*(key: seq[byte]): bool =
key != EmptyBlob
func isValid*(layer: LayerRef): bool =
layer != LayerRef(nil)
func isValid*(tx: KvtTxRef): bool =
tx != KvtTxRef(nil)
# ------------------------------------------------------------------------------
# Public functions, miscellaneous
@ -79,11 +79,11 @@ iterator stack*(tx: KvtTxRef): KvtTxRef =
while frames.len > 0:
yield frames.pop()
iterator rstack*(tx: KvtTxRef): LayerRef =
iterator rstack*(tx: KvtTxRef): KvtTxRef =
var tx = tx
# Stack in reverse order
while tx != nil:
yield tx.layer
yield tx
tx = tx.parent
# ------------------------------------------------------------------------------

17
execution_chain/db/kvt/kvt_desc/desc_error.nim
View File

@ -11,32 +11,15 @@
type
KvtError* = enum
NothingSerious = 0
GenericError
GetNotFound
KeyInvalid
DataInvalid
# RocksDB backend
RdbBeCantCreateDataDir
RdbBeDelayedAlreadyRegistered
RdbBeDelayedLocked
RdbBeDelayedNotReady
RdbBeDriverDelError
RdbBeDriverGetError
RdbBeDriverInitError
RdbBeDriverPutError
RdbBeDriverWriteError
RdbBeHostError
RdbBeHostNotApplicable
# Filter management
FilBackendRoMode
FilSiblingsCommitUnfinshed
# Functions from `kvt_desc`
MustBeOnCentre
NotAllowedOnCentre
StaleDescriptor
# End

42
execution_chain/db/kvt/kvt_desc/desc_structural.nim
View File

@ -1,42 +0,0 @@
# nimbus-eth1
# Copyright (c) 2023-2025 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
## Kvt DB -- structural data types
## ===============================
##
{.push raises: [].}
import
std/tables
export tables
type
LayerRef* = ref Layer
Layer* = object
## Kvt database layer structures. Any layer holds the full
## change relative to the backend.
sTab*: Table[seq[byte],seq[byte]] ## Structural data table
# ------------------------------------------------------------------------------
# Public helpers (misc)
# ------------------------------------------------------------------------------
func init*(T: type LayerRef): T =
## Constructor, returns empty layer
T()
func dup*(ly: LayerRef): LayerRef =
## Duplicate/copy
LayerRef(sTab: ly.sTab)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

21
execution_chain/db/kvt/kvt_init.nim
View File

@ -1,21 +0,0 @@
# nimbus-eth1
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
## Constructors for Key-Value Table DB
## ====================================
##
{.push raises: [].}
import
./kvt_init/memory_only
export
memory_only
# End

4
execution_chain/db/kvt/kvt_init/init_common.nim
View File

@ -1,5 +1,5 @@
# nimbus-eth1
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Copyright (c) 2023-2025 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
@ -20,10 +20,8 @@ const
type
BackendType* = enum
BackendVoid = 0 ## For providing backend-less constructor
BackendMemory ## Same as Aristo
BackendRocksDB ## Same as Aristo
BackendRdbTriggered ## Piggybacked on remote write session
TypedBackendRef* = ref TypedBackendObj
TypedBackendObj* = object of BackendObj

1
execution_chain/db/kvt/kvt_init/memory_db.nim
View File

@ -15,7 +15,6 @@
## backend access
## ::
## import
## kvt/kvt_init,
## kvt/kvt_init/kvt_memory
##
## let rc = newKvtDbRef(BackendMemory)

25
execution_chain/db/kvt/kvt_init/memory_only.nim
View File

@ -18,12 +18,6 @@ import
../kvt_desc/desc_backend,
"."/[init_common, memory_db]
type
VoidBackendRef* = ref object of TypedBackendRef
## Dummy descriptor type, used as `nil` reference
MemOnlyBackend* = VoidBackendRef|MemBackendRef
export
BackendType,
MemBackendRef
@ -37,9 +31,7 @@ func kind*(
): BackendType =
## Retrieves the backend type symbol for a `be` backend database argument
## where `BackendVoid` is returned for the`nil` backend.
if be.isNil:
BackendVoid
else:
doAssert(not be.isNil)
be.TypedBackendRef.beKind
# ------------------------------------------------------------------------------
@ -48,25 +40,14 @@ func kind*(
proc init*(
T: type KvtDbRef; # Target type
B: type MemOnlyBackend; # Backend type
B: type MemBackendRef; # Backend type
): T =
## Memory backend constructor.
##
let db = when B is VoidBackendRef:
KvtDbRef(txRef: KvtTxRef(layer: LayerRef.init()))
elif B is MemBackendRef:
KvtDbRef(txRef: KvtTxRef(layer: LayerRef.init()), backend: memoryBackend())
let db = KvtDbRef(txRef: KvtTxRef(), backend: memoryBackend())
db.txRef.db = db
db
proc init*(
T: type KvtDbRef; # Target type
): T =
## Shortcut for `KvtDbRef.init(VoidBackendRef)`
KvtDbRef.init VoidBackendRef
proc finish*(db: KvtDbRef; eradicate = false) =
## Backend destructor. The argument `eradicate` indicates that a full
## database deletion is requested. If set `false` the outcome might differ

2
execution_chain/db/kvt/kvt_init/persistent.nim
View File

@ -43,7 +43,7 @@ proc init*(
## Generic constructor for `RocksDb` backend
##
let db = KvtDbRef(
txRef: KvtTxRef(layer: LayerRef.init()),
txRef: KvtTxRef(),
backend: rocksDbKvtBackend(baseDb))
db.txRef.db = db
ok db

1
execution_chain/db/kvt/kvt_init/rocks_db.nim
View File

@ -15,7 +15,6 @@
## backend access
## ::
## import
## kvt/kvt_init,
## kvt/kvt_init/kvt_rocksdb
##
## let rc = KvtDb.init(BackendRocksDB, "/var/tmp")

6
execution_chain/db/kvt/kvt_layers.nim
View File

@ -56,20 +56,20 @@ func layersGet*(db: KvtTxRef; key: openArray[byte]|seq[byte]): Opt[seq[byte]] =
func layersPut*(db: KvtTxRef; key: openArray[byte]; data: openArray[byte]) =
## Store a (potentally empty) value on the top layer
db.layer.sTab[@key] = @data
db.sTab[@key] = @data
# ------------------------------------------------------------------------------
# Public functions
# ------------------------------------------------------------------------------
proc mergeAndReset*(trg, src: var Layer) =
proc mergeAndReset*(trg, src: KvtTxRef) =
mergeAndReset(trg.sTab, src.sTab)
# ------------------------------------------------------------------------------
# Public functions
# ------------------------------------------------------------------------------
func layersCc*(db: KvtDbRef; level = high(int)): LayerRef =
func layersCc*(db: KvtDbRef; level = high(int)): KvtTxRef =
## Provide a collapsed copy of layers up to a particular transaction level.
## If the `level` argument is too large, the maximum transaction level is
## returned. For the result layer, the `txUid` value set to `0`.

18
execution_chain/db/kvt/kvt_persistent.nim
View File

@ -1,18 +0,0 @@
# nimbus-eth1
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
import
kvt_init/persistent as init_persistent,
kvt_walk/persistent as walk_persistent
export
init_persistent,
walk_persistent
# End

9
execution_chain/db/kvt/kvt_tx_frame.nim
View File

@ -36,7 +36,6 @@ proc txFrameBegin*(db: KvtDbRef, parent: KvtTxRef): KvtTxRef =
let parent = if parent == nil: db.txRef else: parent
KvtTxRef(
db: db,
layer: LayerRef(),
parent: parent,
)
@ -46,7 +45,6 @@ proc baseTxFrame*(db: KvtDbRef): KvtTxRef =
proc dispose*(
tx: KvtTxRef;
) =
tx[].reset()
proc txFramePersist*(
@ -65,7 +63,7 @@ proc txFramePersist*(
for frame in txFrame.stack():
if frame == db.txRef:
continue
mergeAndReset(db.txRef.layer[], frame.layer[])
mergeAndReset(db.txRef, frame)
frame.dispose()
# Put the now-merged contents in txFrame and make it the new base
@ -73,14 +71,15 @@ proc txFramePersist*(
db.txRef = txFrame
# Store structural single trie entries
for k,v in txFrame.layer.sTab:
for k,v in txFrame.sTab:
be.putKvpFn(batch, k, v)
# TODO above, we only prepare the changes to the database but don't actually
# write them to disk - the code below that updates the frame should
# really run after things have been written (to maintain sync betweeen
# in-memory and on-disk state)
txFrame.layer.sTab.clear()
# Done with txRef, all saved to backend
txFrame.sTab.clear()
# ------------------------------------------------------------------------------
# End

6
execution_chain/db/kvt/kvt_walk/memory_only.nim
View File

@ -1,5 +1,5 @@
# Nimbus-eth1
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Copyright (c) 2023-2025 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
@ -13,7 +13,7 @@
##
import
../kvt_init/[memory_db, memory_only],
".."/[kvt_desc, kvt_init],
".."/kvt_desc,
./walk_private
export
@ -24,7 +24,7 @@ export
# Public iterators (all in one)
# ------------------------------------------------------------------------------
iterator walkPairs*[T: MemBackendRef|VoidBackendRef](
iterator walkPairs*[T: MemBackendRef](
_: type T;
db: KvtDbRef;
): tuple[key: seq[byte], data: seq[byte]] =

5
execution_chain/db/kvt/kvt_walk/walk_private.nim
View File

@ -1,5 +1,5 @@
# Nimbus-eth1
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Copyright (c) 2023-2025 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
@ -10,7 +10,7 @@
import
std/sets,
".."/[kvt_desc, kvt_init, kvt_layers]
".."/[kvt_desc, kvt_layers]
# ------------------------------------------------------------------------------
# Public generic iterators
@ -26,7 +26,6 @@ iterator walkPairsImpl*[T](
if data.isValid:
yield (key,data)
when T isnot VoidBackendRef:
mixin walk
for (key,data) in db.backend.T.walk:

3
nimbus.nimble
View File

@ -72,9 +72,6 @@ proc test(path: string, name: string, params = "", lang = "c") =
task test, "Run tests":
test "tests", "all_tests", "-d:chronicles_log_level=ERROR"
task test_rocksdb, "Run rocksdb tests":
test "tests/db", "test_kvstore_rocksdb", "-d:chronicles_log_level=ERROR"
task test_import, "Run block import test":
let tmp = getTempDir() / "nimbus-eth1-block-import"
if dirExists(tmp):

60
tests/db/test_kvstore_rocksdb.nim
View File

@ -1,60 +0,0 @@
# Nimbus
# Copyright (c) 2023-2025 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
{.used.}
import
std/os,
unittest2,
eth/db/kvstore,
../../execution_chain/db/kvstore_rocksdb,
eth/../tests/db/test_kvstore
suite "KvStore RocksDb Tests":
const
NS_DEFAULT = "default"
NS_OTHER = "other"
test "RocksStoreRef KvStore interface":
let tmp = getTempDir() / "nimbus-test-db"
removeDir(tmp)
let db = RocksStoreRef.init(tmp, "test")[]
defer:
db.close()
testKvStore(kvStore db, false, false)
test "RocksNamespaceRef KvStore interface - default namespace":
let tmp = getTempDir() / "nimbus-test-db"
removeDir(tmp)
let db = RocksStoreRef.init(tmp, "test")[]
defer:
db.close()
let defaultNs = db.openNamespace(NS_DEFAULT)[]
testKvStore(kvStore defaultNs, false, false)
test "RocksNamespaceRef KvStore interface - multiple namespace":
let tmp = getTempDir() / "nimbus-test-db"
removeDir(tmp)
let db = RocksStoreRef.init(tmp, "test",
namespaces = @[NS_DEFAULT, NS_OTHER])[]
defer:
db.close()
let defaultNs = db.openNamespace(NS_DEFAULT)[]
testKvStore(kvStore defaultNs, false, false)
let otherNs = db.openNamespace(NS_OTHER)[]
testKvStore(kvStore otherNs, false, false)

6
tests/test_aristo/test_compute.nim
View File

@ -20,15 +20,13 @@ import
aristo_delete,
aristo_merge,
aristo_desc,
aristo_init,
aristo_init/memory_only,
aristo_persist,
aristo_tx_frame,
]
func x(s: string): seq[byte] =
s.hexToSeqByte
func k(s: string): HashKey =
HashKey.fromBytes(s.x).value
let samples = [
# Somew on-the-fly provided stuff
@ -79,7 +77,7 @@ suite "Aristo compute":
for n, sample in samples:
test "Add and delete entries " & $n:
let
db = AristoDbRef.init VoidBackendRef
db = AristoDbRef.init MemBackendRef
txFrame = db.txRef
root = VertexID(1)

1
tests/test_coredb.nim
View File

@ -160,7 +160,6 @@ proc initRunnerDB(
case dbType:
of AristoDbMemory: AristoDbMemory.newCoreDbRef()
of AristoDbRocks: AristoDbRocks.newCoreDbRef(path, DbOptions.init())
of AristoDbVoid: AristoDbVoid.newCoreDbRef()
else: raiseAssert $dbType
when false: # or true: