status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Core db reorg (#2444) 

* CoreDb: Merged all sub-descriptors into `base_desc` module

* Dissolve `aristo_db/common_desc.nim`

* No need to export `Aristo` methods in `CoreDb`

* Resolve/tighten methods in `aristo_db` sub-moduled

why:
  So they can be straihgt implemented into the `base` module

* Moved/re-implemented `KVT` methods into `base` module

* Moved/re-implemented `MPT` methods into `base` module

* Moved/re-implemented account methods into `base` module

* Moved/re-implemented `CTX` methods into `base` module

* Moved/re-implemented `handler_{aristo,kvt}` into `aristo_db` module

* Moved/re-implemented `TX` methods into `base` module

* Moved/re-implemented base methods into `base` module

* Replaced `toAristoSavedStateBlockNumber()` by proper base method

why:
  Was the last for keeping reason for keeping low level backend access
  methods

* Remove dedicated low level access to `Aristo` backend

why:
  Not needed anymore, for debugging the descriptors can be accessed
  directly

also:
  some clean up stuff

* Re-factor `CoreDb` descriptor layout and adjust base methods

* Moved/re-implemented iterators into `base_iterator*` modules

* Update docu
This commit is contained in:
Jordan Hrycaj 2024-07-03 15:50:27 +00:00 committed by GitHub
parent b1ff4ef3a5
commit ea7c756a9d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
29 changed files with 874 additions and 1981 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
nimbus/db/aristo.nim
View File

@ -23,6 +23,7 @@ import
export
MemBackendRef,
VoidBackendRef,
finish,
init
import

18
nimbus/db/aristo/aristo_api.nim
View File

@ -384,7 +384,7 @@ type
## is returned if there was any.
AristoApiTxBeginFn* =
proc(db: AristoDbRef
proc(db: AristoDbRef;
): Result[AristoTxRef,AristoError]
{.noRaise.}
## Starts a new transaction.
@ -397,6 +397,12 @@ type
## ... continue using db ...
## tx.commit()
AristoApiTxLevelFn* =
proc(tx: AristoTxRef;
): int
{.noRaise.}
## Getter, positive nesting level of transaction argument `tx`
AristoApiTxTopFn* =
proc(db: AristoDbRef;
): Result[AristoTxRef,AristoError]
@ -445,6 +451,7 @@ type
reCentre*: AristoApiReCentreFn
rollback*: AristoApiRollbackFn
txBegin*: AristoApiTxBeginFn
txLevel*: AristoApiTxLevelFn
txTop*: AristoApiTxTopFn
@ -491,6 +498,7 @@ type
AristoApiProfReCentreFn = "reCentre"
AristoApiProfRollbackFn = "rollback"
AristoApiProfTxBeginFn = "txBegin"
AristoApiProfTxLevelFn = "txLevel"
AristoApiProfTxTopFn = "txTop"
AristoApiProfBeGetVtxFn = "be/getVtx"
@ -554,6 +562,7 @@ when AutoValidateApiHooks:
doAssert not api.reCentre.isNil
doAssert not api.rollback.isNil
doAssert not api.txBegin.isNil
doAssert not api.txLevel.isNil
doAssert not api.txTop.isNil
proc validate(prf: AristoApiProfRef) =
@ -621,6 +630,7 @@ func init*(api: var AristoApiObj) =
api.reCentre = reCentre
api.rollback = rollback
api.txBegin = txBegin
api.txLevel = txLevel
api.txTop = txTop
when AutoValidateApiHooks:
api.validate
@ -670,6 +680,7 @@ func dup*(api: AristoApiRef): AristoApiRef =
reCentre: api.reCentre,
rollback: api.rollback,
txBegin: api.txBegin,
txLevel: api.txLevel,
txTop: api.txTop)
when AutoValidateApiHooks:
api.validate
@ -861,6 +872,11 @@ func init*(
AristoApiProfTxBeginFn.profileRunner:
result = api.txBegin(a)
profApi.txLevel =
proc(a: AristoTxRef): auto =
AristoApiProfTxLevelFn.profileRunner:
result = api.txLevel(a)
profApi.txTop =
proc(a: AristoDbRef): auto =
AristoApiProfTxTopFn.profileRunner:

2
nimbus/db/aristo/aristo_tx.nim
View File

@ -31,7 +31,7 @@ func isTop*(tx: AristoTxRef): bool =
## level transaction.
tx.txFrameIsTop()
func level*(tx: AristoTxRef): int =
func txLevel*(tx: AristoTxRef): int =
## Getter, positive nesting level of transaction argument `tx`
tx.txFrameLevel()

2
nimbus/db/aristo/aristo_utils.nim
View File

@ -14,7 +14,7 @@
{.push raises: [].}
import
std/[sequtils, typetraits],
std/sequtils,
eth/common,
results,
"."/[aristo_constants, aristo_desc, aristo_get, aristo_hike, aristo_layers]

96
nimbus/db/core_db/README.md
View File

@ -1,84 +1,26 @@
Core database replacement wrapper object
========================================
This wrapper replaces the *TrieDatabaseRef* and its derivatives by the new
object *CoreDbRef*.
Core database
=============
# **out of date**
Layout of `CoreDb` descriptor objects
-------------------------------------
### Objects dependence:
Relations to current *TrieDatabaseRef* implementation
-----------------------------------------------------
Here are some incomplete translations for objects and constructors.
### Object types:
| **Legacy notation** | **CoreDbRef based replacement** |
|:----------------------------|:--------------------------------------|
| | |
| ChainDB | (don't use/avoid) |
| ChainDbRef | CoreDbRef |
| TrieDatabaseRef | CoreDbKvtRef |
| HexaryTrie | CoreDbMptRef |
| SecureHexaryTrie | CoreDbPhkRef |
| DbTransaction | CoreDbTxRef |
| TransactionID | CoreDbTxID |
### Constructors:
| **Legacy notation** | **CoreDbRef based replacement** |
|:----------------------------|:--------------------------------------|
| | |
| trieDB newChainDB("..") | newCoreDbRef(LegacyDbPersistent,"..") |
| newMemoryDB() | newCoreDbRef(LegacyDbMemory) |
| -- | |
| initHexaryTrie(db,..) | db.mpt(..) (no pruning) |
| | db.mptPrune(..) (w/pruning true/false)|
| -- | |
| initSecureHexaryTrie(db,..) | db.phk(..) (no pruning) |
| | db.phkPrune(..) (w/pruning true/false)|
| -- | |
| newCaptureDB(db,memDB) | db.capture() (see below) |
Usage of the replacement wrapper
--------------------------------
### Objects pedigree:
CoreDbRef -- base descriptor
CoreDbRef -- Base descriptor
| | |
| | +--- CoreDbCtxRef -- MPT context descriptor
| | | |
| | | +-- CoreDbMptRef -- hexary trie instance
| | | | : :
| | | +-- CoreDbMptRef -- hexary trie instance
| | |
| | |
| | +---- CoreDbPhkRef -- pre-hashed key hexary trie instance
| | | : :
| | +---- CoreDbPhkRef -- pre-hashed key hexary trie instance
| | +--- CoreDbCtxRef -- Context descriptor
| | | | | |
| | | | | +--- CoreDbKvtRef -- Key-value table
| | | | |
| | | | +----- CoreDbMptRef -- Generic MPT
| | | |
| | | +------- CoreDbAccRef -- Accounts database
| | |
| | +--------- CoreDbTxRef -- Transaction handle
| |
| |
| +------ CoreDbKvtRef -- single static key-value table
| +----- CoreDbCtxRef
| : : : :
|
|
+-------- CoreDbCaptRef -- tracer support descriptor
+------- CoreDbCtxRef
: : : :
### Instantiating legacy standard database object descriptors works as follows:
let
db = newCoreDbRef(..) # new base descriptor
mpt = db.mpt(..) # hexary trie/Merkle Patricia Tree
phk = db.phk(..) # pre-hashed key hexary trie/MPT
kvt = db.kvt # key-value table
### Tracer support setup by hiding the current *CoreDbRef* behind a replacement:
let
capture = db.capture()
db = capture.recorder # use the recorder in place of db
...
for key,value in capture.recorder.kvt:
... # process recorded data

7
nimbus/db/core_db/TODO.md
View File

@ -1,4 +1,3 @@
* Re-implement *getOldestJournalBlockNumber()* and
*getLatestJournalBlockNumber()* (from the `core_apps` module) via the CoreDb
base api. Currently this api is bypassed (via the *backend()* directive). The
functionality is directly provided by the `Aristo` backend.
* Refactor `handlers_tracer`. This module can reliably work only as a genuine
logger. The restore features were ill concieved, an attempt to be as close
as possible to the legacy tracer.

289
nimbus/db/core_db/backend/aristo_db.nim
View File

@ -11,196 +11,38 @@
{.push raises: [].}
import
std/tables,
eth/common,
../../aristo as use_ari,
../../aristo/[aristo_walk, aristo_serialise],
../../aristo/[aristo_init/memory_only, aristo_walk],
../../kvt as use_kvt,
../../kvt/[kvt_init/memory_only, kvt_walk],
".."/[base, base/base_desc],
./aristo_db/[common_desc, handlers_aristo, handlers_kvt]
import
../../aristo/aristo_init/memory_only as aristo_memory_only
# Caveat:
# additional direct include(s) -- not import(s) -- is placed near
# the end of this source file
# Annotation helper(s)
{.pragma: noRaise, gcsafe, raises: [].}
{.pragma: rlpRaise, gcsafe, raises: [AristoApiRlpError].}
export
AristoApiRlpError,
AristoCoreDbKvtBE,
isAristo
type
AristoCoreDbRef* = ref object of CoreDbRef
## Main descriptor
kdbBase: KvtBaseRef ## Kvt subsystem
adbBase: AristoBaseRef ## Aristo subsystem
#tracer: AristoTracerRef ## Currently active recorder
#AristoTracerRef = ref object of TraceRecorderRef
# ## Sub-handle for tracer
# parent: AristoCoreDbRef
proc newAristoVoidCoreDbRef*(): CoreDbRef {.noRaise.}
# ------------------------------------------------------------------------------
# Private tx and base methods
# ------------------------------------------------------------------------------
proc txMethods(
db: AristoCoreDbRef;
aTx: AristoTxRef;
kTx: KvtTxRef;
): CoreDbTxFns =
## To be constructed by some `CoreDbBaseFns` function
let
adbBase = db.adbBase
kdbBase = db.kdbBase
adbApi = adbBase.api
kdbApi = kdbBase.api
CoreDbTxFns(
levelFn: proc(): int =
aTx.level,
commitFn: proc() =
const info = "commitFn()"
adbApi.commit(aTx).isOkOr:
raiseAssert info & ": " & $error
kdbApi.commit(kTx).isOkOr:
raiseAssert info & ": " & $error
discard,
rollbackFn: proc() =
const info = "rollbackFn()"
adbApi.rollback(aTx).isOkOr:
raiseAssert info & ": " & $error
kdbApi.rollback(kTx).isOkOr:
raiseAssert info & ": " & $error
discard,
disposeFn: proc() =
const info = "disposeFn()"
if adbApi.isTop(aTx):
adbApi.rollback(aTx).isOkOr:
raiseAssert info & ": " & $error
if kdbApi.isTop(kTx):
kdbApi.rollback(kTx).isOkOr:
raiseAssert info & ": " & $error
discard)
when false: # currently disabled
proc cptMethods(
tracer: AristoTracerRef;
): CoreDbCaptFns =
let
tr = tracer # So it can savely be captured
db = tr.parent # Will not change and can be captured
log = tr.topInst() # Ditto
CoreDbCaptFns(
recorderFn: proc(): CoreDbRef =
db,
logDbFn: proc(): TableRef[Blob,Blob] =
log.kLog,
getFlagsFn: proc(): set[CoreDbCaptFlags] =
log.flags,
forgetFn: proc() =
if not tracer.pop():
tr.parent.tracer = AristoTracerRef(nil)
tr.restore())
proc baseMethods(db: AristoCoreDbRef): CoreDbBaseFns =
let
aBase = db.adbBase
kBase = db.kdbBase
when false: # currently disabled
proc tracerSetup(flags: set[CoreDbCaptFlags]): CoreDbCaptRef =
if db.tracer.isNil:
db.tracer = AristoTracerRef(parent: db)
db.tracer.init(kBase, aBase, flags)
else:
db.tracer.push(flags)
CoreDbCaptRef(methods: db.tracer.cptMethods)
proc persistent(bn: Opt[BlockNumber]): CoreDbRc[void] =
const info = "persistentFn()"
let sid =
if bn.isNone: 0u64
else: bn.unsafeGet
? kBase.persistent info
? aBase.persistent(sid, info)
ok()
CoreDbBaseFns(
destroyFn: proc(eradicate: bool) =
aBase.destroy(eradicate)
kBase.destroy(eradicate),
levelFn: proc(): int =
aBase.getLevel,
errorPrintFn: proc(e: CoreDbErrorRef): string =
e.errorPrint(),
newKvtFn: proc(): CoreDbRc[CoreDbKvtRef] =
kBase.newKvtHandler("newKvtFn()"),
newCtxFn: proc(): CoreDbCtxRef =
aBase.ctx,
newCtxFromTxFn: proc(r: Hash256; k: CoreDbColType): CoreDbRc[CoreDbCtxRef] =
CoreDbCtxRef.init(db.adbBase, r, k),
swapCtxFn: proc(ctx: CoreDbCtxRef): CoreDbCtxRef =
aBase.swapCtx(ctx),
beginFn: proc(): CoreDbTxRef =
const info = "beginFn()"
let
aTx = aBase.txBegin info
kTx = kBase.txBegin info
dsc = CoreDbTxRef(methods: db.txMethods(aTx, kTx))
db.bless(dsc),
# # currently disabled
# newCaptureFn: proc(flags:set[CoreDbCaptFlags]): CoreDbRc[CoreDbCaptRef] =
# ok(db.bless flags.tracerSetup()),
persistentFn: proc(bn: Opt[BlockNumber]): CoreDbRc[void] =
persistent(bn))
".."/[base, base/base_desc]
# ------------------------------------------------------------------------------
# Public constructor and helper
# ------------------------------------------------------------------------------
proc create*(dbType: CoreDbType; kdb: KvtDbRef; adb: AristoDbRef): CoreDbRef =
proc create*(dbType: CoreDbType; kvt: KvtDbRef; mpt: AristoDbRef): CoreDbRef =
## Constructor helper
var db = CoreDbRef(dbType: dbType)
db.defCtx = db.bless CoreDbCtxRef(mpt: mpt, kvt: kvt)
# Local extensions
var db = AristoCoreDbRef()
db.adbBase = AristoBaseRef.init(db, adb)
db.kdbBase = KvtBaseRef.init(db, kdb)
when CoreDbEnableApiTracking:
db.kvtApi = KvtApiRef.init()
db.ariApi = AristoApiRef.init()
# Base descriptor
db.dbType = dbType
db.methods = db.baseMethods()
db.bless()
when CoreDbEnableApiProfiling:
block:
let profApi = KvtApiProfRef.init(db.kvtApi, kvt.backend)
db.kvtApi = profApi
kvt.backend = profApi.be
block:
let profApi = AristoApiProfRef.init(db.ariApi, mpt.backend)
db.ariApi = profApi
mpt.backend = profApi.be
bless db
proc newAristoMemoryCoreDbRef*(): CoreDbRef =
AristoDbMemory.create(
result = AristoDbMemory.create(
KvtDbRef.init(use_kvt.MemBackendRef),
AristoDbRef.init(use_ari.MemBackendRef))
@ -209,99 +51,6 @@ proc newAristoVoidCoreDbRef*(): CoreDbRef =
KvtDbRef.init(use_kvt.VoidBackendRef),
AristoDbRef.init(use_ari.VoidBackendRef))
# ------------------------------------------------------------------------------
# Public helpers, e.g. for direct backend access
# ------------------------------------------------------------------------------
func toAristoProfData*(
db: CoreDbRef;
): tuple[aristo: AristoDbProfListRef, kvt: KvtDbProfListRef] =
when CoreDbEnableApiProfiling:
if db.isAristo:
result.aristo = db.AristoCoreDbRef.adbBase.api.AristoApiProfRef.data
result.kvt = db.AristoCoreDbRef.kdbBase.api.KvtApiProfRef.data
func toAristoApi*(kvt: CoreDbKvtRef): KvtApiRef =
if kvt.parent.isAristo:
return AristoCoreDbRef(kvt.parent).kdbBase.api
func toAristoApi*(mpt: CoreDbMptRef): AristoApiRef =
if mpt.parent.isAristo:
return mpt.to(AristoApiRef)
func toAristo*(kBe: CoreDbKvtBackendRef): KvtDbRef =
if not kBe.isNil and kBe.parent.isAristo:
return kBe.AristoCoreDbKvtBE.kdb
func toAristo*(mBe: CoreDbMptBackendRef): AristoDbRef =
if not mBe.isNil and mBe.parent.isAristo:
return mBe.AristoCoreDbMptBE.adb
func toAristo*(mBe: CoreDbAccBackendRef): AristoDbRef =
if not mBe.isNil and mBe.parent.isAristo:
return mBe.AristoCoreDbAccBE.adb
proc toAristoSavedStateBlockNumber*(
mBe: CoreDbMptBackendRef;
): BlockNumber =
if not mBe.isNil and mBe.parent.isAristo:
let rc = mBe.parent.AristoCoreDbRef.adbBase.getSavedState()
if rc.isOk:
return rc.value.serial.BlockNumber
# ------------------------------------------------------------------------------
# Public aristo iterators
# ------------------------------------------------------------------------------
include
./aristo_db/aristo_replicate
# ------------------------
iterator aristoKvtPairsVoid*(dsc: CoreDbKvtRef): (Blob,Blob) {.rlpRaise.} =
let
api = dsc.toAristoApi()
p = api.forkTx(dsc.to(KvtDbRef),0).valueOrApiError "aristoKvtPairs()"
defer: discard api.forget(p)
for (k,v) in use_kvt.VoidBackendRef.walkPairs p:
yield (k,v)
iterator aristoKvtPairsMem*(dsc: CoreDbKvtRef): (Blob,Blob) {.rlpRaise.} =
let
api = dsc.toAristoApi()
p = api.forkTx(dsc.to(KvtDbRef),0).valueOrApiError "aristoKvtPairs()"
defer: discard api.forget(p)
for (k,v) in use_kvt.MemBackendRef.walkPairs p:
yield (k,v)
iterator aristoMptPairs*(dsc: CoreDbMptRef): (Blob,Blob) {.noRaise.} =
let
api = dsc.to(AristoApiRef)
mpt = dsc.to(AristoDbRef)
for (path,data) in mpt.rightPairsGeneric dsc.rootID:
yield (api.pathAsBlob(path), data)
iterator aristoSlotPairs*(
dsc: CoreDbAccRef;
accPath: Hash256;
): (Blob,Blob)
{.noRaise.} =
let
api = dsc.to(AristoApiRef)
mpt = dsc.to(AristoDbRef)
for (path,data) in mpt.rightPairsStorage accPath:
yield (api.pathAsBlob(path), data)
iterator aristoReplicateMem*(dsc: CoreDbMptRef): (Blob,Blob) {.rlpRaise.} =
## Instantiation for `MemBackendRef`
for k,v in aristoReplicate[use_ari.MemBackendRef](dsc):
yield (k,v)
iterator aristoReplicateVoid*(dsc: CoreDbMptRef): (Blob,Blob) {.rlpRaise.} =
## Instantiation for `VoidBackendRef`
for k,v in aristoReplicate[use_ari.VoidBackendRef](dsc):
yield (k,v)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

3
nimbus/db/core_db/backend/aristo_db/TODO.md
View File

@ -1,3 +0,0 @@
* Refactor `handlers_tracer`. This module can reliably work only as a genuine
logger. The restore features were ill concieved, an attempt to be as close
as possible to the legacy tracer.

40
nimbus/db/core_db/backend/aristo_db/aristo_replicate.nim
View File

@ -1,40 +0,0 @@
# Nimbus
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
## Generic iterator, prototype to be included (rather than imported). Using
## an include file avoids duplicating code because the `T` argument is not
## bound to any object type. Otherwise all object types would be required
## when providing this iterator for import.
##
## This is not wanted here, because the import of a **pesistent** object
## would always require extra linking.
template valueOrApiError[U,V](rc: Result[U,V]; info: static[string]): U =
rc.valueOr: raise (ref AristoApiRlpError)(msg: info)
iterator aristoReplicate[T](
dsc: CoreDbMptRef;
): (Blob,Blob)
{.gcsafe, raises: [AristoApiRlpError].} =
## Generic iterator used for building dedicated backend iterators.
##
let
root = dsc.rootID
mpt = dsc.to(AristoDbRef)
api = dsc.to(AristoApiRef)
p = api.forkTx(mpt,0).valueOrApiError "aristoReplicate()"
defer: discard api.forget(p)
for (vid,key,vtx,node) in T.replicate(p):
if key.len == 32:
yield (@(key.data), node.encode)
elif vid == root:
yield (@(key.to(Hash256).data), node.encode)
# End

62
nimbus/db/core_db/backend/aristo_db/common_desc.nim
View File

@ -1,62 +0,0 @@
# Nimbus
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
{.push raises: [].}
import
std/strutils,
../../../../errors,
"../../.."/[aristo, kvt],
../../base/base_desc
type
AristoApiRlpError* = object of CoreDbApiError
## For re-routing exceptions in iterator closure
AristoCoreDbError* = ref object of CoreDbErrorRef
## Error return code
ctx*: string ## Context where the exception or error occured
case isAristo*: bool
of true:
vid*: VertexID
aErr*: AristoError
else:
kErr*: KvtError
# ------------------------------------------------------------------------------
# Public helpers
# ------------------------------------------------------------------------------
func isAristo*(db: CoreDbRef): bool =
db.dbType in {AristoDbMemory, AristoDbRocks, AristoDbVoid}
func toStr*(n: VertexID): string =
result = "$"
if n.isValid:
result &= n.uint64.toHex.strip(
leading=true, trailing=false, chars={'0'}).toLowerAscii
else:
result &= "ø"
func errorPrint*(e: CoreDbErrorRef): string =
if not e.isNil:
let e = e.AristoCoreDbError
result = if e.isAristo: "Aristo" else: "Kvt"
result &= ", ctx=" & $e.ctx & ", "
if e.isAristo:
if e.vid.isValid:
result &= "vid=" & e.vid.toStr & ", "
result &= "error=" & $e.aErr
else:
result &= "error=" & $e.kErr
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

580
nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim
View File

@ -1,580 +0,0 @@
# Nimbus
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
{.push raises: [].}
import
std/typetraits,
eth/common,
stew/byteutils,
../../../aristo,
../../../aristo/aristo_desc,
../../base,
../../base/base_desc,
./common_desc
type
AristoBaseRef* = ref object
parent: CoreDbRef ## Opaque top level descriptor
api*: AristoApiRef ## Api functions can be re-directed
ctx*: AristoCoreDbCtxRef ## Currently active context
AristoCoreDbCtxRef* = ref object of CoreDbCtxRef
base: AristoBaseRef ## Local base descriptor
mpt*: AristoDbRef ## Aristo MPT database
AristoCoreDbAccRef = ref object of CoreDbAccRef
base: AristoBaseRef ## Local base descriptor
AristoCoreDbMptRef = ref object of CoreDbMptRef
base: AristoBaseRef ## Local base descriptor
mptRoot: VertexID ## State root, may be zero unless account
AristoCoreDbMptBE* = ref object of CoreDbMptBackendRef
adb*: AristoDbRef
AristoCoreDbAccBE* = ref object of CoreDbAccBackendRef
adb*: AristoDbRef
static:
doAssert high(CoreDbColType).ord < LEAST_FREE_VID
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
func toError(
e: AristoError;
base: AristoBaseRef;
info: string;
error = Unspecified;
): CoreDbErrorRef =
base.parent.bless(error, AristoCoreDbError(
ctx: info,
isAristo: true,
aErr: e))
func toError(
e: (VertexID,AristoError);
base: AristoBaseRef;
info: string;
error = Unspecified;
): CoreDbErrorRef =
base.parent.bless(error, AristoCoreDbError(
ctx: info,
isAristo: true,
vid: e[0],
aErr: e[1]))
func toRc[T](
rc: Result[T,AristoError];
base: AristoBaseRef;
info: string;
error = Unspecified;
): CoreDbRc[T] =
if rc.isOk:
when T is void:
return ok()
else:
return ok(rc.value)
err((VertexID(0),rc.error).toError(base, info, error))
# ------------------------------------------------------------------------------
# Private `MPT` call back functions
# ------------------------------------------------------------------------------
proc mptMethods(): CoreDbMptFns =
# These templates are a hack to remove a closure environment that was using
# hundreds of mb of memory to have this syntactic convenience
# TODO remove methods / abstraction entirely - it is no longer needed
template base: untyped = cMpt.base
template db: untyped = base.parent # Ditto
template api: untyped = base.api # Ditto
template mpt: untyped = base.ctx.mpt # Ditto
proc mptBackend(cMpt: AristoCoreDbMptRef): CoreDbMptBackendRef =
db.bless AristoCoreDbMptBE(adb: mpt)
proc mptFetch(cMpt: AristoCoreDbMptRef, key: openArray[byte]): CoreDbRc[Blob] =
const info = "fetchFn()"
let data = api.fetchGenericData(mpt, cMpt.mptRoot, key).valueOr:
if error == FetchPathNotFound:
return err(error.toError(base, info, MptNotFound))
return err(error.toError(base, info))
ok(data)
proc mptMerge(cMpt: AristoCoreDbMptRef, k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] =
const info = "mergeFn()"
api.mergeGenericData(mpt, cMpt.mptRoot, k, v).isOkOr:
return err(error.toError(base, info))
ok()
proc mptDelete(cMpt: AristoCoreDbMptRef, key: openArray[byte]): CoreDbRc[void] =
const info = "deleteFn()"
api.deleteGenericData(mpt, cMpt.mptRoot, key).isOkOr:
if error == DelPathNotFound:
return err(error.toError(base, info, MptNotFound))
return err(error.toError(base, info))
ok()
proc mptHasPath(cMpt: AristoCoreDbMptRef, key: openArray[byte]): CoreDbRc[bool] =
const info = "hasPathFn()"
let yn = api.hasPathGeneric(mpt, cMpt.mptRoot, key).valueOr:
return err(error.toError(base, info))
ok(yn)
proc mptState(cMpt: AristoCoreDbMptRef, updateOk: bool): CoreDbRc[Hash256] =
const info = "mptState()"
let state = api.fetchGenericState(mpt, cMpt.mptRoot, updateOk).valueOr:
return err(error.toError(base, info))
ok(state)
## Generic columns database handlers
CoreDbMptFns(
backendFn: proc(cMpt: CoreDbMptRef): CoreDbMptBackendRef =
mptBackend(AristoCoreDbMptRef(cMpt)),
fetchFn: proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[Blob] =
mptFetch(AristoCoreDbMptRef(cMpt), k),
deleteFn: proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[void] =
mptDelete(AristoCoreDbMptRef(cMpt), k),
mergeFn: proc(cMpt: CoreDbMptRef, k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] =
mptMerge(AristoCoreDbMptRef(cMpt), k, v),
hasPathFn: proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[bool] =
mptHasPath(AristoCoreDbMptRef(cMpt), k),
stateFn: proc(cMpt: CoreDbMptRef, updateOk: bool): CoreDbRc[Hash256] =
mptState(AristoCoreDbMptRef(cMpt), updateOk))
# ------------------------------------------------------------------------------
# Private account call back functions
# ------------------------------------------------------------------------------
proc accMethods(): CoreDbAccFns =
## Account columns database handlers
template base: untyped = cAcc.base
template db: untyped = base.parent
template api: untyped = base.api
template mpt: untyped = base.ctx.mpt
proc accBackend(cAcc: AristoCoreDbAccRef): CoreDbAccBackendRef =
db.bless AristoCoreDbAccBE(adb: mpt)
proc accFetch(
cAcc: AristoCoreDbAccRef;
accPath: Hash256;
): CoreDbRc[CoreDbAccount] =
const info = "acc/fetchFn()"
let acc = api.fetchAccountRecord(mpt, accPath).valueOr:
if error != FetchPathNotFound:
return err(error.toError(base, info))
return err(error.toError(base, info, AccNotFound))
ok acc
proc accMerge(
cAcc: AristoCoreDbAccRef;
accPath: Hash256;
accRec: CoreDbAccount;
): CoreDbRc[void] =
const info = "acc/mergeFn()"
let val = AristoAccount(
nonce: accRec.nonce,
balance: accRec.balance,
codeHash: accRec.codeHash)
api.mergeAccountRecord(mpt, accPath, val).isOkOr:
return err(error.toError(base, info))
ok()
proc accDelete(
cAcc: AristoCoreDbAccRef;
accPath: Hash256;
): CoreDbRc[void] =
const info = "acc/deleteFn()"
api.deleteAccountRecord(mpt, accPath).isOkOr:
if error == DelPathNotFound:
# TODO: Would it be conseqient to just return `ok()` here?
return err(error.toError(base, info, AccNotFound))
return err(error.toError(base, info))
ok()
proc accClearStorage(
cAcc: AristoCoreDbAccRef;
accPath: Hash256;
): CoreDbRc[void] =
const info = "acc/clearStoFn()"
api.deleteStorageTree(mpt, accPath).isOkOr:
if error notin {DelStoRootMissing,DelStoAccMissing}:
return err(error.toError(base, info))
ok()
proc accHasPath(
cAcc: AristoCoreDbAccRef;
accPath: Hash256;
): CoreDbRc[bool] =
const info = "hasPathFn()"
let yn = api.hasPathAccount(mpt, accPath).valueOr:
return err(error.toError(base, info))
ok(yn)
proc accState(
cAcc: AristoCoreDbAccRef,
updateOk: bool;
): CoreDbRc[Hash256] =
const info = "accStateFn()"
let state = api.fetchAccountState(mpt, updateOk).valueOr:
return err(error.toError(base, info))
ok(state)
proc slotFetch(
cAcc: AristoCoreDbAccRef;
accPath: Hash256;
stoPath: openArray[byte];
): CoreDbRc[Blob] =
const info = "slotFetchFn()"
let data = api.fetchStorageData(mpt, accPath, stoPath).valueOr:
if error != FetchPathNotFound:
return err(error.toError(base, info))
return err(error.toError(base, info, StoNotFound))
ok(data)
proc slotDelete(
cAcc: AristoCoreDbAccRef;
accPath: Hash256;
stoPath: openArray[byte];
): CoreDbRc[void] =
const info = "slotDeleteFn()"
api.deleteStorageData(mpt, accPath, stoPath).isOkOr:
if error == DelPathNotFound:
return err(error.toError(base, info, StoNotFound))
if error == DelStoRootMissing:
# This is insane but legit. A storage column was announced for an
# account but no data have been added, yet.
return ok()
return err(error.toError(base, info))
ok()
proc slotHasPath(
cAcc: AristoCoreDbAccRef;
accPath: Hash256;
stoPath: openArray[byte];
): CoreDbRc[bool] =
const info = "slotHasPathFn()"
let yn = api.hasPathStorage(mpt, accPath, stoPath).valueOr:
return err(error.toError(base, info))
ok(yn)
proc slotMerge(
cAcc: AristoCoreDbAccRef;
accPath: Hash256;
stoPath: openArray[byte];
stoData: openArray[byte];
): CoreDbRc[void] =
const info = "slotMergeFn()"
api.mergeStorageData(mpt, accPath, stoPath, stoData).isOkOr:
return err(error.toError(base, info))
ok()
proc slotState(
cAcc: AristoCoreDbAccRef;
accPath: Hash256;
updateOk: bool;
): CoreDbRc[Hash256] =
const info = "slotStateFn()"
let state = api.fetchStorageState(mpt, accPath, updateOk).valueOr:
return err(error.toError(base, info))
ok(state)
proc slotStateEmpty(
cAcc: AristoCoreDbAccRef;
accPath: Hash256;
): CoreDbRc[bool] =
const info = "slotStateEmptyFn()"
let yn = api.hasStorageData(mpt, accPath).valueOr:
return err(error.toError(base, info))
ok(not yn)
CoreDbAccFns(
backendFn: proc(cAcc: CoreDbAccRef): CoreDbAccBackendRef =
accBackend(AristoCoreDbAccRef(cAcc)),
fetchFn: proc(
cAcc: CoreDbAccRef;
accPath: Hash256;
): CoreDbRc[CoreDbAccount] =
accFetch(AristoCoreDbAccRef(cAcc), accPath),
deleteFn: proc(
cAcc: CoreDbAccRef;
accPath: Hash256;
): CoreDbRc[void] =
accDelete(AristoCoreDbAccRef(cAcc), accPath),
clearStorageFn: proc(
cAcc: CoreDbAccRef;
accPath: Hash256;
): CoreDbRc[void] =
accClearStorage(AristoCoreDbAccRef(cAcc), accPath),
mergeFn: proc(
cAcc: CoreDbAccRef;
accPath: Hash256;
accRec: CoreDbAccount;
): CoreDbRc[void] =
accMerge(AristoCoreDbAccRef(cAcc), accPath, accRec),
hasPathFn: proc(
cAcc: CoreDbAccRef;
accPath: Hash256;
): CoreDbRc[bool] =
accHasPath(AristoCoreDbAccRef(cAcc), accPath),
stateFn: proc(
cAcc: CoreDbAccRef;
updateOk: bool;
): CoreDbRc[Hash256] =
accState(AristoCoreDbAccRef(cAcc), updateOk),
slotFetchFn: proc(
cAcc: CoreDbAccRef;
accPath: Hash256;
stoPath: openArray[byte];
): CoreDbRc[Blob] =
slotFetch(AristoCoreDbAccRef(cAcc), accPath, stoPath),
slotDeleteFn: proc(
cAcc: CoreDbAccRef;
accPath: Hash256;
stoPath: openArray[byte];
): CoreDbRc[void] =
slotDelete(AristoCoreDbAccRef(cAcc), accPath, stoPath),
slotHasPathFn: proc(
cAcc: CoreDbAccRef;
accPath: Hash256;
stoPath: openArray[byte];
): CoreDbRc[bool] =
slotHasPath(AristoCoreDbAccRef(cAcc), accPath, stoPath),
slotMergeFn: proc(
cAcc: CoreDbAccRef;
accPath: Hash256;
stoPath: openArray[byte];
stoData: openArray[byte];
): CoreDbRc[void] =
slotMerge(AristoCoreDbAccRef(cAcc), accPath, stoPath, stoData),
slotStateFn: proc(
cAcc: CoreDbAccRef;
accPath: Hash256;
updateOk: bool;
): CoreDbRc[Hash256] =
slotState(AristoCoreDbAccRef(cAcc), accPath, updateOk),
slotStateEmptyFn: proc(
cAcc: CoreDbAccRef;
accPath: Hash256;
): CoreDbRc[bool] =
slotStateEmpty(AristoCoreDbAccRef(cAcc), accPath))
# ------------------------------------------------------------------------------
# Private context call back functions
# ------------------------------------------------------------------------------
proc ctxMethods(): CoreDbCtxFns =
template base: untyped = cCtx.base
template db: untyped = base.parent
template api: untyped = base.api
template mpt: untyped = cCtx.mpt
proc ctxGetColumn(cCtx: AristoCoreDbCtxRef; colType: CoreDbColType; clearData: bool): CoreDbMptRef =
const info = "getColumnFn()"
if clearData:
api.deleteGenericTree(mpt, VertexID(colType)).isOkOr:
raiseAssert info & " clearing up failed: " & $error
db.bless AristoCoreDbMptRef(
methods: mptMethods(),
base: base,
mptRoot: VertexID(colType))
proc ctxGetAccounts(cCtx: AristoCoreDbCtxRef): CoreDbAccRef =
db.bless AristoCoreDbAccRef(
methods: accMethods(),
base: base)
proc ctxForget(cCtx: AristoCoreDbCtxRef) =
api.forget(mpt).isOkOr:
raiseAssert "forgetFn(): " & $error
CoreDbCtxFns(
getColumnFn: proc(cCtx: CoreDbCtxRef; colType: CoreDbColType; clearData: bool): CoreDbMptRef =
ctxGetColumn(AristoCoreDbCtxRef(cCtx), colType, clearData),
getAccountsFn: proc(cCtx: CoreDbCtxRef): CoreDbAccRef =
ctxGetAccounts(AristoCoreDbCtxRef(cCtx)),
forgetFn: proc(cCtx: CoreDbCtxRef) =
ctxForget(AristoCoreDbCtxRef(cCtx)))
# ------------------------------------------------------------------------------
# Public handlers and helpers
# ------------------------------------------------------------------------------
proc getSavedState*(base: AristoBaseRef): Result[SavedState,void] =
let be = base.ctx.mpt.backend
if not be.isNil:
let rc = base.api.fetchLastSavedState(base.ctx.mpt)
if rc.isOk:
return ok(rc.value)
err()
# ---------------------
func to*(dsc: CoreDbMptRef, T: type AristoDbRef): T =
AristoCoreDbMptRef(dsc).base.ctx.mpt
func to*(dsc: CoreDbAccRef, T: type AristoDbRef): T =
AristoCoreDbAccRef(dsc).base.ctx.mpt
func to*(dsc: CoreDbMptRef, T: type AristoApiRef): T =
AristoCoreDbMptRef(dsc).base.api
func to*(dsc: CoreDbAccRef, T: type AristoApiRef): T =
AristoCoreDbAccRef(dsc).base.api
func rootID*(dsc: CoreDbMptRef): VertexID =
AristoCoreDbMptRef(dsc).mptRoot
func txTop*(
base: AristoBaseRef;
info: static[string];
): CoreDbRc[AristoTxRef] =
base.api.txTop(base.adb).toRc(base, info)
proc txBegin*(
base: AristoBaseRef;
info: static[string];
): AristoTxRef =
let rc = base.api.txBegin(base.ctx.mpt)
if rc.isErr:
raiseAssert info & ": " & $rc.error
rc.value
proc getLevel*(base: AristoBaseRef): int =
base.api.level(base.ctx.mpt)
# ---------------------
proc swapCtx*(base: AristoBaseRef; ctx: CoreDbCtxRef): CoreDbCtxRef =
doAssert not ctx.isNil
result = base.ctx
# Set read-write access and install
base.ctx = AristoCoreDbCtxRef(ctx)
base.api.reCentre(base.ctx.mpt).isOkOr:
raiseAssert "swapCtx() failed: " & $error
proc persistent*(
base: AristoBaseRef;
fid: uint64;
info: static[string];
): CoreDbRc[void] =
let
api = base.api
mpt = base.ctx.mpt
rc = api.persist(mpt, fid)
if rc.isOk:
ok()
elif api.level(mpt) == 0:
err(rc.error.toError(base, info))
else:
err(rc.error.toError(base, info, TxPending))
# ------------------------------------------------------------------------------
# Public constructors and related
# ------------------------------------------------------------------------------
proc destroy*(base: AristoBaseRef; eradicate: bool) =
base.api.finish(base.ctx.mpt, eradicate)
func init*(T: type AristoBaseRef; db: CoreDbRef; adb: AristoDbRef): T =
result = T(
parent: db,
api: AristoApiRef.init())
# Create initial context
let ctx = AristoCoreDbCtxRef(
methods: ctxMethods(),
base: result,
mpt: adb)
result.ctx = db.bless ctx
when CoreDbEnableApiProfiling:
let profApi = AristoApiProfRef.init(result.api, adb.backend)
result.api = profApi
result.ctx.mpt.backend = profApi.be
proc init*(
T: type CoreDbCtxRef;
base: AristoBaseRef;
colState: Hash256;
colType: CoreDbColType;
): CoreDbRc[CoreDbCtxRef] =
const info = "fromTxFn()"
if colType.ord == 0:
return err(aristo.GenericError.toError(base, info, ColUnacceptable))
let
api = base.api
vid = VertexID(colType)
key = colState.to(HashKey)
# Find `(vid,key)` on transaction stack
inx = block:
let rc = api.findTx(base.ctx.mpt, vid, key)
if rc.isErr:
return err(rc.error.toError(base, info))
rc.value
# Fork MPT descriptor that provides `(vid,key)`
newMpt = block:
let rc = api.forkTx(base.ctx.mpt, inx)
if rc.isErr:
return err(rc.error.toError(base, info))
rc.value
# Create new context
let ctx = AristoCoreDbCtxRef(
methods: ctxMethods(),
base: base,
mpt: newMpt)
ok(base.parent.bless ctx)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

266
nimbus/db/core_db/backend/aristo_db/handlers_kvt.nim
View File

@ -1,266 +0,0 @@
# Nimbus
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
{.push raises: [].}
import
chronicles,
eth/common,
results,
../../../kvt as use_kvt,
../../base,
../../base/base_desc,
./common_desc
type
KvtBaseRef* = ref object
parent: CoreDbRef ## Opaque top level descriptor
kdb: KvtDbRef ## Shared key-value table
api*: KvtApiRef ## Api functions can be re-directed
cache: KvtCoreDbKvtRef ## Shared transaction table wrapper
KvtCoreDbKvtRef = ref object of CoreDbKvtRef
base: KvtBaseRef ## Local base descriptor
kvt: KvtDbRef ## In most cases different from `base.kdb`
AristoCoreDbKvtBE* = ref object of CoreDbKvtBackendRef
kdb*: KvtDbRef
logScope:
topics = "kvt-hdl"
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
func toError(
e: KvtError;
base: KvtBaseRef;
info: string;
error = Unspecified;
): CoreDbErrorRef =
base.parent.bless(error, AristoCoreDbError(
ctx: info,
isAristo: false,
kErr: e))
func toRc[T](
rc: Result[T,KvtError];
base: KvtBaseRef;
info: string;
error = Unspecified;
): CoreDbRc[T] =
if rc.isOk:
when T is void:
return ok()
else:
return ok(rc.value)
err rc.error.toError(base, info, error)
# ------------------------------------------------------------------------------
# Private `kvt` call back functions
# ------------------------------------------------------------------------------
proc kvtMethods(cKvt: KvtCoreDbKvtRef): CoreDbKvtFns =
## Key-value database table handlers
proc kvtBackend(
cKvt:KvtCoreDbKvtRef;
): CoreDbKvtBackendRef =
cKvt.base.parent.bless AristoCoreDbKvtBE(kdb: cKvt.kvt)
proc kvtForget(
cKvt: KvtCoreDbKvtRef;
info: static[string];
): CoreDbRc[void] =
let
base = cKvt.base
kvt = cKvt.kvt
if kvt != base.kdb:
let rc = base.api.forget(kvt)
# There is not much that can be done in case of a `forget()` error.
# So unmark it anyway.
cKvt.kvt = KvtDbRef(nil)
if rc.isErr:
return err(rc.error.toError(base, info))
ok()
proc kvtGet(
cKvt: KvtCoreDbKvtRef;
k: openArray[byte];
info: static[string];
): CoreDbRc[Blob] =
let rc = cKvt.base.api.get(cKvt.kvt, k)
if rc.isOk:
ok(rc.value)
elif rc.error == GetNotFound:
err(rc.error.toError(cKvt.base, info, KvtNotFound))
else:
rc.toRc(cKvt.base, info)
proc kvtLen(
cKvt: KvtCoreDbKvtRef;
k: openArray[byte];
info: static[string];
): CoreDbRc[int] =
let rc = cKvt.base.api.len(cKvt.kvt, k)
if rc.isOk:
ok(rc.value)
elif rc.error == GetNotFound:
err(rc.error.toError(cKvt.base, info, KvtNotFound))
else:
rc.toRc(cKvt.base, info)
proc kvtPut(
cKvt: KvtCoreDbKvtRef;
k: openArray[byte];
v: openArray[byte];
info: static[string];
): CoreDbRc[void] =
let rc = cKvt.base.api.put(cKvt.kvt, k, v)
if rc.isOk:
ok()
else:
err(rc.error.toError(cKvt.base, info))
proc kvtDel(
cKvt: KvtCoreDbKvtRef;
k: openArray[byte];
info: static[string];
): CoreDbRc[void] =
let rc = cKvt.base.api.del(cKvt.kvt, k)
if rc.isOk:
ok()
else:
err(rc.error.toError(cKvt.base, info))
proc kvtHasKey(
cKvt: KvtCoreDbKvtRef;
k: openArray[byte];
info: static[string];
): CoreDbRc[bool] =
let rc = cKvt.base.api.hasKey(cKvt.kvt, k)
if rc.isOk:
ok(rc.value)
else:
err(rc.error.toError(cKvt.base, info))
CoreDbKvtFns(
backendFn: proc(): CoreDbKvtBackendRef =
cKvt.kvtBackend(),
getFn: proc(k: openArray[byte]): CoreDbRc[Blob] =
cKvt.kvtGet(k, "getFn()"),
lenFn: proc(k: openArray[byte]): CoreDbRc[int] =
cKvt.kvtLen(k, "lenFn()"),
delFn: proc(k: openArray[byte]): CoreDbRc[void] =
cKvt.kvtDel(k, "delFn()"),
putFn: proc(k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] =
cKvt.kvtPut(k, v, "putFn()"),
hasKeyFn: proc(k: openArray[byte]): CoreDbRc[bool] =
cKvt.kvtHasKey(k, "hasKeyFn()"),
forgetFn: proc(): CoreDbRc[void] =
cKvt.kvtForget("forgetFn()"))
# ------------------------------------------------------------------------------
# Public handlers and helpers
# ------------------------------------------------------------------------------
func toVoidRc*[T](
rc: Result[T,KvtError];
base: KvtBaseRef;
info: string;
error = Unspecified;
): CoreDbRc[void] =
if rc.isErr:
return err(rc.error.toError(base, info, error))
ok()
# ---------------------
func to*(dsc: CoreDbKvtRef; T: type KvtDbRef): T =
KvtCoreDbKvtRef(dsc).kvt
func txTop*(
base: KvtBaseRef;
info: static[string];
): CoreDbRc[KvtTxRef] =
base.api.txTop(base.kdb).toRc(base, info)
proc txBegin*(
base: KvtBaseRef;
info: static[string];
): KvtTxRef =
let rc = base.api.txBegin(base.kdb)
if rc.isErr:
raiseAssert info & ": " & $rc.error
rc.value
proc persistent*(
base: KvtBaseRef;
info: static[string];
): CoreDbRc[void] =
let
api = base.api
kvt = base.kdb
rc = api.persist(kvt)
if rc.isOk:
ok()
elif api.level(kvt) != 0:
err(rc.error.toError(base, info, TxPending))
elif rc.error == TxPersistDelayed:
# This is OK: Piggybacking on `Aristo` backend
ok()
else:
err(rc.error.toError(base, info))
# ------------------------------------------------------------------------------
# Public constructors and related
# ------------------------------------------------------------------------------
proc newKvtHandler*(
base: KvtBaseRef;
info: static[string];
): CoreDbRc[CoreDbKvtRef] =
ok(base.cache)
proc destroy*(base: KvtBaseRef; eradicate: bool) =
base.api.finish(base.kdb, eradicate) # Close descriptor
func init*(T: type KvtBaseRef; db: CoreDbRef; kdb: KvtDbRef): T =
result = T(
parent: db,
api: KvtApiRef.init(),
kdb: kdb)
# Preallocated shared descriptor
let dsc = KvtCoreDbKvtRef(
base: result,
kvt: kdb)
dsc.methods = dsc.kvtMethods()
result.cache = db.bless dsc
when CoreDbEnableApiProfiling:
let profApi = KvtApiProfRef.init(result.api, kdb.backend)
result.api = profApi
result.kdb.backend = profApi.be
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

76
nimbus/db/core_db/backend/aristo_replicate.nim Normal file
View File

@ -0,0 +1,76 @@
# Nimbus
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
## Generic iterator, prototype to be included (rather than imported). Using
## an include file avoids duplicating code because the `T` argument is not
## bound to any object type. Otherwise all object types would be required
## when providing this iterator for import.
##
## This is not wanted here, because the import of a **pesistent** object
## would always require extra linking.
template valueOrApiError[U,V](rc: Result[U,V]; info: static[string]): U =
rc.valueOr: raise (ref CoreDbApiError)(msg: info)
template dbType(dsc: CoreDbKvtRef | CoreDbMptRef | CoreDbAccRef): CoreDbType =
dsc.distinctBase.parent.dbType
# ---------------
template kvt(dsc: CoreDbKvtRef): KvtDbRef =
dsc.distinctBase.kvt
template call(api: KvtApiRef; fn: untyped; args: varArgs[untyped]): untyped =
when CoreDbEnableApiJumpTable:
api.fn(args)
else:
fn(args)
template call(kvt: CoreDbKvtRef; fn: untyped; args: varArgs[untyped]): untyped =
kvt.distinctBase.parent.kvtApi.call(fn, args)
# ---------------
template mpt(dsc: CoreDbAccRef | CoreDbMptRef): AristoDbRef =
dsc.distinctBase.mpt
template rootID(mpt: CoreDbMptRef): VertexID =
VertexID(CtGeneric)
template call(api: AristoApiRef; fn: untyped; args: varArgs[untyped]): untyped =
when CoreDbEnableApiJumpTable:
api.fn(args)
else:
fn(args)
template call(
acc: CoreDbAccRef | CoreDbMptRef;
fn: untyped;
args: varArgs[untyped];
): untyped =
acc.distinctBase.parent.ariApi.call(fn, args)
# ---------------
iterator aristoReplicate[T](
mpt: CoreDbMptRef;
): (Blob,Blob)
{.gcsafe, raises: [CoreDbApiError].} =
## Generic iterator used for building dedicated backend iterators.
##
let p = mpt.call(forkTx, mpt.mpt, 0).valueOrApiError "aristoReplicate()"
defer: discard mpt.call(forget, p)
for (vid,key,vtx,node) in T.replicate(p):
if key.len == 32:
yield (@(key.data), node.encode)
elif vid == mpt.rootID:
yield (@(key.to(Hash256).data), node.encode)
# End

15
nimbus/db/core_db/backend/aristo_rocksdb.nim
View File

@ -11,7 +11,6 @@
{.push raises: [].}
import
std/sequtils,
eth/common,
rocksdb,
results,
@ -23,10 +22,9 @@ import
../../kvt/kvt_init/rocks_db/rdb_init,
../base,
./aristo_db,
./aristo_db/[common_desc, handlers_aristo],
../../opts
include ./aristo_db/aristo_replicate
include ./aristo_replicate
const
# Expectation messages
@ -34,7 +32,7 @@ const
kvtFail = "Kvt/RocksDB init() failed"
# Annotation helper(s)
{.pragma: rlpRaise, gcsafe, raises: [AristoApiRlpError].}
{.pragma: rlpRaise, gcsafe, raises: [CoreDbApiError].}
proc toRocksDb*(
opts: DbOptions
@ -172,15 +170,6 @@ proc newAristoDualRocksDbCoreDbRef*(path: string, opts: DbOptions): CoreDbRef =
raiseAssert kvtFail & ": " & $error
AristoDbRocks.create(kdb, adb)
# ------------------------------------------------------------------------------
# Public aristo iterators
# ------------------------------------------------------------------------------
iterator aristoReplicateRdb*(dsc: CoreDbMptRef): (Blob, Blob) {.rlpRaise.} =
## Instantiation for `VoidBackendRef`
for k, v in aristoReplicate[use_ari.RdbBackendRef](dsc):
yield (k, v)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

134
nimbus/db/core_db/backend/todo/free_parking.nim Normal file
View File

@ -0,0 +1,134 @@
# Nimbus
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
{.push raises: [].}
import
eth/common,
../../aristo as use_ari,
../../aristo/[aristo_walk],
../../kvt as use_kvt,
../../kvt/[kvt_init/memory_only, kvt_walk],
".."/[base, base/base_desc]
# Caveat:
# additional direct include(s) -- not import(s) -- is placed near
# the end of this source file
# Annotation helper(s)
{.pragma: noRaise, gcsafe, raises: [].}
{.pragma: rlpRaise, gcsafe, raises: [CoreDbApiError].}
# ------------------------------------------------------------------------------
# Private functions, free parking
# ------------------------------------------------------------------------------
when false:
func toError(e: AristoError; s: string; error = Unspecified): CoreDbErrorRef =
CoreDbErrorRef(
error: error,
ctx: s,
isAristo: true,
aErr: e)
func toError(e: KvtError; s: string; error = Unspecified): CoreDbErrorRef =
CoreDbErrorRef(
error: error,
ctx: s,
isAristo: false,
kErr: e)
proc kvtForget(
cKvt: CoreDbKvtRef;
info: static[string];
): CoreDbRc[void] =
## Free parking here
let
base = cKvt.parent.kdbBase
kvt = cKvt.kvt
if kvt != base.kdb:
let rc = base.api.forget(kvt)
# There is not much that can be done in case of a `forget()` error.
# So unmark it anyway.
cKvt.kvt = KvtDbRef(nil)
if rc.isErr:
return err(rc.error.toError(base, info))
ok()
proc cptMethods(
tracer: AristoTracerRef;
): CoreDbCaptFns =
## Free parking here -- currently disabled
let
tr = tracer # So it can savely be captured
db = tr.parent # Will not change and can be captured
log = tr.topInst() # Ditto
CoreDbCaptFns(
recorderFn: proc(): CoreDbRef =
db,
logDbFn: proc(): TableRef[Blob,Blob] =
log.kLog,
getFlagsFn: proc(): set[CoreDbCaptFlags] =
log.flags,
forgetFn: proc() =
if not tracer.pop():
tr.parent.tracer = AristoTracerRef(nil)
tr.restore())
proc tracerSetup(flags: set[CoreDbCaptFlags]): CoreDbCaptRef =
## Free parking here -- currently disabled
if db.tracer.isNil:
db.tracer = AristoTracerRef(parent: db)
db.tracer.init(kBase, aBase, flags)
else:
db.tracer.push(flags)
CoreDbCaptRef(methods: db.tracer.cptMethods)
proc init(
T: type CoreDbCtxRef;
base: CoreDbAriBaseRef;
colState: Hash256;
colType: CoreDbColType;
): CoreDbRc[CoreDbCtxRef] =
const info = "fromTxFn()"
if colType.ord == 0:
return err(use_ari.GenericError.toError(base, info, ColUnacceptable))
let
api = base.api
vid = VertexID(colType)
key = colState.to(HashKey)
# Find `(vid,key)` on transaction stack
inx = block:
let rc = api.findTx(base.parent.ctx.CoreDbCtxRef.mpt, vid, key)
if rc.isErr:
return err(rc.error.toError(base, info))
rc.value
# Fork MPT descriptor that provides `(vid,key)`
newMpt = block:
let rc = api.forkTx(base.parent.ctx.CoreDbCtxRef.mpt, inx)
if rc.isErr:
return err(rc.error.toError(base, info))
rc.value
# Create new context
ok(base.parent.bless CoreDbCtxRef(mpt: newMpt))
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

0
nimbus/db/core_db/backend/aristo_db/handlers_trace.nim → nimbus/db/core_db/backend/todo/handlers_trace.nim
View File

700
nimbus/db/core_db/base.nim
View File

File diff suppressed because it is too large Load Diff

37
nimbus/db/core_db/base/api_tracking.nim
View File

@ -20,21 +20,22 @@ import
type
CoreDbApiTrackRef* =
CoreDbRef | CoreDbKvtRef |
CoreDbCtxRef | CoreDbMptRef | CoreDbAccRef |
CoreDbTxRef | CoreDbCaptRef | CoreDbErrorRef
# CoreDbCaptRef |
CoreDbRef | CoreDbKvtRef | CoreDbCtxRef | CoreDbMptRef | CoreDbAccRef |
CoreDbTxRef
CoreDbFnInx* = enum
## Profiling table index
SummaryItem = "total"
AccClearStorageFn = "clearStorage"
AccDeleteFn = "acc/delete"
AccFetchFn = "acc/fetch"
AccForgetFn = "acc/forget"
AccHasPathFn = "acc/hasPath"
AccMergeFn = "acc/merge"
AccRecastFn = "recast"
AccStateFn = "acc/state"
AccClearStorageFn = "acc/clearStorage"
AccSlotFetchFn = "slotFetch"
AccSlotDeleteFn = "slotDelete"
@ -45,17 +46,13 @@ type
AccSlotStateEmptyOrVoidFn = "slotStateEmptyOrVoid"
AccSlotPairsIt = "slotPairs"
AnyBackendFn = "any/backend"
BaseDbTypeFn = "dbType"
BaseFinishFn = "finish"
BaseLevelFn = "level"
BaseNewCaptureFn = "newCapture"
BaseNewCtxFn = "ctx"
BaseNewCtxFromTxFn = "ctxFromTx"
BaseNewKvtFn = "newKvt"
BaseNewTxFn = "newTransaction"
BasePersistentFn = "persistent"
BaseStateBlockNumberFn = "stateBlockNumber"
BaseSwapCtxFn = "swapCtx"
CptFlagsFn = "cpt/flags"
@ -65,14 +62,9 @@ type
CtxForgetFn = "ctx/forget"
CtxGetAccountsFn = "getAccounts"
CtxGetColumnFn = "getColumn"
CtxNewColFn = "ctx/newColumn"
ErrorPrintFn = "$$"
EthAccRecastFn = "recast"
CtxGetGenericFn = "getGeneric"
KvtDelFn = "del"
KvtForgetFn = "forget"
KvtGetFn = "get"
KvtGetOrEmptyFn = "getOrEmpty"
KvtHasKeyFn = "hasKey"
@ -111,7 +103,11 @@ func toStr*(w: Hash256): string =
if w == EMPTY_ROOT_HASH: "EMPTY_ROOT_HASH" else: w.data.oaToStr
proc toStr*(e: CoreDbErrorRef): string =
$e.error & "(" & e.parent.methods.errorPrintFn(e) & ")"
result = $e.error & "("
result &= (if e.isAristo: "Aristo" else: "Kvt")
result &= ", ctx=" & $e.ctx & ", error="
result &= (if e.isAristo: $e.aErr else: $e.kErr)
result &= ")"
func toLenStr*(w: openArray[byte]): string =
if 0 < w.len and w.len < 5: "<" & w.oaToStr & ">"
@ -156,7 +152,7 @@ proc toStr*(rc: CoreDbRc[CoreDbRef]): string = rc.toStr "db"
proc toStr*(rc: CoreDbRc[CoreDbAccount]): string = rc.toStr "acc"
proc toStr*(rc: CoreDbRc[CoreDbKvtRef]): string = rc.toStr "kvt"
proc toStr*(rc: CoreDbRc[CoreDbTxRef]): string = rc.toStr "tx"
proc toStr*(rc: CoreDbRc[CoreDbCaptRef]): string = rc.toStr "capt"
#proc toStr*(rc: CoreDbRc[CoreDbCaptRef]): string = rc.toStr "capt"
proc toStr*(rc: CoreDbRc[CoreDbCtxRef]): string = rc.toStr "ctx"
proc toStr*(rc: CoreDbRc[CoreDbMptRef]): string = rc.toStr "mpt"
proc toStr*(rc: CoreDbRc[CoreDbAccRef]): string = rc.toStr "acc"
@ -177,9 +173,12 @@ template endNewApiIf*(w: CoreDbApiTrackRef; code: untyped) =
block body:
when typeof(w) is CoreDbRef:
let db = w
else:
elif typeof(w) is CoreDbTxRef:
let db = w.ctx.parent
if w.isNil: break body
let db = w.parent
else:
let db = w.distinctBase.parent
if w.distinctBase.isNil: break body
when CoreDbEnableApiProfiling:
let elapsed {.inject,used.} = getTime() - bnaStart
aristo_profile.update(db.profTab, bnaCtx.ord, elapsed)

295
nimbus/db/core_db/base/base_desc.nim
View File

@ -11,9 +11,9 @@
{.push raises: [].}
import
std/tables,
eth/common,
results,
../../aristo,
../../kvt,
../../aristo/aristo_profile
# Annotation helpers
@ -59,261 +59,82 @@ type
StoNotFound
TxPending
CoreDbColType* = enum
CtGeneric = 2 # columns smaller than 2 are not provided
CoreDbColType* = enum # Keep that legacy type for a while ..
CtGeneric = 2 # Actually only this constant is needed
CoreDbCaptFlags* {.pure.} = enum
PersistPut
PersistDel
# --------------------------------------------------
# Sub-descriptor: Misc methods for main descriptor
# --------------------------------------------------
CoreDbBaseDestroyFn* = proc(eradicate = true) {.noRaise.}
CoreDbBaseErrorPrintFn* = proc(e: CoreDbErrorRef): string {.noRaise.}
CoreDbBaseLevelFn* = proc(): int {.noRaise.}
CoreDbBaseNewKvtFn* = proc(): CoreDbRc[CoreDbKvtRef] {.noRaise.}
CoreDbBaseNewCtxFn* = proc(): CoreDbCtxRef {.noRaise.}
CoreDbBaseNewCtxFromTxFn* = proc(
colState: Hash256; kind: CoreDbColType): CoreDbRc[CoreDbCtxRef] {.noRaise.}
CoreDbBaseSwapCtxFn* = proc(ctx: CoreDbCtxRef): CoreDbCtxRef {.noRaise.}
CoreDbBaseTxBeginFn* = proc(): CoreDbTxRef {.noRaise.}
CoreDbBaseNewCaptFn* =
proc(flgs: set[CoreDbCaptFlags]): CoreDbRc[CoreDbCaptRef] {.noRaise.}
CoreDbBaseGetCaptFn* = proc(): CoreDbRc[CoreDbCaptRef] {.noRaise.}
CoreDbBasePersistentFn* =
proc(bn: Opt[BlockNumber]): CoreDbRc[void] {.noRaise.}
CoreDbBaseFns* = object
destroyFn*: CoreDbBaseDestroyFn
errorPrintFn*: CoreDbBaseErrorPrintFn
levelFn*: CoreDbBaseLevelFn
# Kvt constructor
newKvtFn*: CoreDbBaseNewKvtFn
# MPT context constructor
newCtxFn*: CoreDbBaseNewCtxFn
newCtxFromTxFn*: CoreDbBaseNewCtxFromTxFn
swapCtxFn*: CoreDbBaseSwapCtxFn
# Transactions constructors
beginFn*: CoreDbBaseTxBeginFn
# Capture/tracer constructors
newCaptureFn*: CoreDbBaseNewCaptFn
# Save to disk
persistentFn*: CoreDbBasePersistentFn
# --------------------------------------------------
# Sub-descriptor: KVT methods
# --------------------------------------------------
CoreDbKvtBackendFn* = proc(): CoreDbKvtBackendRef {.noRaise.}
CoreDbKvtGetFn* = proc(k: openArray[byte]): CoreDbRc[Blob] {.noRaise.}
CoreDbKvtLenFn* = proc(k: openArray[byte]): CoreDbRc[int] {.noRaise.}
CoreDbKvtDelFn* = proc(k: openArray[byte]): CoreDbRc[void] {.noRaise.}
CoreDbKvtPutFn* =
proc(k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] {.noRaise.}
CoreDbKvtForgetFn* = proc(): CoreDbRc[void] {.noRaise.}
CoreDbKvtHasKeyFn* = proc(k: openArray[byte]): CoreDbRc[bool] {.noRaise.}
CoreDbKvtFns* = object
## Methods for key-value table
backendFn*: CoreDbKvtBackendFn
getFn*: CoreDbKvtGetFn
lenFn*: CoreDbKvtLenFn
delFn*: CoreDbKvtDelFn
putFn*: CoreDbKvtPutFn
hasKeyFn*: CoreDbKvtHasKeyFn
forgetFn*: CoreDbKvtForgetFn
# --------------------------------------------------
# Sub-descriptor: MPT context methods
# --------------------------------------------------
CoreDbCtxGetColumnFn* = proc(
cCtx: CoreDbCtxRef; colType: CoreDbColType; clearData: bool): CoreDbMptRef {.noRaise.}
CoreDbCtxGetAccountsFn* = proc(cCtx: CoreDbCtxRef): CoreDbAccRef {.noRaise.}
CoreDbCtxForgetFn* = proc(cCtx: CoreDbCtxRef) {.noRaise.}
CoreDbCtxFns* = object
## Methods for context maniulation
getColumnFn*: CoreDbCtxGetColumnFn
getAccountsFn*: CoreDbCtxGetAccountsFn
forgetFn*: CoreDbCtxForgetFn
# --------------------------------------------------
# Sub-descriptor: generic Mpt methods
# --------------------------------------------------
CoreDbMptBackendFn* = proc(cMpt: CoreDbMptRef): CoreDbMptBackendRef {.noRaise.}
CoreDbMptFetchFn* =
proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[Blob] {.noRaise.}
CoreDbMptFetchAccountFn* =
proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[CoreDbAccount] {.noRaise.}
CoreDbMptDeleteFn* =
proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[void] {.noRaise.}
CoreDbMptMergeFn* =
proc(cMpt: CoreDbMptRef, k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] {.noRaise.}
CoreDbMptHasPathFn* = proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[bool] {.noRaise.}
CoreDbMptStateFn* = proc(cMpt: CoreDbMptRef, updateOk: bool): CoreDbRc[Hash256] {.noRaise.}
CoreDbMptFns* = object
## Methods for trie objects
backendFn*: CoreDbMptBackendFn
fetchFn*: CoreDbMptFetchFn
deleteFn*: CoreDbMptDeleteFn
mergeFn*: CoreDbMptMergeFn
hasPathFn*: CoreDbMptHasPathFn
stateFn*: CoreDbMptStateFn
# ----------------------------------------------------
# Sub-descriptor: Account column methods
# ------------------------------------------------------
CoreDbAccBackendFn* = proc(
cAcc: CoreDbAccRef): CoreDbAccBackendRef {.noRaise.}
CoreDbAccFetchFn* = proc(
cAcc: CoreDbAccRef; accPath: Hash256;
): CoreDbRc[CoreDbAccount] {.noRaise.}
CoreDbAccDeleteFn* = proc(
cAcc: CoreDbAccRef, accPath: Hash256): CoreDbRc[void] {.noRaise.}
CoreDbAccClearStorageFn* = proc(
cAcc: CoreDbAccRef; accPath: Hash256): CoreDbRc[void] {.noRaise.}
CoreDbAccMergeFn* = proc(
cAcc: CoreDbAccRef; accPath: Hash256; accRec: CoreDbAccount;
): CoreDbRc[void] {.noRaise.}
CoreDbAccHasPathFn* = proc(
cAcc: CoreDbAccRef; accPath: Hash256): CoreDbRc[bool] {.noRaise.}
CoreDbAccStateFn* = proc(
cAcc: CoreDbAccRef; updateOk: bool): CoreDbRc[Hash256] {.noRaise.}
CoreDbSlotFetchFn* = proc(
cAcc: CoreDbAccRef; accPath: Hash256, stoPath: openArray[byte];
): CoreDbRc[Blob] {.noRaise.}
CoreDbSlotDeleteFn* = proc(
cAcc: CoreDbAccRef; accPath: Hash256, stoPath: openArray[byte];
): CoreDbRc[void] {.noRaise.}
CoreDbSlotHasPathFn* = proc(
cAcc: CoreDbAccRef; accPath: Hash256, stoPath: openArray[byte];
): CoreDbRc[bool] {.noRaise.}
CoreDbSlotMergeFn* = proc(
cAcc: CoreDbAccRef; accPath: Hash256, stoPath, stoData: openArray[byte];
): CoreDbRc[void] {.noRaise.}
CoreDbSlotStateFn* = proc(
cAcc: CoreDbAccRef; accPath: Hash256; updateOk: bool;
): CoreDbRc[Hash256] {.noRaise.}
CoreDbSlotStateEmptyFn* = proc(
cAcc: CoreDbAccRef; accPath: Hash256;
): CoreDbRc[bool] {.noRaise.}
CoreDbAccFns* = object
## Methods for trie objects
backendFn*: CoreDbAccBackendFn
fetchFn*: CoreDbAccFetchFn
clearStorageFn*: CoreDbAccClearStorageFn
deleteFn*: CoreDbAccDeleteFn
hasPathFn*: CoreDbAccHasPathFn
mergeFn*: CoreDbAccMergeFn
stateFn*: CoreDbAccStateFn
slotFetchFn*: CoreDbSlotFetchFn
slotDeleteFn*: CoreDbSlotDeleteFn
slotHasPathFn*: CoreDbSlotHasPathFn
slotMergeFn*: CoreDbSlotMergeFn
slotStateFn*: CoreDbSlotStateFn
slotStateEmptyFn*: CoreDbSlotStateEmptyFn
# --------------------------------------------------
# Sub-descriptor: Transaction frame management
# --------------------------------------------------
CoreDbTxLevelFn* = proc(): int {.noRaise.}
CoreDbTxCommitFn* = proc() {.noRaise.}
CoreDbTxRollbackFn* = proc() {.noRaise.}
CoreDbTxDisposeFn* = proc() {.noRaise.}
CoreDbTxFns* = object
levelFn*: CoreDbTxLevelFn
commitFn*: CoreDbTxCommitFn
rollbackFn*: CoreDbTxRollbackFn
disposeFn*: CoreDbTxDisposeFn
# --------------------------------------------------
# Sub-descriptor: capture recorder methods
# --------------------------------------------------
CoreDbCaptRecorderFn* = proc(): CoreDbRef {.noRaise.}
CoreDbCaptLogDbFn* = proc(): TableRef[Blob,Blob] {.noRaise.}
CoreDbCaptFlagsFn* = proc(): set[CoreDbCaptFlags] {.noRaise.}
CoreDbCaptForgetFn* = proc() {.noRaise.}
CoreDbCaptFns* = object
recorderFn*: CoreDbCaptRecorderFn
logDbFn*: CoreDbCaptLogDbFn
getFlagsFn*: CoreDbCaptFlagsFn
forgetFn*: CoreDbCaptForgetFn
# --------------------------------------------------
# Production descriptors
# --------------------------------------------------
CoreDbRef* = ref object of RootRef
CoreDbRef* = ref object
## Database descriptor
dbType*: CoreDbType ## Type of database backend
defCtx*: CoreDbCtxRef ## Default context
# Optional api interface (can be re-directed/intercepted)
ariApi*: AristoApiRef ## `Aristo` api
kvtApi*: KvtApiRef ## `KVT` api
# Optional profiling and debugging stuff
trackNewApi*: bool ## Debugging, support
trackLedgerApi*: bool ## Debugging, suggestion for subsequent ledger
profTab*: CoreDbProfListRef ## Profiling data (if any)
ledgerHook*: RootRef ## Debugging/profiling, to be used by ledger
methods*: CoreDbBaseFns
CoreDbErrorRef* = ref object of RootRef
CoreDbCtxRef* = ref object
## Shared context for `CoreDbMptRef`, `CoreDbAccRef`, `CoreDbKvtRef`
parent*: CoreDbRef
mpt*: AristoDbRef ## `Aristo` database
kvt*: KvtDbRef ## `KVT` key-value table
CoreDbKvtRef* = distinct CoreDbCtxRef
## Statically initialised Key-Value pair table
CoreDbAccRef* = distinct CoreDbCtxRef
## Similar to `CoreDbKvtRef`, only dealing with `Aristo` accounts
CoreDbMptRef* = distinct CoreDbCtxRef
## Generic MPT
CoreDbTxRef* = ref object
## Transaction descriptor
ctx*: CoreDbCtxRef ## Context (also contains `Aristo` descriptor)
aTx*: AristoTxRef ## `Aristo` transaction (if any)
kTx*: KvtTxRef ## `KVT` transaction (if any)
CoreDbErrorRef* = ref object
## Generic error object
error*: CoreDbErrorCode
parent*: CoreDbRef
ctx*: string ## Context where the exception or error occured
case isAristo*: bool
of true:
aErr*: AristoError
else:
kErr*: KvtError
CoreDbKvtBackendRef* = ref object of RootRef
## Backend wrapper for direct backend access
parent*: CoreDbRef
when false: # TODO
type
# --------------------------------------------------
# Sub-descriptor: capture recorder methods
# --------------------------------------------------
CoreDbCaptRecorderFn* = proc(): CoreDbRef {.noRaise.}
CoreDbCaptLogDbFn* = proc(): TableRef[Blob,Blob] {.noRaise.}
CoreDbCaptFlagsFn* = proc(): set[CoreDbCaptFlags] {.noRaise.}
CoreDbCaptForgetFn* = proc() {.noRaise.}
CoreDbMptBackendRef* = ref object of RootRef
## Backend wrapper for direct backend access
parent*: CoreDbRef
CoreDbCaptFns* = object
recorderFn*: CoreDbCaptRecorderFn
logDbFn*: CoreDbCaptLogDbFn
getFlagsFn*: CoreDbCaptFlagsFn
forgetFn*: CoreDbCaptForgetFn
CoreDbAccBackendRef* = ref object of RootRef
## Backend wrapper for direct backend access
parent*: CoreDbRef
CoreDbKvtRef* = ref object of RootRef
## Statically initialised Key-Value pair table living in `CoreDbRef`
parent*: CoreDbRef
methods*: CoreDbKvtFns
CoreDbCtxRef* = ref object of RootRef
## Context for `CoreDbMptRef` and `CoreDbAccRef`
parent*: CoreDbRef
methods*: CoreDbCtxFns
CoreDbMptRef* = ref object of RootRef
## Hexary/Merkle-Patricia tree derived from `CoreDbRef`, will be
## initialised on-the-fly.
parent*: CoreDbRef
methods*: CoreDbMptFns
CoreDbAccRef* = ref object of RootRef
## Similar to `CoreDbKvtRef`, only dealing with `CoreDbAccount` data
## rather than `Blob` values.
parent*: CoreDbRef
methods*: CoreDbAccFns
CoreDbTxRef* = ref object of RootRef
## Transaction descriptor derived from `CoreDbRef`
parent*: CoreDbRef
methods*: CoreDbTxFns
CoreDbCaptRef* = ref object
## Db transaction tracer derived from `CoreDbRef`
parent*: CoreDbRef
methods*: CoreDbCaptFns
CoreDbCaptRef* = ref object
## Db transaction tracer derived from `CoreDbRef`
parent*: CoreDbRef
methods*: CoreDbCaptFns
# ------------------------------------------------------------------------------
# End

120
nimbus/db/core_db/base/validate.nim
View File

@ -10,135 +10,47 @@
{.push raises: [].}
import
../../aristo,
./base_desc
type
EphemMethodsDesc =
CoreDbKvtBackendRef | CoreDbMptBackendRef | CoreDbAccBackendRef
MethodsDesc =
CoreDbKvtRef |
CoreDbCtxRef | CoreDbMptRef | CoreDbAccRef |
CoreDbTxRef |
CoreDbCaptRef
ValidateDesc* = MethodsDesc | EphemMethodsDesc | CoreDbErrorRef
ValidateSubDesc* = CoreDbCtxRef | CoreDbTxRef # | CoreDbCaptRef
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
proc validateMethodsDesc(base: CoreDbBaseFns) =
doAssert not base.destroyFn.isNil
doAssert not base.errorPrintFn.isNil
doAssert not base.levelFn.isNil
doAssert not base.newKvtFn.isNil
doAssert not base.newCtxFn.isNil
doAssert not base.newCtxFromTxFn.isNil
doAssert not base.swapCtxFn.isNil
doAssert not base.beginFn.isNil
# doAssert not base.newCaptureFn.isNil # currently disabled
doAssert not base.persistentFn.isNil
proc validateMethodsDesc(kvt: CoreDbKvtFns) =
doAssert not kvt.backendFn.isNil
doAssert not kvt.getFn.isNil
doAssert not kvt.lenFn.isNil
doAssert not kvt.delFn.isNil
doAssert not kvt.putFn.isNil
doAssert not kvt.hasKeyFn.isNil
doAssert not kvt.forgetFn.isNil
proc validateMethodsDesc(ctx: CoreDbCtxFns) =
doAssert not ctx.getAccountsFn.isNil
doAssert not ctx.getColumnFn.isNil
doAssert not ctx.forgetFn.isNil
proc validateMethodsDesc(fns: CoreDbMptFns) =
doAssert not fns.backendFn.isNil
doAssert not fns.fetchFn.isNil
doAssert not fns.deleteFn.isNil
doAssert not fns.mergeFn.isNil
doAssert not fns.hasPathFn.isNil
doAssert not fns.stateFn.isNil
proc validateMethodsDesc(fns: CoreDbAccFns) =
doAssert not fns.backendFn.isNil
doAssert not fns.fetchFn.isNil
doAssert not fns.clearStorageFn.isNil
doAssert not fns.deleteFn.isNil
doAssert not fns.hasPathFn.isNil
doAssert not fns.mergeFn.isNil
doAssert not fns.stateFn.isNil
doAssert not fns.slotFetchFn.isNil
doAssert not fns.slotDeleteFn.isNil
doAssert not fns.slotHasPathFn.isNil
doAssert not fns.slotMergeFn.isNil
doAssert not fns.slotStateFn.isNil
doAssert not fns.slotStateEmptyFn.isNil
# ------------
proc validateMethodsDesc(e: CoreDbErrorRef) =
doAssert e.error != CoreDbErrorCode(0)
doAssert not e.isNil
doAssert not e.parent.isNil
proc validateMethodsDesc(eph: EphemMethodsDesc) =
doAssert not eph.isNil
doAssert not eph.parent.isNil
proc validateMethodsDesc(kvt: CoreDbKvtRef) =
doAssert not kvt.isNil
doAssert not kvt.parent.isNil
kvt.methods.validateMethodsDesc
proc validateMethodsDesc(ctx: CoreDbCtxRef) =
proc validateSubDescRef(ctx: CoreDbCtxRef) =
doAssert not ctx.isNil
doAssert not ctx.parent.isNil
ctx.methods.validateMethodsDesc
doAssert not ctx.mpt.isNil
doAssert not ctx.kvt.isNil
proc validateMethodsDesc(mpt: CoreDbMptRef) =
doAssert not mpt.isNil
doAssert not mpt.parent.isNil
mpt.methods.validateMethodsDesc
proc validateMethodsDesc(acc: CoreDbAccRef) =
doAssert not acc.isNil
doAssert not acc.parent.isNil
acc.methods.validateMethodsDesc
proc validateSubDescRef(tx: CoreDbTxRef) =
doAssert not tx.isNil
doAssert not tx.ctx.isNil
doAssert not tx.aTx.isNil
doAssert not tx.kTx.isNil
when false: # currently disabled
proc validateMethodsDesc(cpt: CoreDbCaptRef) =
proc validateSubDescRef(cpt: CoreDbCaptRef) =
doAssert not cpt.isNil
doAssert not cpt.parent.isNil
doAssert not cpt.methods.recorderFn.isNil
doAssert not cpt.methods.getFlagsFn.isNil
doAssert not cpt.methods.forgetFn.isNil
proc validateMethodsDesc(tx: CoreDbTxRef) =
doAssert not tx.isNil
doAssert not tx.parent.isNil
doAssert not tx.methods.levelFn.isNil
doAssert not tx.methods.commitFn.isNil
doAssert not tx.methods.rollbackFn.isNil
doAssert not tx.methods.disposeFn.isNil
proc validateMethodsDesc(db: CoreDbRef) =
doAssert not db.isNil
doAssert db.dbType != CoreDbType(0)
db.methods.validateMethodsDesc
# ------------------------------------------------------------------------------
# Public debugging helpers
# ------------------------------------------------------------------------------
proc validate*(dsc: ValidateDesc) =
dsc.validateMethodsDesc
proc validate*(dsc: ValidateSubDesc) =
dsc.validateSubDescRef
proc validate*(db: CoreDbRef) =
db.validateMethodsDesc
doAssert not db.isNil
doAssert db.dbType != CoreDbType(0)
db.defCtx.validate
# ------------------------------------------------------------------------------
# End

51
nimbus/db/core_db/base_iterators.nim
View File

@ -13,10 +13,23 @@
import
std/typetraits,
eth/common,
./backend/aristo_db,
../aristo as use_ari,
../aristo/[aristo_walk, aristo_serialise],
../kvt as use_kvt,
../kvt/[kvt_init/memory_only, kvt_walk],
./base/[api_tracking, base_desc],
./base
when CoreDbEnableApiJumpTable:
discard
else:
import
../aristo/[aristo_desc, aristo_path, aristo_tx],
../kvt/[kvt_desc, kvt_tx]
include
./backend/aristo_replicate
when CoreDbEnableApiTracking:
import chronicles
@ -35,39 +48,43 @@ iterator pairs*(kvt: CoreDbKvtRef): (Blob, Blob) {.apiRaise.} =
## Iterator supported on memory DB (otherwise implementation dependent)
##
kvt.setTrackNewApi KvtPairsIt
case kvt.parent.dbType:
case kvt.dbType:
of AristoDbMemory:
for k,v in kvt.aristoKvtPairsMem():
let p = kvt.call(forkTx, kvt.kvt, 0).valueOrApiError "kvt/pairs()"
defer: discard kvt.call(forget, p)
for (k,v) in use_kvt.MemBackendRef.walkPairs p:
yield (k,v)
of AristoDbVoid:
for k,v in kvt.aristoKvtPairsVoid():
let p = kvt.call(forkTx, kvt.kvt, 0).valueOrApiError "kvt/pairs()"
defer: discard kvt.call(forget, p)
for (k,v) in use_kvt.VoidBackendRef.walkPairs p:
yield (k,v)
of Ooops, AristoDbRocks:
raiseAssert: "Unsupported database type: " & $kvt.parent.dbType
raiseAssert: "Unsupported database type: " & $kvt.dbType
kvt.ifTrackNewApi: debug newApiTxt, api, elapsed
iterator pairs*(mpt: CoreDbMptRef): (Blob, Blob) =
## Trie traversal, only supported for `CoreDbMptRef`
##
mpt.setTrackNewApi MptPairsIt
case mpt.parent.dbType:
case mpt.dbType:
of AristoDbMemory, AristoDbRocks, AristoDbVoid:
for k,v in mpt.aristoMptPairs():
yield (k,v)
for (path,data) in mpt.mpt.rightPairsGeneric mpt.rootID:
yield (mpt.call(pathAsBlob, path), data)
of Ooops:
raiseAssert: "Unsupported database type: " & $mpt.parent.dbType
raiseAssert: "Unsupported database type: " & $mpt.dbType
mpt.ifTrackNewApi: debug newApiTxt, api, elapsed
iterator slotPairs*(acc: CoreDbAccRef; accPath: Hash256): (Blob, Blob) =
## Trie traversal, only supported for `CoreDbMptRef`
##
acc.setTrackNewApi AccSlotPairsIt
case acc.parent.dbType:
case acc.dbType:
of AristoDbMemory, AristoDbRocks, AristoDbVoid:
for k,v in acc.aristoSlotPairs accPath:
yield (k,v)
for (path,data) in acc.mpt.rightPairsStorage accPath:
yield (acc.call(pathAsBlob, path), data)
of Ooops:
raiseAssert: "Unsupported database type: " & $acc.parent.dbType
raiseAssert: "Unsupported database type: " & $acc.dbType
acc.ifTrackNewApi:
doAssert accPath.len == 32
debug newApiTxt, api, elapsed
@ -76,15 +93,15 @@ iterator replicate*(mpt: CoreDbMptRef): (Blob, Blob) {.apiRaise.} =
## Low level trie dump, only supported for non persistent `CoreDbMptRef`
##
mpt.setTrackNewApi MptReplicateIt
case mpt.parent.dbType:
case mpt.dbType:
of AristoDbMemory:
for k,v in aristoReplicateMem(mpt):
for k,v in aristoReplicate[use_ari.MemBackendRef](mpt):
yield (k,v)
of AristoDbVoid:
for k,v in aristoReplicateVoid(mpt):
for k,v in aristoReplicate[use_ari.VoidBackendRef](mpt):
yield (k,v)
of Ooops, AristoDbRocks:
raiseAssert: "Unsupported database type: " & $mpt.parent.dbType
raiseAssert: "Unsupported database type: " & $mpt.dbType
mpt.ifTrackNewApi: debug newApiTxt, api, elapsed
# ------------------------------------------------------------------------------

22
nimbus/db/core_db/base_iterators_persistent.nim
View File

@ -13,10 +13,16 @@
import
std/typetraits,
eth/common,
./backend/[aristo_db, aristo_rocksdb],
../aristo as use_ari,
../aristo/aristo_init/rocks_db,
../aristo/[aristo_desc, aristo_walk/persistent, aristo_tx],
../kvt, # needed for `aristo_replicate`
./base/[api_tracking, base_desc],
./base
include
./backend/aristo_replicate
when CoreDbEnableApiTracking:
import chronicles
@ -25,7 +31,7 @@ when CoreDbEnableApiTracking:
newApiTxt = logTxt & "API"
# Annotation helper(s)
{.pragma: rlpRaise, gcsafe, raises: [AristoApiRlpError].}
{.pragma: rlpRaise, gcsafe, raises: [CoreDbApiError].}
# ------------------------------------------------------------------------------
# Public iterators
@ -35,18 +41,18 @@ iterator replicatePersistent*(mpt: CoreDbMptRef): (Blob, Blob) {.rlpRaise.} =
## Extended version of `replicate()` for `Aristo` persistent backend.
##
mpt.setTrackNewApi MptReplicateIt
case mpt.parent.dbType:
case mpt.dbType:
of AristoDbMemory:
for k,v in aristoReplicateMem(mpt):
for k,v in aristoReplicate[use_ari.MemBackendRef](mpt):
yield (k,v)
of AristoDbVoid:
for k,v in aristoReplicateVoid(mpt):
for k,v in aristoReplicate[use_ari.VoidBackendRef](mpt):
yield (k,v)
of AristoDbRocks:
for k,v in aristoReplicateRdb(mpt):
yield (k,v)
for k, v in aristoReplicate[rocks_db.RdbBackendRef](mpt):
yield (k, v)
else:
raiseAssert: "Unsupported database type: " & $mpt.parent.dbType
raiseAssert: "Unsupported database type: " & $mpt.dbType
mpt.ifTrackNewApi: debug newApiTxt, api, elapsed
# ------------------------------------------------------------------------------

10
nimbus/db/core_db/core_apps.nim
View File

@ -310,15 +310,7 @@ proc getSavedStateBlockNumber*(
## Returns the block number registered when the database was last time
## updated, or `BlockNumber(0)` if there was no updata found.
##
## This function verifies the state consistency of the database and throws
## an assert exception if that fails. So the function will only apply to a
## saved database state. For an an opportunistic use, the `relax` argument
## can be set `true` so this function also returns the block number if the
## state consistency check fails.
##
# FIXME: This construct following will be replaced by a proper
# `CoreDb` method.
db.ctx.getColumn(CtGeneric).backend.toAristoSavedStateBlockNumber()
db.stateBlockNumber()
proc getBlockHeader*(
db: CoreDbRef;

15
nimbus/db/core_db/memory_only.nim
View File

@ -24,20 +24,7 @@ export
base,
base_iterators,
common,
core_apps,
# see `aristo_db`
isAristo,
toAristo,
toAristoProfData,
toAristoSavedStateBlockNumber,
# Standard interface for calculating merkle hash signatures (see `aristo`)
MerkleSignRef,
merkleSignBegin,
merkleSignAdd,
merkleSignCommit,
to
core_apps
# ------------------------------------------------------------------------------
# Public constructors

1
nimbus/db/kvt.nim
View File

@ -23,6 +23,7 @@ import
export
MemBackendRef,
VoidBackendRef,
finish,
init
import

2
nimbus/db/kvt/kvt_tx.nim
View File

@ -32,7 +32,7 @@ func isTop*(tx: KvtTxRef): bool =
## level transaction.
tx.txFrameIsTop()
func level*(tx: KvtTxRef): int =
func txLevel*(tx: KvtTxRef): int =
## Getter, positive nesting level of transaction argument `tx`
tx.txFrameLevel()

3
nimbus/db/ledger/base/api_tracking.nim
View File

@ -97,9 +97,6 @@ func toStr*(w: EthAddress): string =
func toStr*(w: Hash256): string =
w.data.oaToStr
func toStr*(w: CoreDbMptRef): string =
if w.CoreDbMptRef.isNil: "nil" else: "MptRef"
func toStr*(w: CodeBytesRef): string =
if w.CodeBytesRef.isNil: "nil"
else: "[" & $w.bytes.len & "]"

2
tests/test_coredb.nim
View File

@ -357,7 +357,7 @@ when isMainModule:
#dbType = CdbAristoDualRocks,
capture = capture,
pruneHistory = true,
#profilingOk = true,
profilingOk = true,
#finalDiskCleanUpOk = false,
oldLogAlign = true
)

6
tests/test_coredb/test_chainsync.nim
View File

@ -184,9 +184,9 @@ proc test_chainSync*(
# API data need to be captured so it will be available after the services
# have terminated.
when CoreDbEnableApiProfiling:
# terminated.
(aristoProfData, kvtProfData) = com.db.toAristoProfData()
cdbProfData = com.db.dbProfData()
aristoProfData = com.db.ariApi.AristoApiProfRef.data
kvtProfData = com.db.kvtApi.KvtApiProfRef.data
cdbProfData = com.db.profTab
when LedgerEnableApiProfiling:
ldgProfData = com.db.ldgProfData()