From ea7c756a9deaa73eceb31cf62d7864272c6f7d4b Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Wed, 3 Jul 2024 15:50:27 +0000 Subject: [PATCH] Core db reorg (#2444) * CoreDb: Merged all sub-descriptors into `base_desc` module * Dissolve `aristo_db/common_desc.nim` * No need to export `Aristo` methods in `CoreDb` * Resolve/tighten methods in `aristo_db` sub-moduled why: So they can be straihgt implemented into the `base` module * Moved/re-implemented `KVT` methods into `base` module * Moved/re-implemented `MPT` methods into `base` module * Moved/re-implemented account methods into `base` module * Moved/re-implemented `CTX` methods into `base` module * Moved/re-implemented `handler_{aristo,kvt}` into `aristo_db` module * Moved/re-implemented `TX` methods into `base` module * Moved/re-implemented base methods into `base` module * Replaced `toAristoSavedStateBlockNumber()` by proper base method why: Was the last for keeping reason for keeping low level backend access methods * Remove dedicated low level access to `Aristo` backend why: Not needed anymore, for debugging the descriptors can be accessed directly also: some clean up stuff * Re-factor `CoreDb` descriptor layout and adjust base methods * Moved/re-implemented iterators into `base_iterator*` modules * Update docu --- nimbus/db/aristo.nim | 1 + nimbus/db/aristo/aristo_api.nim | 18 +- nimbus/db/aristo/aristo_tx.nim | 2 +- nimbus/db/aristo/aristo_utils.nim | 2 +- nimbus/db/core_db/README.md | 96 +-- nimbus/db/core_db/TODO.md | 7 +- nimbus/db/core_db/backend/aristo_db.nim | 289 +------- nimbus/db/core_db/backend/aristo_db/TODO.md | 3 - .../backend/aristo_db/aristo_replicate.nim | 40 - .../core_db/backend/aristo_db/common_desc.nim | 62 -- .../backend/aristo_db/handlers_aristo.nim | 580 --------------- .../backend/aristo_db/handlers_kvt.nim | 266 ------- .../db/core_db/backend/aristo_replicate.nim | 76 ++ nimbus/db/core_db/backend/aristo_rocksdb.nim | 15 +- .../db/core_db/backend/todo/free_parking.nim | 134 ++++ .../{aristo_db => todo}/handlers_trace.nim | 0 nimbus/db/core_db/base.nim | 700 ++++++++++++------ nimbus/db/core_db/base/api_tracking.nim | 37 +- nimbus/db/core_db/base/base_desc.nim | 295 ++------ nimbus/db/core_db/base/validate.nim | 120 +-- nimbus/db/core_db/base_iterators.nim | 51 +- .../db/core_db/base_iterators_persistent.nim | 22 +- nimbus/db/core_db/core_apps.nim | 10 +- nimbus/db/core_db/memory_only.nim | 15 +- nimbus/db/kvt.nim | 1 + nimbus/db/kvt/kvt_tx.nim | 2 +- nimbus/db/ledger/base/api_tracking.nim | 3 - tests/test_coredb.nim | 2 +- tests/test_coredb/test_chainsync.nim | 6 +- 29 files changed, 874 insertions(+), 1981 deletions(-) delete mode 100644 nimbus/db/core_db/backend/aristo_db/TODO.md delete mode 100644 nimbus/db/core_db/backend/aristo_db/aristo_replicate.nim delete mode 100644 nimbus/db/core_db/backend/aristo_db/common_desc.nim delete mode 100644 nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim delete mode 100644 nimbus/db/core_db/backend/aristo_db/handlers_kvt.nim create mode 100644 nimbus/db/core_db/backend/aristo_replicate.nim create mode 100644 nimbus/db/core_db/backend/todo/free_parking.nim rename nimbus/db/core_db/backend/{aristo_db => todo}/handlers_trace.nim (100%) diff --git a/nimbus/db/aristo.nim b/nimbus/db/aristo.nim index f5976c9a4..87e4d5886 100644 --- a/nimbus/db/aristo.nim +++ b/nimbus/db/aristo.nim @@ -23,6 +23,7 @@ import export MemBackendRef, VoidBackendRef, + finish, init import diff --git a/nimbus/db/aristo/aristo_api.nim b/nimbus/db/aristo/aristo_api.nim index a5a667ffe..8d5dd07fc 100644 --- a/nimbus/db/aristo/aristo_api.nim +++ b/nimbus/db/aristo/aristo_api.nim @@ -384,7 +384,7 @@ type ## is returned if there was any. AristoApiTxBeginFn* = - proc(db: AristoDbRef + proc(db: AristoDbRef; ): Result[AristoTxRef,AristoError] {.noRaise.} ## Starts a new transaction. @@ -397,6 +397,12 @@ type ## ... continue using db ... ## tx.commit() + AristoApiTxLevelFn* = + proc(tx: AristoTxRef; + ): int + {.noRaise.} + ## Getter, positive nesting level of transaction argument `tx` + AristoApiTxTopFn* = proc(db: AristoDbRef; ): Result[AristoTxRef,AristoError] @@ -445,6 +451,7 @@ type reCentre*: AristoApiReCentreFn rollback*: AristoApiRollbackFn txBegin*: AristoApiTxBeginFn + txLevel*: AristoApiTxLevelFn txTop*: AristoApiTxTopFn @@ -491,6 +498,7 @@ type AristoApiProfReCentreFn = "reCentre" AristoApiProfRollbackFn = "rollback" AristoApiProfTxBeginFn = "txBegin" + AristoApiProfTxLevelFn = "txLevel" AristoApiProfTxTopFn = "txTop" AristoApiProfBeGetVtxFn = "be/getVtx" @@ -554,6 +562,7 @@ when AutoValidateApiHooks: doAssert not api.reCentre.isNil doAssert not api.rollback.isNil doAssert not api.txBegin.isNil + doAssert not api.txLevel.isNil doAssert not api.txTop.isNil proc validate(prf: AristoApiProfRef) = @@ -621,6 +630,7 @@ func init*(api: var AristoApiObj) = api.reCentre = reCentre api.rollback = rollback api.txBegin = txBegin + api.txLevel = txLevel api.txTop = txTop when AutoValidateApiHooks: api.validate @@ -670,6 +680,7 @@ func dup*(api: AristoApiRef): AristoApiRef = reCentre: api.reCentre, rollback: api.rollback, txBegin: api.txBegin, + txLevel: api.txLevel, txTop: api.txTop) when AutoValidateApiHooks: api.validate @@ -861,6 +872,11 @@ func init*( AristoApiProfTxBeginFn.profileRunner: result = api.txBegin(a) + profApi.txLevel = + proc(a: AristoTxRef): auto = + AristoApiProfTxLevelFn.profileRunner: + result = api.txLevel(a) + profApi.txTop = proc(a: AristoDbRef): auto = AristoApiProfTxTopFn.profileRunner: diff --git a/nimbus/db/aristo/aristo_tx.nim b/nimbus/db/aristo/aristo_tx.nim index 0c4adcc7f..ec6ddd554 100644 --- a/nimbus/db/aristo/aristo_tx.nim +++ b/nimbus/db/aristo/aristo_tx.nim @@ -31,7 +31,7 @@ func isTop*(tx: AristoTxRef): bool = ## level transaction. tx.txFrameIsTop() -func level*(tx: AristoTxRef): int = +func txLevel*(tx: AristoTxRef): int = ## Getter, positive nesting level of transaction argument `tx` tx.txFrameLevel() diff --git a/nimbus/db/aristo/aristo_utils.nim b/nimbus/db/aristo/aristo_utils.nim index cbe8dc2c9..609fe1ca4 100644 --- a/nimbus/db/aristo/aristo_utils.nim +++ b/nimbus/db/aristo/aristo_utils.nim @@ -14,7 +14,7 @@ {.push raises: [].} import - std/[sequtils, typetraits], + std/sequtils, eth/common, results, "."/[aristo_constants, aristo_desc, aristo_get, aristo_hike, aristo_layers] diff --git a/nimbus/db/core_db/README.md b/nimbus/db/core_db/README.md index 8700be252..d074d51e6 100644 --- a/nimbus/db/core_db/README.md +++ b/nimbus/db/core_db/README.md @@ -1,84 +1,26 @@ -Core database replacement wrapper object -======================================== -This wrapper replaces the *TrieDatabaseRef* and its derivatives by the new -object *CoreDbRef*. +Core database +============= -# **out of date** +Layout of `CoreDb` descriptor objects +------------------------------------- +### Objects dependence: -Relations to current *TrieDatabaseRef* implementation ------------------------------------------------------ -Here are some incomplete translations for objects and constructors. - -### Object types: - -| **Legacy notation** | **CoreDbRef based replacement** | -|:----------------------------|:--------------------------------------| -| | | -| ChainDB | (don't use/avoid) | -| ChainDbRef | CoreDbRef | -| TrieDatabaseRef | CoreDbKvtRef | -| HexaryTrie | CoreDbMptRef | -| SecureHexaryTrie | CoreDbPhkRef | -| DbTransaction | CoreDbTxRef | -| TransactionID | CoreDbTxID | - - -### Constructors: - -| **Legacy notation** | **CoreDbRef based replacement** | -|:----------------------------|:--------------------------------------| -| | | -| trieDB newChainDB("..") | newCoreDbRef(LegacyDbPersistent,"..") | -| newMemoryDB() | newCoreDbRef(LegacyDbMemory) | -| -- | | -| initHexaryTrie(db,..) | db.mpt(..) (no pruning) | -| | db.mptPrune(..) (w/pruning true/false)| -| -- | | -| initSecureHexaryTrie(db,..) | db.phk(..) (no pruning) | -| | db.phkPrune(..) (w/pruning true/false)| -| -- | | -| newCaptureDB(db,memDB) | db.capture() (see below) | - - -Usage of the replacement wrapper --------------------------------- - -### Objects pedigree: - - CoreDbRef -- base descriptor + CoreDbRef -- Base descriptor | | | - | | +--- CoreDbCtxRef -- MPT context descriptor - | | | | - | | | +-- CoreDbMptRef -- hexary trie instance - | | | | : : - | | | +-- CoreDbMptRef -- hexary trie instance - | | | - | | | - | | +---- CoreDbPhkRef -- pre-hashed key hexary trie instance - | | | : : - | | +---- CoreDbPhkRef -- pre-hashed key hexary trie instance + | | +--- CoreDbCtxRef -- Context descriptor + | | | | | | + | | | | | +--- CoreDbKvtRef -- Key-value table + | | | | | + | | | | +----- CoreDbMptRef -- Generic MPT + | | | | + | | | +------- CoreDbAccRef -- Accounts database + | | | + | | +--------- CoreDbTxRef -- Transaction handle | | - | | - | +------ CoreDbKvtRef -- single static key-value table + | +----- CoreDbCtxRef + | : : : : | - | - +-------- CoreDbCaptRef -- tracer support descriptor + +------- CoreDbCtxRef + : : : : -### Instantiating legacy standard database object descriptors works as follows: - - let - db = newCoreDbRef(..) # new base descriptor - mpt = db.mpt(..) # hexary trie/Merkle Patricia Tree - phk = db.phk(..) # pre-hashed key hexary trie/MPT - kvt = db.kvt # key-value table - -### Tracer support setup by hiding the current *CoreDbRef* behind a replacement: - - let - capture = db.capture() - db = capture.recorder # use the recorder in place of db - ... - - for key,value in capture.recorder.kvt: - ... # process recorded data diff --git a/nimbus/db/core_db/TODO.md b/nimbus/db/core_db/TODO.md index c1ae877cb..f27020078 100644 --- a/nimbus/db/core_db/TODO.md +++ b/nimbus/db/core_db/TODO.md @@ -1,4 +1,3 @@ -* Re-implement *getOldestJournalBlockNumber()* and - *getLatestJournalBlockNumber()* (from the `core_apps` module) via the CoreDb - base api. Currently this api is bypassed (via the *backend()* directive). The - functionality is directly provided by the `Aristo` backend. +* Refactor `handlers_tracer`. This module can reliably work only as a genuine + logger. The restore features were ill concieved, an attempt to be as close + as possible to the legacy tracer. diff --git a/nimbus/db/core_db/backend/aristo_db.nim b/nimbus/db/core_db/backend/aristo_db.nim index 207c04367..d4e27cee0 100644 --- a/nimbus/db/core_db/backend/aristo_db.nim +++ b/nimbus/db/core_db/backend/aristo_db.nim @@ -11,196 +11,38 @@ {.push raises: [].} import - std/tables, - eth/common, ../../aristo as use_ari, - ../../aristo/[aristo_walk, aristo_serialise], + ../../aristo/[aristo_init/memory_only, aristo_walk], ../../kvt as use_kvt, ../../kvt/[kvt_init/memory_only, kvt_walk], - ".."/[base, base/base_desc], - ./aristo_db/[common_desc, handlers_aristo, handlers_kvt] - -import - ../../aristo/aristo_init/memory_only as aristo_memory_only - -# Caveat: -# additional direct include(s) -- not import(s) -- is placed near -# the end of this source file - -# Annotation helper(s) -{.pragma: noRaise, gcsafe, raises: [].} -{.pragma: rlpRaise, gcsafe, raises: [AristoApiRlpError].} - -export - AristoApiRlpError, - AristoCoreDbKvtBE, - isAristo - -type - AristoCoreDbRef* = ref object of CoreDbRef - ## Main descriptor - kdbBase: KvtBaseRef ## Kvt subsystem - adbBase: AristoBaseRef ## Aristo subsystem - #tracer: AristoTracerRef ## Currently active recorder - - #AristoTracerRef = ref object of TraceRecorderRef - # ## Sub-handle for tracer - # parent: AristoCoreDbRef - -proc newAristoVoidCoreDbRef*(): CoreDbRef {.noRaise.} - -# ------------------------------------------------------------------------------ -# Private tx and base methods -# ------------------------------------------------------------------------------ - -proc txMethods( - db: AristoCoreDbRef; - aTx: AristoTxRef; - kTx: KvtTxRef; - ): CoreDbTxFns = - ## To be constructed by some `CoreDbBaseFns` function - let - adbBase = db.adbBase - kdbBase = db.kdbBase - - adbApi = adbBase.api - kdbApi = kdbBase.api - - CoreDbTxFns( - levelFn: proc(): int = - aTx.level, - - commitFn: proc() = - const info = "commitFn()" - adbApi.commit(aTx).isOkOr: - raiseAssert info & ": " & $error - kdbApi.commit(kTx).isOkOr: - raiseAssert info & ": " & $error - discard, - - rollbackFn: proc() = - const info = "rollbackFn()" - adbApi.rollback(aTx).isOkOr: - raiseAssert info & ": " & $error - kdbApi.rollback(kTx).isOkOr: - raiseAssert info & ": " & $error - discard, - - disposeFn: proc() = - const info = "disposeFn()" - if adbApi.isTop(aTx): - adbApi.rollback(aTx).isOkOr: - raiseAssert info & ": " & $error - if kdbApi.isTop(kTx): - kdbApi.rollback(kTx).isOkOr: - raiseAssert info & ": " & $error - discard) - -when false: # currently disabled - proc cptMethods( - tracer: AristoTracerRef; - ): CoreDbCaptFns = - let - tr = tracer # So it can savely be captured - db = tr.parent # Will not change and can be captured - log = tr.topInst() # Ditto - - CoreDbCaptFns( - recorderFn: proc(): CoreDbRef = - db, - - logDbFn: proc(): TableRef[Blob,Blob] = - log.kLog, - - getFlagsFn: proc(): set[CoreDbCaptFlags] = - log.flags, - - forgetFn: proc() = - if not tracer.pop(): - tr.parent.tracer = AristoTracerRef(nil) - tr.restore()) - - -proc baseMethods(db: AristoCoreDbRef): CoreDbBaseFns = - let - aBase = db.adbBase - kBase = db.kdbBase - - when false: # currently disabled - proc tracerSetup(flags: set[CoreDbCaptFlags]): CoreDbCaptRef = - if db.tracer.isNil: - db.tracer = AristoTracerRef(parent: db) - db.tracer.init(kBase, aBase, flags) - else: - db.tracer.push(flags) - CoreDbCaptRef(methods: db.tracer.cptMethods) - - proc persistent(bn: Opt[BlockNumber]): CoreDbRc[void] = - const info = "persistentFn()" - let sid = - if bn.isNone: 0u64 - else: bn.unsafeGet - ? kBase.persistent info - ? aBase.persistent(sid, info) - ok() - - CoreDbBaseFns( - destroyFn: proc(eradicate: bool) = - aBase.destroy(eradicate) - kBase.destroy(eradicate), - - levelFn: proc(): int = - aBase.getLevel, - - errorPrintFn: proc(e: CoreDbErrorRef): string = - e.errorPrint(), - - newKvtFn: proc(): CoreDbRc[CoreDbKvtRef] = - kBase.newKvtHandler("newKvtFn()"), - - newCtxFn: proc(): CoreDbCtxRef = - aBase.ctx, - - newCtxFromTxFn: proc(r: Hash256; k: CoreDbColType): CoreDbRc[CoreDbCtxRef] = - CoreDbCtxRef.init(db.adbBase, r, k), - - swapCtxFn: proc(ctx: CoreDbCtxRef): CoreDbCtxRef = - aBase.swapCtx(ctx), - - beginFn: proc(): CoreDbTxRef = - const info = "beginFn()" - let - aTx = aBase.txBegin info - kTx = kBase.txBegin info - dsc = CoreDbTxRef(methods: db.txMethods(aTx, kTx)) - db.bless(dsc), - - # # currently disabled - # newCaptureFn: proc(flags:set[CoreDbCaptFlags]): CoreDbRc[CoreDbCaptRef] = - # ok(db.bless flags.tracerSetup()), - - persistentFn: proc(bn: Opt[BlockNumber]): CoreDbRc[void] = - persistent(bn)) + ".."/[base, base/base_desc] # ------------------------------------------------------------------------------ # Public constructor and helper # ------------------------------------------------------------------------------ -proc create*(dbType: CoreDbType; kdb: KvtDbRef; adb: AristoDbRef): CoreDbRef = +proc create*(dbType: CoreDbType; kvt: KvtDbRef; mpt: AristoDbRef): CoreDbRef = ## Constructor helper + var db = CoreDbRef(dbType: dbType) + db.defCtx = db.bless CoreDbCtxRef(mpt: mpt, kvt: kvt) - # Local extensions - var db = AristoCoreDbRef() - db.adbBase = AristoBaseRef.init(db, adb) - db.kdbBase = KvtBaseRef.init(db, kdb) + when CoreDbEnableApiTracking: + db.kvtApi = KvtApiRef.init() + db.ariApi = AristoApiRef.init() - # Base descriptor - db.dbType = dbType - db.methods = db.baseMethods() - db.bless() + when CoreDbEnableApiProfiling: + block: + let profApi = KvtApiProfRef.init(db.kvtApi, kvt.backend) + db.kvtApi = profApi + kvt.backend = profApi.be + block: + let profApi = AristoApiProfRef.init(db.ariApi, mpt.backend) + db.ariApi = profApi + mpt.backend = profApi.be + bless db proc newAristoMemoryCoreDbRef*(): CoreDbRef = - AristoDbMemory.create( + result = AristoDbMemory.create( KvtDbRef.init(use_kvt.MemBackendRef), AristoDbRef.init(use_ari.MemBackendRef)) @@ -209,99 +51,6 @@ proc newAristoVoidCoreDbRef*(): CoreDbRef = KvtDbRef.init(use_kvt.VoidBackendRef), AristoDbRef.init(use_ari.VoidBackendRef)) -# ------------------------------------------------------------------------------ -# Public helpers, e.g. for direct backend access -# ------------------------------------------------------------------------------ - -func toAristoProfData*( - db: CoreDbRef; - ): tuple[aristo: AristoDbProfListRef, kvt: KvtDbProfListRef] = - when CoreDbEnableApiProfiling: - if db.isAristo: - result.aristo = db.AristoCoreDbRef.adbBase.api.AristoApiProfRef.data - result.kvt = db.AristoCoreDbRef.kdbBase.api.KvtApiProfRef.data - -func toAristoApi*(kvt: CoreDbKvtRef): KvtApiRef = - if kvt.parent.isAristo: - return AristoCoreDbRef(kvt.parent).kdbBase.api - -func toAristoApi*(mpt: CoreDbMptRef): AristoApiRef = - if mpt.parent.isAristo: - return mpt.to(AristoApiRef) - -func toAristo*(kBe: CoreDbKvtBackendRef): KvtDbRef = - if not kBe.isNil and kBe.parent.isAristo: - return kBe.AristoCoreDbKvtBE.kdb - -func toAristo*(mBe: CoreDbMptBackendRef): AristoDbRef = - if not mBe.isNil and mBe.parent.isAristo: - return mBe.AristoCoreDbMptBE.adb - -func toAristo*(mBe: CoreDbAccBackendRef): AristoDbRef = - if not mBe.isNil and mBe.parent.isAristo: - return mBe.AristoCoreDbAccBE.adb - -proc toAristoSavedStateBlockNumber*( - mBe: CoreDbMptBackendRef; - ): BlockNumber = - if not mBe.isNil and mBe.parent.isAristo: - let rc = mBe.parent.AristoCoreDbRef.adbBase.getSavedState() - if rc.isOk: - return rc.value.serial.BlockNumber - -# ------------------------------------------------------------------------------ -# Public aristo iterators -# ------------------------------------------------------------------------------ - -include - ./aristo_db/aristo_replicate - -# ------------------------ - -iterator aristoKvtPairsVoid*(dsc: CoreDbKvtRef): (Blob,Blob) {.rlpRaise.} = - let - api = dsc.toAristoApi() - p = api.forkTx(dsc.to(KvtDbRef),0).valueOrApiError "aristoKvtPairs()" - defer: discard api.forget(p) - for (k,v) in use_kvt.VoidBackendRef.walkPairs p: - yield (k,v) - -iterator aristoKvtPairsMem*(dsc: CoreDbKvtRef): (Blob,Blob) {.rlpRaise.} = - let - api = dsc.toAristoApi() - p = api.forkTx(dsc.to(KvtDbRef),0).valueOrApiError "aristoKvtPairs()" - defer: discard api.forget(p) - for (k,v) in use_kvt.MemBackendRef.walkPairs p: - yield (k,v) - -iterator aristoMptPairs*(dsc: CoreDbMptRef): (Blob,Blob) {.noRaise.} = - let - api = dsc.to(AristoApiRef) - mpt = dsc.to(AristoDbRef) - for (path,data) in mpt.rightPairsGeneric dsc.rootID: - yield (api.pathAsBlob(path), data) - -iterator aristoSlotPairs*( - dsc: CoreDbAccRef; - accPath: Hash256; - ): (Blob,Blob) - {.noRaise.} = - let - api = dsc.to(AristoApiRef) - mpt = dsc.to(AristoDbRef) - for (path,data) in mpt.rightPairsStorage accPath: - yield (api.pathAsBlob(path), data) - -iterator aristoReplicateMem*(dsc: CoreDbMptRef): (Blob,Blob) {.rlpRaise.} = - ## Instantiation for `MemBackendRef` - for k,v in aristoReplicate[use_ari.MemBackendRef](dsc): - yield (k,v) - -iterator aristoReplicateVoid*(dsc: CoreDbMptRef): (Blob,Blob) {.rlpRaise.} = - ## Instantiation for `VoidBackendRef` - for k,v in aristoReplicate[use_ari.VoidBackendRef](dsc): - yield (k,v) - # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/backend/aristo_db/TODO.md b/nimbus/db/core_db/backend/aristo_db/TODO.md deleted file mode 100644 index f27020078..000000000 --- a/nimbus/db/core_db/backend/aristo_db/TODO.md +++ /dev/null @@ -1,3 +0,0 @@ -* Refactor `handlers_tracer`. This module can reliably work only as a genuine - logger. The restore features were ill concieved, an attempt to be as close - as possible to the legacy tracer. diff --git a/nimbus/db/core_db/backend/aristo_db/aristo_replicate.nim b/nimbus/db/core_db/backend/aristo_db/aristo_replicate.nim deleted file mode 100644 index ae8116b25..000000000 --- a/nimbus/db/core_db/backend/aristo_db/aristo_replicate.nim +++ /dev/null @@ -1,40 +0,0 @@ -# Nimbus -# Copyright (c) 2023-2024 Status Research & Development GmbH -# Licensed under either of -# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or -# http://www.apache.org/licenses/LICENSE-2.0) -# * MIT license ([LICENSE-MIT](LICENSE-MIT) or -# http://opensource.org/licenses/MIT) -# at your option. This file may not be copied, modified, or distributed except -# according to those terms. - -## Generic iterator, prototype to be included (rather than imported). Using -## an include file avoids duplicating code because the `T` argument is not -## bound to any object type. Otherwise all object types would be required -## when providing this iterator for import. -## -## This is not wanted here, because the import of a **pesistent** object -## would always require extra linking. - -template valueOrApiError[U,V](rc: Result[U,V]; info: static[string]): U = - rc.valueOr: raise (ref AristoApiRlpError)(msg: info) - -iterator aristoReplicate[T]( - dsc: CoreDbMptRef; - ): (Blob,Blob) - {.gcsafe, raises: [AristoApiRlpError].} = - ## Generic iterator used for building dedicated backend iterators. - ## - let - root = dsc.rootID - mpt = dsc.to(AristoDbRef) - api = dsc.to(AristoApiRef) - p = api.forkTx(mpt,0).valueOrApiError "aristoReplicate()" - defer: discard api.forget(p) - for (vid,key,vtx,node) in T.replicate(p): - if key.len == 32: - yield (@(key.data), node.encode) - elif vid == root: - yield (@(key.to(Hash256).data), node.encode) - -# End diff --git a/nimbus/db/core_db/backend/aristo_db/common_desc.nim b/nimbus/db/core_db/backend/aristo_db/common_desc.nim deleted file mode 100644 index 69a7b698d..000000000 --- a/nimbus/db/core_db/backend/aristo_db/common_desc.nim +++ /dev/null @@ -1,62 +0,0 @@ -# Nimbus -# Copyright (c) 2023-2024 Status Research & Development GmbH -# Licensed under either of -# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or -# http://www.apache.org/licenses/LICENSE-2.0) -# * MIT license ([LICENSE-MIT](LICENSE-MIT) or -# http://opensource.org/licenses/MIT) -# at your option. This file may not be copied, modified, or distributed except -# according to those terms. - -{.push raises: [].} - -import - std/strutils, - ../../../../errors, - "../../.."/[aristo, kvt], - ../../base/base_desc - -type - AristoApiRlpError* = object of CoreDbApiError - ## For re-routing exceptions in iterator closure - - AristoCoreDbError* = ref object of CoreDbErrorRef - ## Error return code - ctx*: string ## Context where the exception or error occured - case isAristo*: bool - of true: - vid*: VertexID - aErr*: AristoError - else: - kErr*: KvtError - -# ------------------------------------------------------------------------------ -# Public helpers -# ------------------------------------------------------------------------------ - -func isAristo*(db: CoreDbRef): bool = - db.dbType in {AristoDbMemory, AristoDbRocks, AristoDbVoid} - -func toStr*(n: VertexID): string = - result = "$" - if n.isValid: - result &= n.uint64.toHex.strip( - leading=true, trailing=false, chars={'0'}).toLowerAscii - else: - result &= "ø" - -func errorPrint*(e: CoreDbErrorRef): string = - if not e.isNil: - let e = e.AristoCoreDbError - result = if e.isAristo: "Aristo" else: "Kvt" - result &= ", ctx=" & $e.ctx & ", " - if e.isAristo: - if e.vid.isValid: - result &= "vid=" & e.vid.toStr & ", " - result &= "error=" & $e.aErr - else: - result &= "error=" & $e.kErr - -# ------------------------------------------------------------------------------ -# End -# ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim b/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim deleted file mode 100644 index 4e71aeb4d..000000000 --- a/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim +++ /dev/null @@ -1,580 +0,0 @@ -# Nimbus -# Copyright (c) 2023-2024 Status Research & Development GmbH -# Licensed under either of -# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or -# http://www.apache.org/licenses/LICENSE-2.0) -# * MIT license ([LICENSE-MIT](LICENSE-MIT) or -# http://opensource.org/licenses/MIT) -# at your option. This file may not be copied, modified, or distributed except -# according to those terms. - -{.push raises: [].} - -import - std/typetraits, - eth/common, - stew/byteutils, - ../../../aristo, - ../../../aristo/aristo_desc, - ../../base, - ../../base/base_desc, - ./common_desc - -type - AristoBaseRef* = ref object - parent: CoreDbRef ## Opaque top level descriptor - api*: AristoApiRef ## Api functions can be re-directed - ctx*: AristoCoreDbCtxRef ## Currently active context - - AristoCoreDbCtxRef* = ref object of CoreDbCtxRef - base: AristoBaseRef ## Local base descriptor - mpt*: AristoDbRef ## Aristo MPT database - - AristoCoreDbAccRef = ref object of CoreDbAccRef - base: AristoBaseRef ## Local base descriptor - - AristoCoreDbMptRef = ref object of CoreDbMptRef - base: AristoBaseRef ## Local base descriptor - mptRoot: VertexID ## State root, may be zero unless account - - AristoCoreDbMptBE* = ref object of CoreDbMptBackendRef - adb*: AristoDbRef - - AristoCoreDbAccBE* = ref object of CoreDbAccBackendRef - adb*: AristoDbRef - -static: - doAssert high(CoreDbColType).ord < LEAST_FREE_VID - -# ------------------------------------------------------------------------------ -# Private helpers -# ------------------------------------------------------------------------------ - -func toError( - e: AristoError; - base: AristoBaseRef; - info: string; - error = Unspecified; - ): CoreDbErrorRef = - base.parent.bless(error, AristoCoreDbError( - ctx: info, - isAristo: true, - aErr: e)) - -func toError( - e: (VertexID,AristoError); - base: AristoBaseRef; - info: string; - error = Unspecified; - ): CoreDbErrorRef = - base.parent.bless(error, AristoCoreDbError( - ctx: info, - isAristo: true, - vid: e[0], - aErr: e[1])) - -func toRc[T]( - rc: Result[T,AristoError]; - base: AristoBaseRef; - info: string; - error = Unspecified; - ): CoreDbRc[T] = - if rc.isOk: - when T is void: - return ok() - else: - return ok(rc.value) - err((VertexID(0),rc.error).toError(base, info, error)) - -# ------------------------------------------------------------------------------ -# Private `MPT` call back functions -# ------------------------------------------------------------------------------ - -proc mptMethods(): CoreDbMptFns = - # These templates are a hack to remove a closure environment that was using - # hundreds of mb of memory to have this syntactic convenience - # TODO remove methods / abstraction entirely - it is no longer needed - template base: untyped = cMpt.base - template db: untyped = base.parent # Ditto - template api: untyped = base.api # Ditto - template mpt: untyped = base.ctx.mpt # Ditto - - proc mptBackend(cMpt: AristoCoreDbMptRef): CoreDbMptBackendRef = - db.bless AristoCoreDbMptBE(adb: mpt) - - proc mptFetch(cMpt: AristoCoreDbMptRef, key: openArray[byte]): CoreDbRc[Blob] = - const info = "fetchFn()" - let data = api.fetchGenericData(mpt, cMpt.mptRoot, key).valueOr: - if error == FetchPathNotFound: - return err(error.toError(base, info, MptNotFound)) - return err(error.toError(base, info)) - ok(data) - - proc mptMerge(cMpt: AristoCoreDbMptRef, k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] = - const info = "mergeFn()" - api.mergeGenericData(mpt, cMpt.mptRoot, k, v).isOkOr: - return err(error.toError(base, info)) - ok() - - proc mptDelete(cMpt: AristoCoreDbMptRef, key: openArray[byte]): CoreDbRc[void] = - const info = "deleteFn()" - api.deleteGenericData(mpt, cMpt.mptRoot, key).isOkOr: - if error == DelPathNotFound: - return err(error.toError(base, info, MptNotFound)) - return err(error.toError(base, info)) - ok() - - proc mptHasPath(cMpt: AristoCoreDbMptRef, key: openArray[byte]): CoreDbRc[bool] = - const info = "hasPathFn()" - let yn = api.hasPathGeneric(mpt, cMpt.mptRoot, key).valueOr: - return err(error.toError(base, info)) - ok(yn) - - proc mptState(cMpt: AristoCoreDbMptRef, updateOk: bool): CoreDbRc[Hash256] = - const info = "mptState()" - let state = api.fetchGenericState(mpt, cMpt.mptRoot, updateOk).valueOr: - return err(error.toError(base, info)) - ok(state) - - ## Generic columns database handlers - CoreDbMptFns( - backendFn: proc(cMpt: CoreDbMptRef): CoreDbMptBackendRef = - mptBackend(AristoCoreDbMptRef(cMpt)), - - fetchFn: proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[Blob] = - mptFetch(AristoCoreDbMptRef(cMpt), k), - - deleteFn: proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[void] = - mptDelete(AristoCoreDbMptRef(cMpt), k), - - mergeFn: proc(cMpt: CoreDbMptRef, k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] = - mptMerge(AristoCoreDbMptRef(cMpt), k, v), - - hasPathFn: proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[bool] = - mptHasPath(AristoCoreDbMptRef(cMpt), k), - - stateFn: proc(cMpt: CoreDbMptRef, updateOk: bool): CoreDbRc[Hash256] = - mptState(AristoCoreDbMptRef(cMpt), updateOk)) - -# ------------------------------------------------------------------------------ -# Private account call back functions -# ------------------------------------------------------------------------------ - -proc accMethods(): CoreDbAccFns = - ## Account columns database handlers - template base: untyped = cAcc.base - template db: untyped = base.parent - template api: untyped = base.api - template mpt: untyped = base.ctx.mpt - - proc accBackend(cAcc: AristoCoreDbAccRef): CoreDbAccBackendRef = - db.bless AristoCoreDbAccBE(adb: mpt) - - proc accFetch( - cAcc: AristoCoreDbAccRef; - accPath: Hash256; - ): CoreDbRc[CoreDbAccount] = - const info = "acc/fetchFn()" - - let acc = api.fetchAccountRecord(mpt, accPath).valueOr: - if error != FetchPathNotFound: - return err(error.toError(base, info)) - return err(error.toError(base, info, AccNotFound)) - ok acc - - proc accMerge( - cAcc: AristoCoreDbAccRef; - accPath: Hash256; - accRec: CoreDbAccount; - ): CoreDbRc[void] = - const info = "acc/mergeFn()" - - let val = AristoAccount( - nonce: accRec.nonce, - balance: accRec.balance, - codeHash: accRec.codeHash) - api.mergeAccountRecord(mpt, accPath, val).isOkOr: - return err(error.toError(base, info)) - ok() - - proc accDelete( - cAcc: AristoCoreDbAccRef; - accPath: Hash256; - ): CoreDbRc[void] = - const info = "acc/deleteFn()" - - api.deleteAccountRecord(mpt, accPath).isOkOr: - if error == DelPathNotFound: - # TODO: Would it be conseqient to just return `ok()` here? - return err(error.toError(base, info, AccNotFound)) - return err(error.toError(base, info)) - ok() - - proc accClearStorage( - cAcc: AristoCoreDbAccRef; - accPath: Hash256; - ): CoreDbRc[void] = - const info = "acc/clearStoFn()" - - api.deleteStorageTree(mpt, accPath).isOkOr: - if error notin {DelStoRootMissing,DelStoAccMissing}: - return err(error.toError(base, info)) - ok() - - proc accHasPath( - cAcc: AristoCoreDbAccRef; - accPath: Hash256; - ): CoreDbRc[bool] = - const info = "hasPathFn()" - - let yn = api.hasPathAccount(mpt, accPath).valueOr: - return err(error.toError(base, info)) - ok(yn) - - proc accState( - cAcc: AristoCoreDbAccRef, - updateOk: bool; - ): CoreDbRc[Hash256] = - const info = "accStateFn()" - let state = api.fetchAccountState(mpt, updateOk).valueOr: - return err(error.toError(base, info)) - ok(state) - - - proc slotFetch( - cAcc: AristoCoreDbAccRef; - accPath: Hash256; - stoPath: openArray[byte]; - ): CoreDbRc[Blob] = - const info = "slotFetchFn()" - - let data = api.fetchStorageData(mpt, accPath, stoPath).valueOr: - if error != FetchPathNotFound: - return err(error.toError(base, info)) - return err(error.toError(base, info, StoNotFound)) - ok(data) - - proc slotDelete( - cAcc: AristoCoreDbAccRef; - accPath: Hash256; - stoPath: openArray[byte]; - ): CoreDbRc[void] = - const info = "slotDeleteFn()" - - api.deleteStorageData(mpt, accPath, stoPath).isOkOr: - if error == DelPathNotFound: - return err(error.toError(base, info, StoNotFound)) - if error == DelStoRootMissing: - # This is insane but legit. A storage column was announced for an - # account but no data have been added, yet. - return ok() - return err(error.toError(base, info)) - ok() - - proc slotHasPath( - cAcc: AristoCoreDbAccRef; - accPath: Hash256; - stoPath: openArray[byte]; - ): CoreDbRc[bool] = - const info = "slotHasPathFn()" - - let yn = api.hasPathStorage(mpt, accPath, stoPath).valueOr: - return err(error.toError(base, info)) - ok(yn) - - proc slotMerge( - cAcc: AristoCoreDbAccRef; - accPath: Hash256; - stoPath: openArray[byte]; - stoData: openArray[byte]; - ): CoreDbRc[void] = - const info = "slotMergeFn()" - - api.mergeStorageData(mpt, accPath, stoPath, stoData).isOkOr: - return err(error.toError(base, info)) - ok() - - proc slotState( - cAcc: AristoCoreDbAccRef; - accPath: Hash256; - updateOk: bool; - ): CoreDbRc[Hash256] = - const info = "slotStateFn()" - let state = api.fetchStorageState(mpt, accPath, updateOk).valueOr: - return err(error.toError(base, info)) - ok(state) - - proc slotStateEmpty( - cAcc: AristoCoreDbAccRef; - accPath: Hash256; - ): CoreDbRc[bool] = - const info = "slotStateEmptyFn()" - - let yn = api.hasStorageData(mpt, accPath).valueOr: - return err(error.toError(base, info)) - ok(not yn) - - - CoreDbAccFns( - backendFn: proc(cAcc: CoreDbAccRef): CoreDbAccBackendRef = - accBackend(AristoCoreDbAccRef(cAcc)), - - fetchFn: proc( - cAcc: CoreDbAccRef; - accPath: Hash256; - ): CoreDbRc[CoreDbAccount] = - accFetch(AristoCoreDbAccRef(cAcc), accPath), - - deleteFn: proc( - cAcc: CoreDbAccRef; - accPath: Hash256; - ): CoreDbRc[void] = - accDelete(AristoCoreDbAccRef(cAcc), accPath), - - clearStorageFn: proc( - cAcc: CoreDbAccRef; - accPath: Hash256; - ): CoreDbRc[void] = - accClearStorage(AristoCoreDbAccRef(cAcc), accPath), - - mergeFn: proc( - cAcc: CoreDbAccRef; - accPath: Hash256; - accRec: CoreDbAccount; - ): CoreDbRc[void] = - accMerge(AristoCoreDbAccRef(cAcc), accPath, accRec), - - hasPathFn: proc( - cAcc: CoreDbAccRef; - accPath: Hash256; - ): CoreDbRc[bool] = - accHasPath(AristoCoreDbAccRef(cAcc), accPath), - - stateFn: proc( - cAcc: CoreDbAccRef; - updateOk: bool; - ): CoreDbRc[Hash256] = - accState(AristoCoreDbAccRef(cAcc), updateOk), - - slotFetchFn: proc( - cAcc: CoreDbAccRef; - accPath: Hash256; - stoPath: openArray[byte]; - ): CoreDbRc[Blob] = - slotFetch(AristoCoreDbAccRef(cAcc), accPath, stoPath), - - slotDeleteFn: proc( - cAcc: CoreDbAccRef; - accPath: Hash256; - stoPath: openArray[byte]; - ): CoreDbRc[void] = - slotDelete(AristoCoreDbAccRef(cAcc), accPath, stoPath), - - slotHasPathFn: proc( - cAcc: CoreDbAccRef; - accPath: Hash256; - stoPath: openArray[byte]; - ): CoreDbRc[bool] = - slotHasPath(AristoCoreDbAccRef(cAcc), accPath, stoPath), - - slotMergeFn: proc( - cAcc: CoreDbAccRef; - accPath: Hash256; - stoPath: openArray[byte]; - stoData: openArray[byte]; - ): CoreDbRc[void] = - slotMerge(AristoCoreDbAccRef(cAcc), accPath, stoPath, stoData), - - slotStateFn: proc( - cAcc: CoreDbAccRef; - accPath: Hash256; - updateOk: bool; - ): CoreDbRc[Hash256] = - slotState(AristoCoreDbAccRef(cAcc), accPath, updateOk), - - slotStateEmptyFn: proc( - cAcc: CoreDbAccRef; - accPath: Hash256; - ): CoreDbRc[bool] = - slotStateEmpty(AristoCoreDbAccRef(cAcc), accPath)) - -# ------------------------------------------------------------------------------ -# Private context call back functions -# ------------------------------------------------------------------------------ - -proc ctxMethods(): CoreDbCtxFns = - template base: untyped = cCtx.base - template db: untyped = base.parent - template api: untyped = base.api - template mpt: untyped = cCtx.mpt - - proc ctxGetColumn(cCtx: AristoCoreDbCtxRef; colType: CoreDbColType; clearData: bool): CoreDbMptRef = - const info = "getColumnFn()" - if clearData: - api.deleteGenericTree(mpt, VertexID(colType)).isOkOr: - raiseAssert info & " clearing up failed: " & $error - db.bless AristoCoreDbMptRef( - methods: mptMethods(), - base: base, - mptRoot: VertexID(colType)) - - proc ctxGetAccounts(cCtx: AristoCoreDbCtxRef): CoreDbAccRef = - db.bless AristoCoreDbAccRef( - methods: accMethods(), - base: base) - - proc ctxForget(cCtx: AristoCoreDbCtxRef) = - api.forget(mpt).isOkOr: - raiseAssert "forgetFn(): " & $error - - - CoreDbCtxFns( - getColumnFn: proc(cCtx: CoreDbCtxRef; colType: CoreDbColType; clearData: bool): CoreDbMptRef = - ctxGetColumn(AristoCoreDbCtxRef(cCtx), colType, clearData), - - getAccountsFn: proc(cCtx: CoreDbCtxRef): CoreDbAccRef = - ctxGetAccounts(AristoCoreDbCtxRef(cCtx)), - - forgetFn: proc(cCtx: CoreDbCtxRef) = - ctxForget(AristoCoreDbCtxRef(cCtx))) - -# ------------------------------------------------------------------------------ -# Public handlers and helpers -# ------------------------------------------------------------------------------ - -proc getSavedState*(base: AristoBaseRef): Result[SavedState,void] = - let be = base.ctx.mpt.backend - if not be.isNil: - let rc = base.api.fetchLastSavedState(base.ctx.mpt) - if rc.isOk: - return ok(rc.value) - err() - -# --------------------- - -func to*(dsc: CoreDbMptRef, T: type AristoDbRef): T = - AristoCoreDbMptRef(dsc).base.ctx.mpt - -func to*(dsc: CoreDbAccRef, T: type AristoDbRef): T = - AristoCoreDbAccRef(dsc).base.ctx.mpt - -func to*(dsc: CoreDbMptRef, T: type AristoApiRef): T = - AristoCoreDbMptRef(dsc).base.api - -func to*(dsc: CoreDbAccRef, T: type AristoApiRef): T = - AristoCoreDbAccRef(dsc).base.api - -func rootID*(dsc: CoreDbMptRef): VertexID = - AristoCoreDbMptRef(dsc).mptRoot - -func txTop*( - base: AristoBaseRef; - info: static[string]; - ): CoreDbRc[AristoTxRef] = - base.api.txTop(base.adb).toRc(base, info) - -proc txBegin*( - base: AristoBaseRef; - info: static[string]; - ): AristoTxRef = - let rc = base.api.txBegin(base.ctx.mpt) - if rc.isErr: - raiseAssert info & ": " & $rc.error - rc.value - -proc getLevel*(base: AristoBaseRef): int = - base.api.level(base.ctx.mpt) - -# --------------------- - -proc swapCtx*(base: AristoBaseRef; ctx: CoreDbCtxRef): CoreDbCtxRef = - doAssert not ctx.isNil - result = base.ctx - - # Set read-write access and install - base.ctx = AristoCoreDbCtxRef(ctx) - base.api.reCentre(base.ctx.mpt).isOkOr: - raiseAssert "swapCtx() failed: " & $error - - -proc persistent*( - base: AristoBaseRef; - fid: uint64; - info: static[string]; - ): CoreDbRc[void] = - let - api = base.api - mpt = base.ctx.mpt - rc = api.persist(mpt, fid) - if rc.isOk: - ok() - elif api.level(mpt) == 0: - err(rc.error.toError(base, info)) - else: - err(rc.error.toError(base, info, TxPending)) - -# ------------------------------------------------------------------------------ -# Public constructors and related -# ------------------------------------------------------------------------------ - -proc destroy*(base: AristoBaseRef; eradicate: bool) = - base.api.finish(base.ctx.mpt, eradicate) - - -func init*(T: type AristoBaseRef; db: CoreDbRef; adb: AristoDbRef): T = - result = T( - parent: db, - api: AristoApiRef.init()) - - # Create initial context - let ctx = AristoCoreDbCtxRef( - methods: ctxMethods(), - base: result, - mpt: adb) - result.ctx = db.bless ctx - - when CoreDbEnableApiProfiling: - let profApi = AristoApiProfRef.init(result.api, adb.backend) - result.api = profApi - result.ctx.mpt.backend = profApi.be - - -proc init*( - T: type CoreDbCtxRef; - base: AristoBaseRef; - colState: Hash256; - colType: CoreDbColType; - ): CoreDbRc[CoreDbCtxRef] = - const info = "fromTxFn()" - - if colType.ord == 0: - return err(aristo.GenericError.toError(base, info, ColUnacceptable)) - let - api = base.api - vid = VertexID(colType) - key = colState.to(HashKey) - - # Find `(vid,key)` on transaction stack - inx = block: - let rc = api.findTx(base.ctx.mpt, vid, key) - if rc.isErr: - return err(rc.error.toError(base, info)) - rc.value - - # Fork MPT descriptor that provides `(vid,key)` - newMpt = block: - let rc = api.forkTx(base.ctx.mpt, inx) - if rc.isErr: - return err(rc.error.toError(base, info)) - rc.value - - # Create new context - let ctx = AristoCoreDbCtxRef( - methods: ctxMethods(), - base: base, - mpt: newMpt) - ok(base.parent.bless ctx) - -# ------------------------------------------------------------------------------ -# End -# ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/backend/aristo_db/handlers_kvt.nim b/nimbus/db/core_db/backend/aristo_db/handlers_kvt.nim deleted file mode 100644 index 8a613408c..000000000 --- a/nimbus/db/core_db/backend/aristo_db/handlers_kvt.nim +++ /dev/null @@ -1,266 +0,0 @@ -# Nimbus -# Copyright (c) 2023-2024 Status Research & Development GmbH -# Licensed under either of -# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or -# http://www.apache.org/licenses/LICENSE-2.0) -# * MIT license ([LICENSE-MIT](LICENSE-MIT) or -# http://opensource.org/licenses/MIT) -# at your option. This file may not be copied, modified, or distributed except -# according to those terms. - -{.push raises: [].} - -import - chronicles, - eth/common, - results, - ../../../kvt as use_kvt, - ../../base, - ../../base/base_desc, - ./common_desc - -type - KvtBaseRef* = ref object - parent: CoreDbRef ## Opaque top level descriptor - kdb: KvtDbRef ## Shared key-value table - api*: KvtApiRef ## Api functions can be re-directed - cache: KvtCoreDbKvtRef ## Shared transaction table wrapper - - KvtCoreDbKvtRef = ref object of CoreDbKvtRef - base: KvtBaseRef ## Local base descriptor - kvt: KvtDbRef ## In most cases different from `base.kdb` - - AristoCoreDbKvtBE* = ref object of CoreDbKvtBackendRef - kdb*: KvtDbRef - -logScope: - topics = "kvt-hdl" - -# ------------------------------------------------------------------------------ -# Private helpers -# ------------------------------------------------------------------------------ - -func toError( - e: KvtError; - base: KvtBaseRef; - info: string; - error = Unspecified; - ): CoreDbErrorRef = - base.parent.bless(error, AristoCoreDbError( - ctx: info, - isAristo: false, - kErr: e)) - -func toRc[T]( - rc: Result[T,KvtError]; - base: KvtBaseRef; - info: string; - error = Unspecified; - ): CoreDbRc[T] = - if rc.isOk: - when T is void: - return ok() - else: - return ok(rc.value) - err rc.error.toError(base, info, error) - -# ------------------------------------------------------------------------------ -# Private `kvt` call back functions -# ------------------------------------------------------------------------------ - -proc kvtMethods(cKvt: KvtCoreDbKvtRef): CoreDbKvtFns = - ## Key-value database table handlers - - proc kvtBackend( - cKvt:KvtCoreDbKvtRef; - ): CoreDbKvtBackendRef = - cKvt.base.parent.bless AristoCoreDbKvtBE(kdb: cKvt.kvt) - - proc kvtForget( - cKvt: KvtCoreDbKvtRef; - info: static[string]; - ): CoreDbRc[void] = - let - base = cKvt.base - kvt = cKvt.kvt - if kvt != base.kdb: - let rc = base.api.forget(kvt) - - # There is not much that can be done in case of a `forget()` error. - # So unmark it anyway. - cKvt.kvt = KvtDbRef(nil) - - if rc.isErr: - return err(rc.error.toError(base, info)) - ok() - - proc kvtGet( - cKvt: KvtCoreDbKvtRef; - k: openArray[byte]; - info: static[string]; - ): CoreDbRc[Blob] = - let rc = cKvt.base.api.get(cKvt.kvt, k) - if rc.isOk: - ok(rc.value) - elif rc.error == GetNotFound: - err(rc.error.toError(cKvt.base, info, KvtNotFound)) - else: - rc.toRc(cKvt.base, info) - - proc kvtLen( - cKvt: KvtCoreDbKvtRef; - k: openArray[byte]; - info: static[string]; - ): CoreDbRc[int] = - let rc = cKvt.base.api.len(cKvt.kvt, k) - if rc.isOk: - ok(rc.value) - elif rc.error == GetNotFound: - err(rc.error.toError(cKvt.base, info, KvtNotFound)) - else: - rc.toRc(cKvt.base, info) - - proc kvtPut( - cKvt: KvtCoreDbKvtRef; - k: openArray[byte]; - v: openArray[byte]; - info: static[string]; - ): CoreDbRc[void] = - let rc = cKvt.base.api.put(cKvt.kvt, k, v) - if rc.isOk: - ok() - else: - err(rc.error.toError(cKvt.base, info)) - - proc kvtDel( - cKvt: KvtCoreDbKvtRef; - k: openArray[byte]; - info: static[string]; - ): CoreDbRc[void] = - let rc = cKvt.base.api.del(cKvt.kvt, k) - if rc.isOk: - ok() - else: - err(rc.error.toError(cKvt.base, info)) - - proc kvtHasKey( - cKvt: KvtCoreDbKvtRef; - k: openArray[byte]; - info: static[string]; - ): CoreDbRc[bool] = - let rc = cKvt.base.api.hasKey(cKvt.kvt, k) - if rc.isOk: - ok(rc.value) - else: - err(rc.error.toError(cKvt.base, info)) - - CoreDbKvtFns( - backendFn: proc(): CoreDbKvtBackendRef = - cKvt.kvtBackend(), - - getFn: proc(k: openArray[byte]): CoreDbRc[Blob] = - cKvt.kvtGet(k, "getFn()"), - - lenFn: proc(k: openArray[byte]): CoreDbRc[int] = - cKvt.kvtLen(k, "lenFn()"), - - delFn: proc(k: openArray[byte]): CoreDbRc[void] = - cKvt.kvtDel(k, "delFn()"), - - putFn: proc(k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] = - cKvt.kvtPut(k, v, "putFn()"), - - hasKeyFn: proc(k: openArray[byte]): CoreDbRc[bool] = - cKvt.kvtHasKey(k, "hasKeyFn()"), - - forgetFn: proc(): CoreDbRc[void] = - cKvt.kvtForget("forgetFn()")) - -# ------------------------------------------------------------------------------ -# Public handlers and helpers -# ------------------------------------------------------------------------------ - -func toVoidRc*[T]( - rc: Result[T,KvtError]; - base: KvtBaseRef; - info: string; - error = Unspecified; - ): CoreDbRc[void] = - if rc.isErr: - return err(rc.error.toError(base, info, error)) - ok() - -# --------------------- - -func to*(dsc: CoreDbKvtRef; T: type KvtDbRef): T = - KvtCoreDbKvtRef(dsc).kvt - -func txTop*( - base: KvtBaseRef; - info: static[string]; - ): CoreDbRc[KvtTxRef] = - base.api.txTop(base.kdb).toRc(base, info) - -proc txBegin*( - base: KvtBaseRef; - info: static[string]; - ): KvtTxRef = - let rc = base.api.txBegin(base.kdb) - if rc.isErr: - raiseAssert info & ": " & $rc.error - rc.value - -proc persistent*( - base: KvtBaseRef; - info: static[string]; - ): CoreDbRc[void] = - let - api = base.api - kvt = base.kdb - rc = api.persist(kvt) - if rc.isOk: - ok() - elif api.level(kvt) != 0: - err(rc.error.toError(base, info, TxPending)) - elif rc.error == TxPersistDelayed: - # This is OK: Piggybacking on `Aristo` backend - ok() - else: - err(rc.error.toError(base, info)) - -# ------------------------------------------------------------------------------ -# Public constructors and related -# ------------------------------------------------------------------------------ - -proc newKvtHandler*( - base: KvtBaseRef; - info: static[string]; - ): CoreDbRc[CoreDbKvtRef] = - ok(base.cache) - - -proc destroy*(base: KvtBaseRef; eradicate: bool) = - base.api.finish(base.kdb, eradicate) # Close descriptor - - -func init*(T: type KvtBaseRef; db: CoreDbRef; kdb: KvtDbRef): T = - result = T( - parent: db, - api: KvtApiRef.init(), - kdb: kdb) - - # Preallocated shared descriptor - let dsc = KvtCoreDbKvtRef( - base: result, - kvt: kdb) - dsc.methods = dsc.kvtMethods() - result.cache = db.bless dsc - - when CoreDbEnableApiProfiling: - let profApi = KvtApiProfRef.init(result.api, kdb.backend) - result.api = profApi - result.kdb.backend = profApi.be - -# ------------------------------------------------------------------------------ -# End -# ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/backend/aristo_replicate.nim b/nimbus/db/core_db/backend/aristo_replicate.nim new file mode 100644 index 000000000..30abfdda0 --- /dev/null +++ b/nimbus/db/core_db/backend/aristo_replicate.nim @@ -0,0 +1,76 @@ +# Nimbus +# Copyright (c) 2023-2024 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +## Generic iterator, prototype to be included (rather than imported). Using +## an include file avoids duplicating code because the `T` argument is not +## bound to any object type. Otherwise all object types would be required +## when providing this iterator for import. +## +## This is not wanted here, because the import of a **pesistent** object +## would always require extra linking. + +template valueOrApiError[U,V](rc: Result[U,V]; info: static[string]): U = + rc.valueOr: raise (ref CoreDbApiError)(msg: info) + +template dbType(dsc: CoreDbKvtRef | CoreDbMptRef | CoreDbAccRef): CoreDbType = + dsc.distinctBase.parent.dbType + +# --------------- + +template kvt(dsc: CoreDbKvtRef): KvtDbRef = + dsc.distinctBase.kvt + +template call(api: KvtApiRef; fn: untyped; args: varArgs[untyped]): untyped = + when CoreDbEnableApiJumpTable: + api.fn(args) + else: + fn(args) + +template call(kvt: CoreDbKvtRef; fn: untyped; args: varArgs[untyped]): untyped = + kvt.distinctBase.parent.kvtApi.call(fn, args) + +# --------------- + +template mpt(dsc: CoreDbAccRef | CoreDbMptRef): AristoDbRef = + dsc.distinctBase.mpt + +template rootID(mpt: CoreDbMptRef): VertexID = + VertexID(CtGeneric) + +template call(api: AristoApiRef; fn: untyped; args: varArgs[untyped]): untyped = + when CoreDbEnableApiJumpTable: + api.fn(args) + else: + fn(args) + +template call( + acc: CoreDbAccRef | CoreDbMptRef; + fn: untyped; + args: varArgs[untyped]; + ): untyped = + acc.distinctBase.parent.ariApi.call(fn, args) + +# --------------- + +iterator aristoReplicate[T]( + mpt: CoreDbMptRef; + ): (Blob,Blob) + {.gcsafe, raises: [CoreDbApiError].} = + ## Generic iterator used for building dedicated backend iterators. + ## + let p = mpt.call(forkTx, mpt.mpt, 0).valueOrApiError "aristoReplicate()" + defer: discard mpt.call(forget, p) + for (vid,key,vtx,node) in T.replicate(p): + if key.len == 32: + yield (@(key.data), node.encode) + elif vid == mpt.rootID: + yield (@(key.to(Hash256).data), node.encode) + +# End diff --git a/nimbus/db/core_db/backend/aristo_rocksdb.nim b/nimbus/db/core_db/backend/aristo_rocksdb.nim index 8620f1afa..ace85fcca 100644 --- a/nimbus/db/core_db/backend/aristo_rocksdb.nim +++ b/nimbus/db/core_db/backend/aristo_rocksdb.nim @@ -11,7 +11,6 @@ {.push raises: [].} import - std/sequtils, eth/common, rocksdb, results, @@ -23,10 +22,9 @@ import ../../kvt/kvt_init/rocks_db/rdb_init, ../base, ./aristo_db, - ./aristo_db/[common_desc, handlers_aristo], ../../opts -include ./aristo_db/aristo_replicate +include ./aristo_replicate const # Expectation messages @@ -34,7 +32,7 @@ const kvtFail = "Kvt/RocksDB init() failed" # Annotation helper(s) -{.pragma: rlpRaise, gcsafe, raises: [AristoApiRlpError].} +{.pragma: rlpRaise, gcsafe, raises: [CoreDbApiError].} proc toRocksDb*( opts: DbOptions @@ -172,15 +170,6 @@ proc newAristoDualRocksDbCoreDbRef*(path: string, opts: DbOptions): CoreDbRef = raiseAssert kvtFail & ": " & $error AristoDbRocks.create(kdb, adb) -# ------------------------------------------------------------------------------ -# Public aristo iterators -# ------------------------------------------------------------------------------ - -iterator aristoReplicateRdb*(dsc: CoreDbMptRef): (Blob, Blob) {.rlpRaise.} = - ## Instantiation for `VoidBackendRef` - for k, v in aristoReplicate[use_ari.RdbBackendRef](dsc): - yield (k, v) - # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/backend/todo/free_parking.nim b/nimbus/db/core_db/backend/todo/free_parking.nim new file mode 100644 index 000000000..bf4d20914 --- /dev/null +++ b/nimbus/db/core_db/backend/todo/free_parking.nim @@ -0,0 +1,134 @@ +# Nimbus +# Copyright (c) 2023-2024 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +{.push raises: [].} + +import + eth/common, + ../../aristo as use_ari, + ../../aristo/[aristo_walk], + ../../kvt as use_kvt, + ../../kvt/[kvt_init/memory_only, kvt_walk], + ".."/[base, base/base_desc] + +# Caveat: +# additional direct include(s) -- not import(s) -- is placed near +# the end of this source file + +# Annotation helper(s) +{.pragma: noRaise, gcsafe, raises: [].} +{.pragma: rlpRaise, gcsafe, raises: [CoreDbApiError].} + +# ------------------------------------------------------------------------------ +# Private functions, free parking +# ------------------------------------------------------------------------------ + +when false: + func toError(e: AristoError; s: string; error = Unspecified): CoreDbErrorRef = + CoreDbErrorRef( + error: error, + ctx: s, + isAristo: true, + aErr: e) + + func toError(e: KvtError; s: string; error = Unspecified): CoreDbErrorRef = + CoreDbErrorRef( + error: error, + ctx: s, + isAristo: false, + kErr: e) + + proc kvtForget( + cKvt: CoreDbKvtRef; + info: static[string]; + ): CoreDbRc[void] = + ## Free parking here + let + base = cKvt.parent.kdbBase + kvt = cKvt.kvt + if kvt != base.kdb: + let rc = base.api.forget(kvt) + + # There is not much that can be done in case of a `forget()` error. + # So unmark it anyway. + cKvt.kvt = KvtDbRef(nil) + + if rc.isErr: + return err(rc.error.toError(base, info)) + ok() + + proc cptMethods( + tracer: AristoTracerRef; + ): CoreDbCaptFns = + ## Free parking here -- currently disabled + let + tr = tracer # So it can savely be captured + db = tr.parent # Will not change and can be captured + log = tr.topInst() # Ditto + + CoreDbCaptFns( + recorderFn: proc(): CoreDbRef = + db, + + logDbFn: proc(): TableRef[Blob,Blob] = + log.kLog, + + getFlagsFn: proc(): set[CoreDbCaptFlags] = + log.flags, + + forgetFn: proc() = + if not tracer.pop(): + tr.parent.tracer = AristoTracerRef(nil) + tr.restore()) + + proc tracerSetup(flags: set[CoreDbCaptFlags]): CoreDbCaptRef = + ## Free parking here -- currently disabled + if db.tracer.isNil: + db.tracer = AristoTracerRef(parent: db) + db.tracer.init(kBase, aBase, flags) + else: + db.tracer.push(flags) + CoreDbCaptRef(methods: db.tracer.cptMethods) + + proc init( + T: type CoreDbCtxRef; + base: CoreDbAriBaseRef; + colState: Hash256; + colType: CoreDbColType; + ): CoreDbRc[CoreDbCtxRef] = + const info = "fromTxFn()" + + if colType.ord == 0: + return err(use_ari.GenericError.toError(base, info, ColUnacceptable)) + let + api = base.api + vid = VertexID(colType) + key = colState.to(HashKey) + + # Find `(vid,key)` on transaction stack + inx = block: + let rc = api.findTx(base.parent.ctx.CoreDbCtxRef.mpt, vid, key) + if rc.isErr: + return err(rc.error.toError(base, info)) + rc.value + + # Fork MPT descriptor that provides `(vid,key)` + newMpt = block: + let rc = api.forkTx(base.parent.ctx.CoreDbCtxRef.mpt, inx) + if rc.isErr: + return err(rc.error.toError(base, info)) + rc.value + + # Create new context + ok(base.parent.bless CoreDbCtxRef(mpt: newMpt)) + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/backend/aristo_db/handlers_trace.nim b/nimbus/db/core_db/backend/todo/handlers_trace.nim similarity index 100% rename from nimbus/db/core_db/backend/aristo_db/handlers_trace.nim rename to nimbus/db/core_db/backend/todo/handlers_trace.nim diff --git a/nimbus/db/core_db/base.nim b/nimbus/db/core_db/base.nim index 0fbfde291..15ba86216 100644 --- a/nimbus/db/core_db/base.nim +++ b/nimbus/db/core_db/base.nim @@ -11,19 +11,14 @@ {.push raises: [].} import + std/typetraits, eth/common, "../.."/[constants, errors], + ../kvt, + ../aristo, ./base/[api_tracking, base_desc] -from ../aristo - import EmptyBlob, isValid - const - EnableAccountKeyValidation = defined(release).not - ## If this flag is enabled, the length of an account key is verified - ## to habe exactly 32 bytes. An assert is thrown if seen otherwise (a - ## notoriously week spot of the `openArray[byte]` argument type.) - EnableApiTracking = false ## When enabled, functions using this tracking facility need to import ## `chronicles`, as well. Also, some `func` designators might need to @@ -35,38 +30,50 @@ const EnableApiProfiling = true ## Enables functions profiling if `EnableApiTracking` is also set `true`. + EnableApiJumpTable* = false + ## This flag enables the functions jump table even if `EnableApiTracking` + ## and `EnableApiProfiling` is set `false`. This should be used for + ## debugging, only. + AutoValidateDescriptors = defined(release).not ## No validatinon needed for production suite. - export + CoreDbAccRef, CoreDbAccount, CoreDbApiError, - CoreDbCaptFlags, + #CoreDbCaptFlags, + #CoreDbCaptRef, CoreDbColType, CoreDbCtxRef, CoreDbErrorCode, CoreDbErrorRef, CoreDbFnInx, - CoreDbKvtBackendRef, - CoreDbMptBackendRef, + CoreDbKvtRef, + CoreDbMptRef, CoreDbPersistentTypes, CoreDbProfListRef, CoreDbRef, - CoreDbType, - CoreDbAccRef, - CoreDbCaptRef, - CoreDbKvtRef, - CoreDbMptRef, - CoreDbTxRef + CoreDbTxRef, + CoreDbType const CoreDbEnableApiTracking* = EnableApiTracking CoreDbEnableApiProfiling* = EnableApiTracking and EnableApiProfiling + CoreDbEnableApiJumpTable* = + CoreDbEnableApiTracking or CoreDbEnableApiProfiling or EnableApiJumpTable when AutoValidateDescriptors: import ./base/validate +when CoreDbEnableApiJumpTable: + discard +else: + import + ../aristo/[ + aristo_delete, aristo_desc, aristo_fetch, aristo_merge, aristo_tx], + ../kvt/[kvt_desc, kvt_utils, kvt_tx] + # More settings const logTxt = "CoreDb " @@ -80,8 +87,8 @@ const # Private helpers # ------------------------------------------------------------------------------ -when EnableApiTracking: - when EnableApiProfiling: +when CoreDbEnableApiTracking: + when CoreDbEnableApiProfiling: {.warning: "*** Provided API profiling for CoreDB (disabled by default)".} else: {.warning: "*** Provided API logging for CoreDB (disabled by default)".} @@ -103,7 +110,7 @@ template setTrackNewApi( ) = ## Template with code section that will be discarded if logging is ## disabled at compile time when `EnableApiTracking` is `false`. - when EnableApiTracking: + when CoreDbEnableApiTracking: w.beginNewApi(s) code const api {.inject,used.} = s @@ -120,6 +127,76 @@ template ifTrackNewApi*(w: CoreDbApiTrackRef; code: untyped) = w.endNewApiIf: code +# ------------------------------------------------------------------------------ +# Private KVT helpers +# ------------------------------------------------------------------------------ + +template kvt(dsc: CoreDbKvtRef): KvtDbRef = + dsc.distinctBase.kvt + +template ctx(kvt: CoreDbKvtRef): CoreDbCtxRef = + kvt.distinctBase + +# --------------- + +template call(api: KvtApiRef; fn: untyped; args: varArgs[untyped]): untyped = + when CoreDbEnableApiJumpTable: + api.fn(args) + else: + fn(args) + +template call(kvt: CoreDbKvtRef; fn: untyped; args: varArgs[untyped]): untyped = + kvt.distinctBase.parent.kvtApi.call(fn, args) + +# --------------- + +func toError(e: KvtError; s: string; error = Unspecified): CoreDbErrorRef = + CoreDbErrorRef( + error: error, + ctx: s, + isAristo: false, + kErr: e) + +# ------------------------------------------------------------------------------ +# Private Aristo helpers +# ------------------------------------------------------------------------------ + +template mpt(dsc: CoreDbAccRef | CoreDbMptRef): AristoDbRef = + dsc.distinctBase.mpt + +template mpt(tx: CoreDbTxRef): AristoDbRef = + tx.ctx.mpt + +template ctx(acc: CoreDbAccRef): CoreDbCtxRef = + acc.distinctBase + +template rootID(mpt: CoreDbMptRef): VertexID = + VertexID(CtGeneric) + +# --------------- + +template call(api: AristoApiRef; fn: untyped; args: varArgs[untyped]): untyped = + when CoreDbEnableApiJumpTable: + api.fn(args) + else: + fn(args) + +template call( + acc: CoreDbAccRef | CoreDbMptRef; + fn: untyped; + args: varArgs[untyped]; + ): untyped = + acc.distinctBase.parent.ariApi.call(fn, args) + +# --------------- + +func toError(e: AristoError; s: string; error = Unspecified): CoreDbErrorRef = + CoreDbErrorRef( + error: error, + ctx: s, + isAristo: true, + aErr: e) + # ------------------------------------------------------------------------------ # Public constructor helper # ------------------------------------------------------------------------------ @@ -132,84 +209,67 @@ proc bless*(db: CoreDbRef): CoreDbRef = db.profTab = CoreDbProfListRef.init() db -proc bless*(db: CoreDbRef; kvt: CoreDbKvtRef): CoreDbKvtRef = - ## Complete sub-module descriptor, fill in `parent`. - kvt.parent = db +proc bless*(db: CoreDbRef; ctx: CoreDbCtxRef): CoreDbCtxRef = + ctx.parent = db when AutoValidateDescriptors: - kvt.validate - kvt - -proc bless*[T: CoreDbKvtRef | - CoreDbCtxRef | CoreDbMptRef | CoreDbAccRef | - CoreDbTxRef | CoreDbCaptRef | - CoreDbKvtBackendRef | CoreDbMptBackendRef | CoreDbAccBackendRef] ( - db: CoreDbRef; - dsc: T; - ): auto = - ## Complete sub-module descriptor, fill in `parent`. - dsc.parent = db + ctx.validate + ctx + +proc bless*(ctx: CoreDbCtxRef; dsc: CoreDbMptRef | CoreDbTxRef): auto = + dsc.ctx = ctx when AutoValidateDescriptors: dsc.validate dsc -proc bless*( - db: CoreDbRef; - error: CoreDbErrorCode; - dsc: CoreDbErrorRef; - ): CoreDbErrorRef = - dsc.parent = db - dsc.error = error - when AutoValidateDescriptors: - dsc.validate - dsc +# ------------------------------------------------------------------------------ +# Public context constructors and administration +# ------------------------------------------------------------------------------ +proc ctx*(db: CoreDbRef): CoreDbCtxRef = + ## Get the defauly context. This is a base descriptor which provides the + ## KVT, MPT, the accounts descriptors as well as the transaction descriptor. + ## They are kept all in sync, i.e. `persistent()` will store exactly this + ## context. + ## + db.defCtx -proc prettyText*(e: CoreDbErrorRef): string = - ## Pretty print argument object (for tracking use `$$()`) - if e.isNil: "$ø" else: e.toStr() +proc swapCtx*(db: CoreDbRef; ctx: CoreDbCtxRef): CoreDbCtxRef = + ## Activate argument context `ctx` as default and return the previously + ## active context. This function goes typically together with `forget()`. A + ## valid scenario might look like + ## :: + ## proc doSomething(db: CoreDbRef; ctx: CoreDbCtxRef) = + ## let saved = db.swapCtx ctx + ## defer: db.swapCtx(saved).forget() + ## ... + ## + doAssert not ctx.isNil + db.setTrackNewApi BaseSwapCtxFn + result = db.defCtx + + # Set read-write access and install + CoreDbAccRef(ctx).call(reCentre, db.ctx.mpt).isOkOr: + raiseAssert $api & " failed: " & $error + CoreDbKvtRef(ctx).call(reCentre, db.ctx.kvt).isOkOr: + raiseAssert $api & " failed: " & $error + db.defCtx = ctx + db.ifTrackNewApi: debug newApiTxt, api, elapsed + +proc forget*(ctx: CoreDbCtxRef) = + ## Dispose `ctx` argument context and related columns created with this + ## context. This function fails if `ctx` is the default context. + ## + ctx.setTrackNewApi CtxForgetFn + CoreDbAccRef(ctx).call(forget, ctx.mpt).isOkOr: + raiseAssert $api & ": " & $error + CoreDbKvtRef(ctx).call(forget, ctx.kvt).isOkOr: + raiseAssert $api & ": " & $error + ctx.ifTrackNewApi: debug newApiTxt, api, elapsed # ------------------------------------------------------------------------------ # Public main descriptor methods # ------------------------------------------------------------------------------ -proc dbProfData*(db: CoreDbRef): CoreDbProfListRef = - ## Return profiling data table (only available in profiling mode). If - ## available (i.e. non-nil), result data can be organised by the functions - ## available with `aristo_profile`. - when CoreDbEnableApiProfiling: - db.profTab - -proc dbType*(db: CoreDbRef): CoreDbType = - ## Getter, print DB type identifier - ## - db.setTrackNewApi BaseDbTypeFn - result = db.dbType - db.ifTrackNewApi: debug newApiTxt, api, elapsed, result - -proc parent*[T: CoreDbKvtRef | - CoreDbCtxRef | CoreDbMptRef | CoreDbAccRef | - CoreDbTxRef | - CoreDbCaptRef | - CoreDbErrorRef]( - child: T): CoreDbRef = - ## Getter, common method for all sub-modules - ## - result = child.parent - -proc backend*(dsc: CoreDbKvtRef | CoreDbMptRef | CoreDbAccRef): auto = - ## Getter, retrieves the *raw* backend object for special/localised support. - ## - dsc.setTrackNewApi AnyBackendFn - result = dsc.methods.backendFn() - dsc.ifTrackNewApi: debug newApiTxt, api, elapsed - -proc backend*(mpt: CoreDbMptRef): auto = - ## Getter, retrieves the *raw* backend object for special/localised support. - ## - mpt.setTrackNewApi AnyBackendFn - result = mpt.methods.backendFn(mpt) - mpt.ifTrackNewApi: debug newApiTxt, api, elapsed - proc finish*(db: CoreDbRef; eradicate = false) = ## Database destructor. If the argument `eradicate` is set `false`, the ## database is left as-is and only the in-memory handlers are cleaned up. @@ -219,60 +279,129 @@ proc finish*(db: CoreDbRef; eradicate = false) = ## backend removes the database on `true`. ## db.setTrackNewApi BaseFinishFn - db.methods.destroyFn eradicate + CoreDbKvtRef(db.ctx).call(finish, db.ctx.kvt, eradicate) + CoreDbAccRef(db.ctx).call(finish, db.ctx.mpt, eradicate) db.ifTrackNewApi: debug newApiTxt, api, elapsed proc `$$`*(e: CoreDbErrorRef): string = ## Pretty print error symbol, note that this directive may have side effects ## as it calls a backend function. ## - e.setTrackNewApi ErrorPrintFn - result = e.prettyText() - e.ifTrackNewApi: debug newApiTxt, api, elapsed, result + if e.isNil: "$ø" else: e.toStr() + +proc persistent*( + db: CoreDbRef; + blockNumber: BlockNumber; + ): CoreDbRc[void] + {.discardable.} = + ## This function stored cached data from the default context (see `ctx()` + ## below) to the persistent database. + ## + ## It also stores the argument block number `blockNumber` as a state record + ## which can be retrieved via `stateBlockNumber()`. + ## + db.setTrackNewApi BasePersistentFn + block body: + block: + let rc = CoreDbKvtRef(db.ctx).call(persist, db.ctx.kvt) + if rc.isOk or rc.error == TxPersistDelayed: + # The latter clause is OK: Piggybacking on `Aristo` backend + discard + elif CoreDbKvtRef(db.ctx).call(level, db.ctx.kvt) != 0: + result = err(rc.error.toError($api, TxPending)) + break body + else: + result = err(rc.error.toError $api) + break body + block: + let rc = CoreDbAccRef(db.ctx).call(persist, db.ctx.mpt, blockNumber) + if rc.isOk: + discard + elif CoreDbAccRef(db.ctx).call(level, db.ctx.mpt) != 0: + result = err(rc.error.toError($api, TxPending)) + break body + else: + result = err(rc.error.toError $api) + break body + result = ok() + db.ifTrackNewApi: debug newApiTxt, api, elapsed, blockNumber, result + +proc stateBlockNumber*(db: CoreDbRef): BlockNumber = + ## Rhis function returns the block number stored with the latest `persist()` + ## directive. + ## + db.setTrackNewApi BaseStateBlockNumberFn + result = block: + let rc = CoreDbAccRef(db.ctx).call(fetchLastSavedState, db.ctx.mpt) + if rc.isOk: + rc.value.serial.BlockNumber + else: + 0u64 + db.ifTrackNewApi: debug newApiTxt, api, elapsed, result # ------------------------------------------------------------------------------ # Public key-value table methods # ------------------------------------------------------------------------------ -proc newKvt*(db: CoreDbRef): CoreDbKvtRef = - ## Constructor, will defect on failure. - ## +proc getKvt*(ctx: CoreDbCtxRef): CoreDbKvtRef = ## This function subscribes to the common base object shared with other ## KVT descriptors. Any changes are immediately visible to subscribers. ## On destruction (when the constructed object gets out of scope), changes ## are not saved to the backend database but are still cached and available. ## - db.setTrackNewApi BaseNewKvtFn - result = db.methods.newKvtFn().valueOr: - raiseAssert error.prettyText() - db.ifTrackNewApi: debug newApiTxt, api, elapsed + CoreDbKvtRef(ctx) + +# ----------- KVT --------------- proc get*(kvt: CoreDbKvtRef; key: openArray[byte]): CoreDbRc[Blob] = ## This function always returns a non-empty `Blob` or an error code. kvt.setTrackNewApi KvtGetFn - result = kvt.methods.getFn key + result = block: + let rc = kvt.call(get, kvt.kvt, key) + if rc.isOk: + ok(rc.value) + elif rc.error == GetNotFound: + err(rc.error.toError($api, KvtNotFound)) + else: + err(rc.error.toError $api) + kvt.ifTrackNewApi: debug newApiTxt, api, elapsed, key=key.toStr, result + +proc getOrEmpty*(kvt: CoreDbKvtRef; key: openArray[byte]): CoreDbRc[Blob] = + ## Variant of `get()` returning an empty `Blob` if the key is not found + ## on the database. + ## + kvt.setTrackNewApi KvtGetOrEmptyFn + result = block: + let rc = kvt.call(get, kvt.kvt, key) + if rc.isOk: + ok(rc.value) + elif rc.error == GetNotFound: + CoreDbRc[Blob].ok(EmptyBlob) + else: + err(rc.error.toError $api) kvt.ifTrackNewApi: debug newApiTxt, api, elapsed, key=key.toStr, result proc len*(kvt: CoreDbKvtRef; key: openArray[byte]): CoreDbRc[int] = ## This function returns the size of the value associated with `key`. kvt.setTrackNewApi KvtLenFn - result = kvt.methods.lenFn key - kvt.ifTrackNewApi: debug newApiTxt, api, elapsed, key=key.toStr, result - -proc getOrEmpty*(kvt: CoreDbKvtRef; key: openArray[byte]): CoreDbRc[Blob] = - ## This function sort of mimics the behaviour of the legacy database - ## returning an empty `Blob` if the argument `key` is not found on the - ## database. - ## - kvt.setTrackNewApi KvtGetOrEmptyFn - result = kvt.methods.getFn key - if result.isErr and result.error.error == KvtNotFound: - result = CoreDbRc[Blob].ok(EmptyBlob) + result = block: + let rc = kvt.call(len, kvt.kvt, key) + if rc.isOk: + ok(rc.value) + elif rc.error == GetNotFound: + err(rc.error.toError($api, KvtNotFound)) + else: + err(rc.error.toError $api) kvt.ifTrackNewApi: debug newApiTxt, api, elapsed, key=key.toStr, result proc del*(kvt: CoreDbKvtRef; key: openArray[byte]): CoreDbRc[void] = kvt.setTrackNewApi KvtDelFn - result = kvt.methods.delFn key + result = block: + let rc = kvt.call(del, kvt.kvt, key) + if rc.isOk: + ok() + else: + err(rc.error.toError $api) kvt.ifTrackNewApi: debug newApiTxt, api, elapsed, key=key.toStr, result proc put*( @@ -281,7 +410,12 @@ proc put*( val: openArray[byte]; ): CoreDbRc[void] = kvt.setTrackNewApi KvtPutFn - result = kvt.methods.putFn(key, val) + result = block: + let rc = kvt.call(put, kvt.kvt, key, val) + if rc.isOk: + ok() + else: + err(rc.error.toError $api) kvt.ifTrackNewApi: debug newApiTxt, api, elapsed, key=key.toStr, val=val.toLenStr, result @@ -289,63 +423,46 @@ proc hasKey*(kvt: CoreDbKvtRef; key: openArray[byte]): CoreDbRc[bool] = ## Would be named `contains` if it returned `bool` rather than `Result[]`. ## kvt.setTrackNewApi KvtHasKeyFn - result = kvt.methods.hasKeyFn key + result = block: + let rc = kvt.call(hasKey, kvt.kvt, key) + if rc.isOk: + ok(rc.value) + else: + err(rc.error.toError $api) kvt.ifTrackNewApi: debug newApiTxt, api, elapsed, key=key.toStr, result -# ------------------------------------------------------------------------------ -# Public Merkle Patricia Tree context constructors and administration -# ------------------------------------------------------------------------------ - -proc ctx*(db: CoreDbRef): CoreDbCtxRef = - ## Get currently active column context. - ## - db.setTrackNewApi BaseNewCtxFn - result = db.methods.newCtxFn() - db.ifTrackNewApi: debug newApiTxt, api, elapsed - -proc swapCtx*(db: CoreDbRef; ctx: CoreDbCtxRef): CoreDbCtxRef = - ## Activate argument context `ctx` and return the previously active column - ## context. This function goes typically together with `forget()`. A valid - ## scenario might look like - ## :: - ## proc doSomething(db: CoreDbRef; ctx: CoreDbCtxRef) = - ## let saved = db.swapCtx ctx - ## defer: db.swapCtx(saved).forget() - ## ... - ## - db.setTrackNewApi BaseSwapCtxFn - result = db.methods.swapCtxFn ctx - db.ifTrackNewApi: debug newApiTxt, api, elapsed - -proc forget*(ctx: CoreDbCtxRef) = - ## Dispose `ctx` argument context and related columns created with this - ## context. This function fails if `ctx` is the default context. - ## - ctx.setTrackNewApi CtxForgetFn - ctx.methods.forgetFn(ctx) - ctx.ifTrackNewApi: debug newApiTxt, api, elapsed - # ------------------------------------------------------------------------------ # Public functions for generic columns # ------------------------------------------------------------------------------ -proc getColumn*( +proc getGeneric*( ctx: CoreDbCtxRef; - colType: CoreDbColType; clearData = false; ): CoreDbMptRef = - ## ... + ## Get a generic MPT, viewed as column ## - ctx.setTrackNewApi CtxGetColumnFn - result = ctx.methods.getColumnFn(ctx, colType, clearData) - ctx.ifTrackNewApi: debug newApiTxt, api, colType, clearData, elapsed + ctx.setTrackNewApi CtxGetGenericFn + result = CoreDbMptRef(ctx) + if clearData: + result.call(deleteGenericTree, ctx.mpt, result.rootID).isOkOr: + raiseAssert $api & ": " & $error + ctx.ifTrackNewApi: debug newApiTxt, api, clearData, elapsed + +# ----------- generic MPT --------------- proc fetch*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[Blob] = ## Fetch data from the argument `mpt`. The function always returns a ## non-empty `Blob` or an error code. ## mpt.setTrackNewApi MptFetchFn - result = mpt.methods.fetchFn(mpt, key) + result = block: + let rc = mpt.call(fetchGenericData, mpt.mpt, mpt.rootID, key) + if rc.isOk: + ok(rc.value) + elif rc.error == FetchPathNotFound: + err(rc.error.toError($api, MptNotFound)) + else: + err(rc.error.toError $api) mpt.ifTrackNewApi: debug newApiTxt, api, elapsed, key=key.toStr, result proc fetchOrEmpty*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[Blob] = @@ -353,14 +470,26 @@ proc fetchOrEmpty*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[Blob] = ## on the database. ## mpt.setTrackNewApi MptFetchOrEmptyFn - result = mpt.methods.fetchFn(mpt, key) - if result.isErr and result.error.error == MptNotFound: - result = CoreDbRc[Blob].ok(EmptyBlob) + result = block: + let rc = mpt.call(fetchGenericData, mpt.mpt, mpt.rootID, key) + if rc.isOk: + ok(rc.value) + elif rc.error == FetchPathNotFound: + CoreDbRc[Blob].ok(EmptyBlob) + else: + err(rc.error.toError $api) mpt.ifTrackNewApi: debug newApiTxt, api, elapsed, key=key.toStr, result proc delete*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[void] = mpt.setTrackNewApi MptDeleteFn - result = mpt.methods.deleteFn(mpt, key) + result = block: + let rc = mpt.call(deleteGenericData, mpt.mpt, mpt.rootID, key) + if rc.isOk: + ok() + elif rc.error == DelPathNotFound: + err(rc.error.toError($api, MptNotFound)) + else: + err(rc.error.toError $api) mpt.ifTrackNewApi: debug newApiTxt, api, elapsed, key=key.toStr, result proc merge*( @@ -369,7 +498,12 @@ proc merge*( val: openArray[byte]; ): CoreDbRc[void] = mpt.setTrackNewApi MptMergeFn - result = mpt.methods.mergeFn(mpt, key, val) + result = block: + let rc = mpt.call(mergeGenericData, mpt.mpt, mpt.rootID, key, val) + if rc.isOk: + ok() + else: + err(rc.error.toError $api) mpt.ifTrackNewApi: debug newApiTxt, api, elapsed, key=key.toStr, val=val.toLenStr, result @@ -378,7 +512,12 @@ proc hasPath*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[bool] = ## than a `Result[]`. ## mpt.setTrackNewApi MptHasPathFn - result = mpt.methods.hasPathFn(mpt, key) + result = block: + let rc = mpt.call(hasPathGeneric, mpt.mpt, mpt.rootID, key) + if rc.isOk: + ok(rc.value) + else: + err(rc.error.toError $api) mpt.ifTrackNewApi: debug newApiTxt, api, elapsed, key=key.toStr, result proc state*(mpt: CoreDbMptRef; updateOk = false): CoreDbRc[Hash256] = @@ -389,7 +528,12 @@ proc state*(mpt: CoreDbMptRef; updateOk = false): CoreDbRc[Hash256] = ## database will be updated first (if needed, at all). ## mpt.setTrackNewApi MptStateFn - result = mpt.methods.stateFn(mpt, updateOk) + result = block: + let rc = mpt.call(fetchGenericState, mpt.mpt, mpt.rootID, updateOk) + if rc.isOk: + ok(rc.value) + else: + err(rc.error.toError $api) mpt.ifTrackNewApi: debug newApiTxt, api, elapsed, updateOK, result # ------------------------------------------------------------------------------ @@ -400,7 +544,7 @@ proc getAccounts*(ctx: CoreDbCtxRef): CoreDbAccRef = ## Accounts column constructor, will defect on failure. ## ctx.setTrackNewApi CtxGetAccountsFn - result = ctx.methods.getAccountsFn(ctx) + result = CoreDbAccRef(ctx) ctx.ifTrackNewApi: debug newApiTxt, api, elapsed # ----------- accounts --------------- @@ -412,10 +556,15 @@ proc fetch*( ## Fetch the account data record for the particular account indexed by ## the key `accPath`. ## - when EnableAccountKeyValidation: - doAssert accPath.len == 32 acc.setTrackNewApi AccFetchFn - result = acc.methods.fetchFn(acc, accPath) + result = block: + let rc = acc.call(fetchAccountRecord, acc.mpt, accPath) + if rc.isOk: + ok(rc.value) + elif rc.error == FetchPathNotFound: + err(rc.error.toError($api, AccNotFound)) + else: + err(rc.error.toError $api) acc.ifTrackNewApi: debug newApiTxt, api, elapsed, accPath=accPath.toStr, result @@ -426,10 +575,16 @@ proc delete*( ## Delete the particular account indexed by the key `accPath`. This ## will also destroy an associated storage area. ## - when EnableAccountKeyValidation: - doAssert accPath.len == 32 acc.setTrackNewApi AccDeleteFn - result = acc.methods.deleteFn(acc, accPath) + result = block: + let rc = acc.call(deleteAccountRecord, acc.mpt, accPath) + if rc.isOk: + ok() + elif rc.error == DelPathNotFound: + # TODO: Would it be conseqient to just return `ok()` here? + err(rc.error.toError($api, AccNotFound)) + else: + err(rc.error.toError $api) acc.ifTrackNewApi: debug newApiTxt, api, elapsed, accPath=accPath.toStr, result @@ -440,10 +595,13 @@ proc clearStorage*( ## Delete all data slots from the storage area associated with the ## particular account indexed by the key `accPath`. ## - when EnableAccountKeyValidation: - doAssert accPath.len == 32 acc.setTrackNewApi AccClearStorageFn - result = acc.methods.clearStorageFn(acc, accPath) + result = block: + let rc = acc.call(deleteStorageTree, acc.mpt, accPath) + if rc.isOk or rc.error in {DelStoRootMissing,DelStoAccMissing}: + ok() + else: + err(rc.error.toError $api) acc.ifTrackNewApi: debug newApiTxt, api, elapsed, accPath=accPath.toStr, result @@ -456,7 +614,12 @@ proc merge*( ## `account` argument uniquely idendifies the particular account address. ## acc.setTrackNewApi AccMergeFn - result = acc.methods.mergeFn(acc, accPath, accRec) + result = block: + let rc = acc.call(mergeAccountRecord, acc.mpt, accPath, accRec) + if rc.isOk: + ok() + else: + err(rc.error.toError $api) acc.ifTrackNewApi: debug newApiTxt, api, elapsed, accPath=accPath.toStr, result @@ -466,10 +629,13 @@ proc hasPath*( ): CoreDbRc[bool] = ## Would be named `contains` if it returned `bool` rather than `Result[]`. ## - when EnableAccountKeyValidation: - doAssert accPath.len == 32 acc.setTrackNewApi AccHasPathFn - result = acc.methods.hasPathFn(acc, accPath) + result = block: + let rc = acc.call(hasPathAccount, acc.mpt, accPath) + if rc.isOk: + ok(rc.value) + else: + err(rc.error.toError $api) acc.ifTrackNewApi: debug newApiTxt, api, elapsed, accPath=accPath.toStr, result @@ -481,7 +647,12 @@ proc state*(acc: CoreDbAccRef; updateOk = false): CoreDbRc[Hash256] = ## database will be updated first (if needed, at all). ## acc.setTrackNewApi AccStateFn - result = acc.methods.stateFn(acc, updateOk) + result = block: + let rc = acc.call(fetchAccountState, acc.mpt, updateOk) + if rc.isOk: + ok(rc.value) + else: + err(rc.error.toError $api) acc.ifTrackNewApi: debug newApiTxt, api, elapsed, updateOK, result # ------------ storage --------------- @@ -493,9 +664,15 @@ proc slotFetch*( ): CoreDbRc[Blob] = ## Like `fetch()` but with cascaded index `(accPath,slot)`. acc.setTrackNewApi AccSlotFetchFn - result = acc.methods.slotFetchFn(acc, accPath, slot) + result = block: + let rc = acc.call(fetchStorageData, acc.mpt, accPath, slot) + if rc.isOk: + ok(rc.value) + elif rc.error == FetchPathNotFound: + err(rc.error.toError($api, StoNotFound)) + else: + err(rc.error.toError $api) acc.ifTrackNewApi: - doAssert accPath.len == 32 debug newApiTxt, api, elapsed, accPath=accPath.toStr, slot=slot.toStr, result @@ -505,10 +682,17 @@ proc slotDelete*( slot: openArray[byte]; ): CoreDbRc[void] = ## Like `delete()` but with cascaded index `(accPath,slot)`. - when EnableAccountKeyValidation: - doAssert accPath.len == 32 acc.setTrackNewApi AccSlotDeleteFn - result = acc.methods.slotDeleteFn(acc, accPath, slot) + result = block: + let rc = acc.call(deleteStorageData, acc.mpt, accPath, slot) + if rc.isOk or rc.error == DelStoRootMissing: + # The second `if` clause is insane but legit: A storage column was + # announced for an account but no data have been added, yet. + ok() + elif rc.error == DelPathNotFound: + err(rc.error.toError($api, StoNotFound)) + else: + err(rc.error.toError $api) acc.ifTrackNewApi: debug newApiTxt, api, elapsed, accPath=accPath.toStr, slot=slot.toStr, result @@ -519,10 +703,13 @@ proc slotHasPath*( slot: openArray[byte]; ): CoreDbRc[bool] = ## Like `hasPath()` but with cascaded index `(accPath,slot)`. - when EnableAccountKeyValidation: - doAssert accPath.len == 32 acc.setTrackNewApi AccSlotHasPathFn - result = acc.methods.slotHasPathFn(acc, accPath, slot) + result = block: + let rc = acc.call(hasPathStorage, acc.mpt, accPath, slot) + if rc.isOk: + ok(rc.value) + else: + err(rc.error.toError $api) acc.ifTrackNewApi: debug newApiTxt, api, elapsed, accPath=accPath.toStr, slot=slot.toStr, result @@ -534,10 +721,13 @@ proc slotMerge*( data: openArray[byte]; ): CoreDbRc[void] = ## Like `merge()` but with cascaded index `(accPath,slot)`. - when EnableAccountKeyValidation: - doAssert accPath.len == 32 acc.setTrackNewApi AccSlotMergeFn - result = acc.methods.slotMergeFn(acc, accPath, slot, data) + result = block: + let rc = acc.call(mergeStorageData, acc.mpt, accPath, slot, data) + if rc.isOk: + ok() + else: + err(rc.error.toError $api) acc.ifTrackNewApi: debug newApiTxt, api, elapsed, accPath=accPath.toStr, slot=slot.toStr, result @@ -554,10 +744,13 @@ proc slotState*( ## If the argument `updateOk` is set `true`, the Merkle hashes of the ## database will be updated first (if needed, at all). ## - when EnableAccountKeyValidation: - doAssert accPath.len == 32 acc.setTrackNewApi AccSlotStateFn - result = acc.methods.slotStateFn(acc, accPath, updateOk) + result = block: + let rc = acc.call(fetchStorageState, acc.mpt, accPath, updateOk) + if rc.isOk: + ok(rc.value) + else: + err(rc.error.toError $api) acc.ifTrackNewApi: debug newApiTxt, api, elapsed, accPath=accPath.toStr, updateOk, result @@ -568,10 +761,13 @@ proc slotStateEmpty*( ## This function returns `true` if the storage data column is empty or ## missing. ## - when EnableAccountKeyValidation: - doAssert accPath.len == 32 acc.setTrackNewApi AccSlotStateEmptyFn - result = acc.methods.slotStateEmptyFn(acc, accPath) + result = block: + let rc = acc.call(hasStorageData, acc.mpt, accPath) + if rc.isOk: + ok(not rc.value) + else: + err(rc.error.toError $api) acc.ifTrackNewApi: debug newApiTxt, api, elapsed, accPath=accPath.toStr, result @@ -580,10 +776,13 @@ proc slotStateEmptyOrVoid*( accPath: Hash256; ): bool = ## Convenience wrapper, returns `true` where `slotStateEmpty()` would fail. - when EnableAccountKeyValidation: - doAssert accPath.len == 32 acc.setTrackNewApi AccSlotStateEmptyOrVoidFn - result = acc.methods.slotStateEmptyFn(acc, accPath).valueOr: true + result = block: + let rc = acc.call(hasStorageData, acc.mpt, accPath) + if rc.isOk: + not rc.value + else: + true acc.ifTrackNewApi: debug newApiTxt, api, elapsed, accPath=accPath.toStr, result @@ -595,15 +794,13 @@ proc recast*( accRec: CoreDbAccount; updateOk = false; ): CoreDbRc[Account] = - ## Convert the argument `statement` to the portable Ethereum representation + ## Complete the argument `accRec` to the portable Ethereum representation ## of an account statement. This conversion may fail if the storage colState - ## hash (see `hash()` above) is currently unavailable. + ## hash (see `slotState()` above) is currently unavailable. ## - when EnableAccountKeyValidation: - doAssert accPath.len == 32 - acc.setTrackNewApi EthAccRecastFn - let rc = acc.methods.slotStateFn(acc, accPath, updateOk) - result = + acc.setTrackNewApi AccRecastFn + let rc = acc.call(fetchStorageState, acc.mpt, accPath, updateOk) + result = block: if rc.isOk: ok Account( nonce: accRec.nonce, @@ -611,7 +808,7 @@ proc recast*( codeHash: accRec.codeHash, storageRoot: rc.value) else: - err(rc.error) + err(rc.error.toError $api) acc.ifTrackNewApi: let slotState = if rc.isOk: rc.value.toStr else: "n/a" debug newApiTxt, api, elapsed, accPath=accPath.toStr, slotState, result @@ -624,72 +821,81 @@ proc level*(db: CoreDbRef): int = ## Retrieve transaction level (zero if there is no pending transaction). ## db.setTrackNewApi BaseLevelFn - result = db.methods.levelFn() + result = CoreDbAccRef(db.ctx).call(level, db.ctx.mpt) db.ifTrackNewApi: debug newApiTxt, api, elapsed, result -proc persistent*( - db: CoreDbRef; - blockNumber: BlockNumber; - ): CoreDbRc[void] {.discardable.} = - ## Variant of `persistent()` which stores a block number within the recovery - ## journal record. This recoed will be addressable by the `blockNumber` (e.g. - ## for recovery.) The argument block number `blockNumber` must be greater - ## than all previously stored block numbers. - ## - ## The function is intended to be used in a way so hat the argument block - ## number `blockNumber` is associated with the state root to be recovered - ## from a particular journal entry. This means that the correct block number - ## will be the one of the state *before* a state change takes place. Using - ## it that way, `pesistent()` must only be run after some blocks were fully - ## executed. - ## - ## Example: - ## :: - ## # Save block number for the current state - ## let stateBlockNumber = db.getCanonicalHead().blockNumber - ## .. - ## # Process blocks - ## .. - ## db.persistent(stateBlockNumber) - ## - db.setTrackNewApi BasePersistentFn - result = db.methods.persistentFn Opt.some(blockNumber) - db.ifTrackNewApi: debug newApiTxt, api, elapsed, blockNumber, result - -proc newTransaction*(db: CoreDbRef): CoreDbTxRef = +proc newTransaction*(ctx: CoreDbCtxRef): CoreDbTxRef = ## Constructor ## - db.setTrackNewApi BaseNewTxFn - result = db.methods.beginFn() - db.ifTrackNewApi: - debug newApiTxt, api, elapsed, newLevel=db.methods.levelFn() - + ctx.setTrackNewApi BaseNewTxFn + let + kTx = CoreDbKvtRef(ctx).call(txBegin, ctx.kvt).valueOr: + raiseAssert $api & ": " & $error + aTx = CoreDbAccRef(ctx).call(txBegin, ctx.mpt).valueOr: + raiseAssert $api & ": " & $error + result = ctx.bless CoreDbTxRef(kTx: kTx, aTx: aTx) + ctx.ifTrackNewApi: + let newLevel = CoreDbAccRef(ctx).call(level, ctx.mpt) + debug newApiTxt, api, elapsed, newLevel proc level*(tx: CoreDbTxRef): int = ## Print positive transaction level for argument `tx` ## tx.setTrackNewApi TxLevelFn - result = tx.methods.levelFn() + result = CoreDbAccRef(tx.ctx).call(txLevel, tx.aTx) tx.ifTrackNewApi: debug newApiTxt, api, elapsed, result proc commit*(tx: CoreDbTxRef) = tx.setTrackNewApi TxCommitFn: - let prvLevel {.used.} = tx.methods.levelFn() - tx.methods.commitFn() + let prvLevel {.used.} = CoreDbAccRef(tx.ctx).call(txLevel, tx.aTx) + CoreDbAccRef(tx.ctx).call(commit, tx.aTx).isOkOr: + raiseAssert $api & ": " & $error + CoreDbKvtRef(tx.ctx).call(commit, tx.kTx).isOkOr: + raiseAssert $api & ": " & $error tx.ifTrackNewApi: debug newApiTxt, api, elapsed, prvLevel proc rollback*(tx: CoreDbTxRef) = tx.setTrackNewApi TxRollbackFn: - let prvLevel {.used.} = tx.methods.levelFn() - tx.methods.rollbackFn() + let prvLevel {.used.} = CoreDbAccRef(tx.ctx).call(txLevel, tx.aTx) + CoreDbAccRef(tx.ctx).call(rollback, tx.aTx).isOkOr: + raiseAssert $api & ": " & $error + CoreDbKvtRef(tx.ctx).call(rollback, tx.kTx).isOkOr: + raiseAssert $api & ": " & $error tx.ifTrackNewApi: debug newApiTxt, api, elapsed, prvLevel proc dispose*(tx: CoreDbTxRef) = tx.setTrackNewApi TxDisposeFn: - let prvLevel {.used.} = tx.methods.levelFn() - tx.methods.disposeFn() + let prvLevel {.used.} = CoreDbAccRef(tx.ctx).call(txLevel, tx.aTx) + if CoreDbAccRef(tx.ctx).call(isTop, tx.aTx): + CoreDbAccRef(tx.ctx).call(rollback, tx.aTx).isOkOr: + raiseAssert $api & ": " & $error + if CoreDbKvtRef(tx.ctx).call(isTop, tx.kTx): + CoreDbKvtRef(tx.ctx).call(rollback, tx.kTx).isOkOr: + raiseAssert $api & ": " & $error tx.ifTrackNewApi: debug newApiTxt, api, elapsed, prvLevel +# ------------------------------------------------------------------------------ +# Legacy and convenience methods +# ------------------------------------------------------------------------------ + +proc newKvt*(db: CoreDbRef): CoreDbKvtRef = + ## Variant of `getKvt()` retrieving the KVT from the default context + db.ctx.getKvt + +proc newTransaction*(db: CoreDbRef): CoreDbTxRef = + ## Variant of `newTransaction()` starting the transaction on the default + ## context + db.ctx.newTransaction + +proc getColumn*( + ctx: CoreDbCtxRef; + colType: CoreDbColType; + clearData = false; + ): CoreDbMptRef = + ## Variant of `getGenteric()` forcing `colType` to be `CtGeneric` + doAssert colType == CtGeneric + ctx.getGeneric clearData + # ------------------------------------------------------------------------------ # Public tracer methods # ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/base/api_tracking.nim b/nimbus/db/core_db/base/api_tracking.nim index dbbccc878..280342eb2 100644 --- a/nimbus/db/core_db/base/api_tracking.nim +++ b/nimbus/db/core_db/base/api_tracking.nim @@ -20,21 +20,22 @@ import type CoreDbApiTrackRef* = - CoreDbRef | CoreDbKvtRef | - CoreDbCtxRef | CoreDbMptRef | CoreDbAccRef | - CoreDbTxRef | CoreDbCaptRef | CoreDbErrorRef + # CoreDbCaptRef | + CoreDbRef | CoreDbKvtRef | CoreDbCtxRef | CoreDbMptRef | CoreDbAccRef | + CoreDbTxRef CoreDbFnInx* = enum ## Profiling table index SummaryItem = "total" + AccClearStorageFn = "clearStorage" AccDeleteFn = "acc/delete" AccFetchFn = "acc/fetch" AccForgetFn = "acc/forget" AccHasPathFn = "acc/hasPath" AccMergeFn = "acc/merge" + AccRecastFn = "recast" AccStateFn = "acc/state" - AccClearStorageFn = "acc/clearStorage" AccSlotFetchFn = "slotFetch" AccSlotDeleteFn = "slotDelete" @@ -45,17 +46,13 @@ type AccSlotStateEmptyOrVoidFn = "slotStateEmptyOrVoid" AccSlotPairsIt = "slotPairs" - AnyBackendFn = "any/backend" - - BaseDbTypeFn = "dbType" BaseFinishFn = "finish" BaseLevelFn = "level" BaseNewCaptureFn = "newCapture" - BaseNewCtxFn = "ctx" BaseNewCtxFromTxFn = "ctxFromTx" - BaseNewKvtFn = "newKvt" BaseNewTxFn = "newTransaction" BasePersistentFn = "persistent" + BaseStateBlockNumberFn = "stateBlockNumber" BaseSwapCtxFn = "swapCtx" CptFlagsFn = "cpt/flags" @@ -65,14 +62,9 @@ type CtxForgetFn = "ctx/forget" CtxGetAccountsFn = "getAccounts" - CtxGetColumnFn = "getColumn" - CtxNewColFn = "ctx/newColumn" - - ErrorPrintFn = "$$" - EthAccRecastFn = "recast" + CtxGetGenericFn = "getGeneric" KvtDelFn = "del" - KvtForgetFn = "forget" KvtGetFn = "get" KvtGetOrEmptyFn = "getOrEmpty" KvtHasKeyFn = "hasKey" @@ -111,7 +103,11 @@ func toStr*(w: Hash256): string = if w == EMPTY_ROOT_HASH: "EMPTY_ROOT_HASH" else: w.data.oaToStr proc toStr*(e: CoreDbErrorRef): string = - $e.error & "(" & e.parent.methods.errorPrintFn(e) & ")" + result = $e.error & "(" + result &= (if e.isAristo: "Aristo" else: "Kvt") + result &= ", ctx=" & $e.ctx & ", error=" + result &= (if e.isAristo: $e.aErr else: $e.kErr) + result &= ")" func toLenStr*(w: openArray[byte]): string = if 0 < w.len and w.len < 5: "<" & w.oaToStr & ">" @@ -156,7 +152,7 @@ proc toStr*(rc: CoreDbRc[CoreDbRef]): string = rc.toStr "db" proc toStr*(rc: CoreDbRc[CoreDbAccount]): string = rc.toStr "acc" proc toStr*(rc: CoreDbRc[CoreDbKvtRef]): string = rc.toStr "kvt" proc toStr*(rc: CoreDbRc[CoreDbTxRef]): string = rc.toStr "tx" -proc toStr*(rc: CoreDbRc[CoreDbCaptRef]): string = rc.toStr "capt" +#proc toStr*(rc: CoreDbRc[CoreDbCaptRef]): string = rc.toStr "capt" proc toStr*(rc: CoreDbRc[CoreDbCtxRef]): string = rc.toStr "ctx" proc toStr*(rc: CoreDbRc[CoreDbMptRef]): string = rc.toStr "mpt" proc toStr*(rc: CoreDbRc[CoreDbAccRef]): string = rc.toStr "acc" @@ -177,9 +173,12 @@ template endNewApiIf*(w: CoreDbApiTrackRef; code: untyped) = block body: when typeof(w) is CoreDbRef: let db = w - else: + elif typeof(w) is CoreDbTxRef: + let db = w.ctx.parent if w.isNil: break body - let db = w.parent + else: + let db = w.distinctBase.parent + if w.distinctBase.isNil: break body when CoreDbEnableApiProfiling: let elapsed {.inject,used.} = getTime() - bnaStart aristo_profile.update(db.profTab, bnaCtx.ord, elapsed) diff --git a/nimbus/db/core_db/base/base_desc.nim b/nimbus/db/core_db/base/base_desc.nim index 1e29f4873..21a05be86 100644 --- a/nimbus/db/core_db/base/base_desc.nim +++ b/nimbus/db/core_db/base/base_desc.nim @@ -11,9 +11,9 @@ {.push raises: [].} import - std/tables, - eth/common, + results, ../../aristo, + ../../kvt, ../../aristo/aristo_profile # Annotation helpers @@ -59,261 +59,82 @@ type StoNotFound TxPending - CoreDbColType* = enum - CtGeneric = 2 # columns smaller than 2 are not provided + CoreDbColType* = enum # Keep that legacy type for a while .. + CtGeneric = 2 # Actually only this constant is needed CoreDbCaptFlags* {.pure.} = enum PersistPut PersistDel - # -------------------------------------------------- - # Sub-descriptor: Misc methods for main descriptor - # -------------------------------------------------- - CoreDbBaseDestroyFn* = proc(eradicate = true) {.noRaise.} - CoreDbBaseErrorPrintFn* = proc(e: CoreDbErrorRef): string {.noRaise.} - CoreDbBaseLevelFn* = proc(): int {.noRaise.} - CoreDbBaseNewKvtFn* = proc(): CoreDbRc[CoreDbKvtRef] {.noRaise.} - CoreDbBaseNewCtxFn* = proc(): CoreDbCtxRef {.noRaise.} - CoreDbBaseNewCtxFromTxFn* = proc( - colState: Hash256; kind: CoreDbColType): CoreDbRc[CoreDbCtxRef] {.noRaise.} - CoreDbBaseSwapCtxFn* = proc(ctx: CoreDbCtxRef): CoreDbCtxRef {.noRaise.} - CoreDbBaseTxBeginFn* = proc(): CoreDbTxRef {.noRaise.} - CoreDbBaseNewCaptFn* = - proc(flgs: set[CoreDbCaptFlags]): CoreDbRc[CoreDbCaptRef] {.noRaise.} - CoreDbBaseGetCaptFn* = proc(): CoreDbRc[CoreDbCaptRef] {.noRaise.} - CoreDbBasePersistentFn* = - proc(bn: Opt[BlockNumber]): CoreDbRc[void] {.noRaise.} - - CoreDbBaseFns* = object - destroyFn*: CoreDbBaseDestroyFn - errorPrintFn*: CoreDbBaseErrorPrintFn - levelFn*: CoreDbBaseLevelFn - - # Kvt constructor - newKvtFn*: CoreDbBaseNewKvtFn - - # MPT context constructor - newCtxFn*: CoreDbBaseNewCtxFn - newCtxFromTxFn*: CoreDbBaseNewCtxFromTxFn - swapCtxFn*: CoreDbBaseSwapCtxFn - - # Transactions constructors - beginFn*: CoreDbBaseTxBeginFn - - # Capture/tracer constructors - newCaptureFn*: CoreDbBaseNewCaptFn - - # Save to disk - persistentFn*: CoreDbBasePersistentFn - - - # -------------------------------------------------- - # Sub-descriptor: KVT methods - # -------------------------------------------------- - CoreDbKvtBackendFn* = proc(): CoreDbKvtBackendRef {.noRaise.} - CoreDbKvtGetFn* = proc(k: openArray[byte]): CoreDbRc[Blob] {.noRaise.} - CoreDbKvtLenFn* = proc(k: openArray[byte]): CoreDbRc[int] {.noRaise.} - CoreDbKvtDelFn* = proc(k: openArray[byte]): CoreDbRc[void] {.noRaise.} - CoreDbKvtPutFn* = - proc(k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] {.noRaise.} - CoreDbKvtForgetFn* = proc(): CoreDbRc[void] {.noRaise.} - CoreDbKvtHasKeyFn* = proc(k: openArray[byte]): CoreDbRc[bool] {.noRaise.} - - CoreDbKvtFns* = object - ## Methods for key-value table - backendFn*: CoreDbKvtBackendFn - getFn*: CoreDbKvtGetFn - lenFn*: CoreDbKvtLenFn - delFn*: CoreDbKvtDelFn - putFn*: CoreDbKvtPutFn - hasKeyFn*: CoreDbKvtHasKeyFn - forgetFn*: CoreDbKvtForgetFn - - # -------------------------------------------------- - # Sub-descriptor: MPT context methods - # -------------------------------------------------- - CoreDbCtxGetColumnFn* = proc( - cCtx: CoreDbCtxRef; colType: CoreDbColType; clearData: bool): CoreDbMptRef {.noRaise.} - CoreDbCtxGetAccountsFn* = proc(cCtx: CoreDbCtxRef): CoreDbAccRef {.noRaise.} - CoreDbCtxForgetFn* = proc(cCtx: CoreDbCtxRef) {.noRaise.} - - CoreDbCtxFns* = object - ## Methods for context maniulation - getColumnFn*: CoreDbCtxGetColumnFn - getAccountsFn*: CoreDbCtxGetAccountsFn - forgetFn*: CoreDbCtxForgetFn - - # -------------------------------------------------- - # Sub-descriptor: generic Mpt methods - # -------------------------------------------------- - CoreDbMptBackendFn* = proc(cMpt: CoreDbMptRef): CoreDbMptBackendRef {.noRaise.} - CoreDbMptFetchFn* = - proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[Blob] {.noRaise.} - CoreDbMptFetchAccountFn* = - proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[CoreDbAccount] {.noRaise.} - CoreDbMptDeleteFn* = - proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[void] {.noRaise.} - CoreDbMptMergeFn* = - proc(cMpt: CoreDbMptRef, k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] {.noRaise.} - CoreDbMptHasPathFn* = proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[bool] {.noRaise.} - CoreDbMptStateFn* = proc(cMpt: CoreDbMptRef, updateOk: bool): CoreDbRc[Hash256] {.noRaise.} - - CoreDbMptFns* = object - ## Methods for trie objects - backendFn*: CoreDbMptBackendFn - fetchFn*: CoreDbMptFetchFn - deleteFn*: CoreDbMptDeleteFn - mergeFn*: CoreDbMptMergeFn - hasPathFn*: CoreDbMptHasPathFn - stateFn*: CoreDbMptStateFn - - - # ---------------------------------------------------- - # Sub-descriptor: Account column methods - # ------------------------------------------------------ - CoreDbAccBackendFn* = proc( - cAcc: CoreDbAccRef): CoreDbAccBackendRef {.noRaise.} - CoreDbAccFetchFn* = proc( - cAcc: CoreDbAccRef; accPath: Hash256; - ): CoreDbRc[CoreDbAccount] {.noRaise.} - CoreDbAccDeleteFn* = proc( - cAcc: CoreDbAccRef, accPath: Hash256): CoreDbRc[void] {.noRaise.} - CoreDbAccClearStorageFn* = proc( - cAcc: CoreDbAccRef; accPath: Hash256): CoreDbRc[void] {.noRaise.} - CoreDbAccMergeFn* = proc( - cAcc: CoreDbAccRef; accPath: Hash256; accRec: CoreDbAccount; - ): CoreDbRc[void] {.noRaise.} - CoreDbAccHasPathFn* = proc( - cAcc: CoreDbAccRef; accPath: Hash256): CoreDbRc[bool] {.noRaise.} - CoreDbAccStateFn* = proc( - cAcc: CoreDbAccRef; updateOk: bool): CoreDbRc[Hash256] {.noRaise.} - - CoreDbSlotFetchFn* = proc( - cAcc: CoreDbAccRef; accPath: Hash256, stoPath: openArray[byte]; - ): CoreDbRc[Blob] {.noRaise.} - CoreDbSlotDeleteFn* = proc( - cAcc: CoreDbAccRef; accPath: Hash256, stoPath: openArray[byte]; - ): CoreDbRc[void] {.noRaise.} - CoreDbSlotHasPathFn* = proc( - cAcc: CoreDbAccRef; accPath: Hash256, stoPath: openArray[byte]; - ): CoreDbRc[bool] {.noRaise.} - CoreDbSlotMergeFn* = proc( - cAcc: CoreDbAccRef; accPath: Hash256, stoPath, stoData: openArray[byte]; - ): CoreDbRc[void] {.noRaise.} - CoreDbSlotStateFn* = proc( - cAcc: CoreDbAccRef; accPath: Hash256; updateOk: bool; - ): CoreDbRc[Hash256] {.noRaise.} - CoreDbSlotStateEmptyFn* = proc( - cAcc: CoreDbAccRef; accPath: Hash256; - ): CoreDbRc[bool] {.noRaise.} - - CoreDbAccFns* = object - ## Methods for trie objects - backendFn*: CoreDbAccBackendFn - fetchFn*: CoreDbAccFetchFn - clearStorageFn*: CoreDbAccClearStorageFn - deleteFn*: CoreDbAccDeleteFn - hasPathFn*: CoreDbAccHasPathFn - mergeFn*: CoreDbAccMergeFn - stateFn*: CoreDbAccStateFn - - slotFetchFn*: CoreDbSlotFetchFn - slotDeleteFn*: CoreDbSlotDeleteFn - slotHasPathFn*: CoreDbSlotHasPathFn - slotMergeFn*: CoreDbSlotMergeFn - slotStateFn*: CoreDbSlotStateFn - slotStateEmptyFn*: CoreDbSlotStateEmptyFn - - # -------------------------------------------------- - # Sub-descriptor: Transaction frame management - # -------------------------------------------------- - CoreDbTxLevelFn* = proc(): int {.noRaise.} - CoreDbTxCommitFn* = proc() {.noRaise.} - CoreDbTxRollbackFn* = proc() {.noRaise.} - CoreDbTxDisposeFn* = proc() {.noRaise.} - - CoreDbTxFns* = object - levelFn*: CoreDbTxLevelFn - commitFn*: CoreDbTxCommitFn - rollbackFn*: CoreDbTxRollbackFn - disposeFn*: CoreDbTxDisposeFn - - - # -------------------------------------------------- - # Sub-descriptor: capture recorder methods - # -------------------------------------------------- - CoreDbCaptRecorderFn* = proc(): CoreDbRef {.noRaise.} - CoreDbCaptLogDbFn* = proc(): TableRef[Blob,Blob] {.noRaise.} - CoreDbCaptFlagsFn* = proc(): set[CoreDbCaptFlags] {.noRaise.} - CoreDbCaptForgetFn* = proc() {.noRaise.} - - CoreDbCaptFns* = object - recorderFn*: CoreDbCaptRecorderFn - logDbFn*: CoreDbCaptLogDbFn - getFlagsFn*: CoreDbCaptFlagsFn - forgetFn*: CoreDbCaptForgetFn - - # -------------------------------------------------- # Production descriptors # -------------------------------------------------- - CoreDbRef* = ref object of RootRef + CoreDbRef* = ref object ## Database descriptor dbType*: CoreDbType ## Type of database backend + defCtx*: CoreDbCtxRef ## Default context + + # Optional api interface (can be re-directed/intercepted) + ariApi*: AristoApiRef ## `Aristo` api + kvtApi*: KvtApiRef ## `KVT` api + + # Optional profiling and debugging stuff trackNewApi*: bool ## Debugging, support trackLedgerApi*: bool ## Debugging, suggestion for subsequent ledger profTab*: CoreDbProfListRef ## Profiling data (if any) ledgerHook*: RootRef ## Debugging/profiling, to be used by ledger - methods*: CoreDbBaseFns - CoreDbErrorRef* = ref object of RootRef + CoreDbCtxRef* = ref object + ## Shared context for `CoreDbMptRef`, `CoreDbAccRef`, `CoreDbKvtRef` + parent*: CoreDbRef + mpt*: AristoDbRef ## `Aristo` database + kvt*: KvtDbRef ## `KVT` key-value table + + CoreDbKvtRef* = distinct CoreDbCtxRef + ## Statically initialised Key-Value pair table + + CoreDbAccRef* = distinct CoreDbCtxRef + ## Similar to `CoreDbKvtRef`, only dealing with `Aristo` accounts + + CoreDbMptRef* = distinct CoreDbCtxRef + ## Generic MPT + + CoreDbTxRef* = ref object + ## Transaction descriptor + ctx*: CoreDbCtxRef ## Context (also contains `Aristo` descriptor) + aTx*: AristoTxRef ## `Aristo` transaction (if any) + kTx*: KvtTxRef ## `KVT` transaction (if any) + + CoreDbErrorRef* = ref object ## Generic error object error*: CoreDbErrorCode - parent*: CoreDbRef + ctx*: string ## Context where the exception or error occured + case isAristo*: bool + of true: + aErr*: AristoError + else: + kErr*: KvtError - CoreDbKvtBackendRef* = ref object of RootRef - ## Backend wrapper for direct backend access - parent*: CoreDbRef +when false: # TODO + type + # -------------------------------------------------- + # Sub-descriptor: capture recorder methods + # -------------------------------------------------- + CoreDbCaptRecorderFn* = proc(): CoreDbRef {.noRaise.} + CoreDbCaptLogDbFn* = proc(): TableRef[Blob,Blob] {.noRaise.} + CoreDbCaptFlagsFn* = proc(): set[CoreDbCaptFlags] {.noRaise.} + CoreDbCaptForgetFn* = proc() {.noRaise.} - CoreDbMptBackendRef* = ref object of RootRef - ## Backend wrapper for direct backend access - parent*: CoreDbRef + CoreDbCaptFns* = object + recorderFn*: CoreDbCaptRecorderFn + logDbFn*: CoreDbCaptLogDbFn + getFlagsFn*: CoreDbCaptFlagsFn + forgetFn*: CoreDbCaptForgetFn - CoreDbAccBackendRef* = ref object of RootRef - ## Backend wrapper for direct backend access - parent*: CoreDbRef - - CoreDbKvtRef* = ref object of RootRef - ## Statically initialised Key-Value pair table living in `CoreDbRef` - parent*: CoreDbRef - methods*: CoreDbKvtFns - - CoreDbCtxRef* = ref object of RootRef - ## Context for `CoreDbMptRef` and `CoreDbAccRef` - parent*: CoreDbRef - methods*: CoreDbCtxFns - - CoreDbMptRef* = ref object of RootRef - ## Hexary/Merkle-Patricia tree derived from `CoreDbRef`, will be - ## initialised on-the-fly. - parent*: CoreDbRef - methods*: CoreDbMptFns - - CoreDbAccRef* = ref object of RootRef - ## Similar to `CoreDbKvtRef`, only dealing with `CoreDbAccount` data - ## rather than `Blob` values. - parent*: CoreDbRef - methods*: CoreDbAccFns - - CoreDbTxRef* = ref object of RootRef - ## Transaction descriptor derived from `CoreDbRef` - parent*: CoreDbRef - methods*: CoreDbTxFns - - CoreDbCaptRef* = ref object - ## Db transaction tracer derived from `CoreDbRef` - parent*: CoreDbRef - methods*: CoreDbCaptFns + CoreDbCaptRef* = ref object + ## Db transaction tracer derived from `CoreDbRef` + parent*: CoreDbRef + methods*: CoreDbCaptFns # ------------------------------------------------------------------------------ # End diff --git a/nimbus/db/core_db/base/validate.nim b/nimbus/db/core_db/base/validate.nim index 616936077..8407546f8 100644 --- a/nimbus/db/core_db/base/validate.nim +++ b/nimbus/db/core_db/base/validate.nim @@ -10,135 +10,47 @@ {.push raises: [].} import + ../../aristo, ./base_desc type - EphemMethodsDesc = - CoreDbKvtBackendRef | CoreDbMptBackendRef | CoreDbAccBackendRef - - MethodsDesc = - CoreDbKvtRef | - CoreDbCtxRef | CoreDbMptRef | CoreDbAccRef | - CoreDbTxRef | - CoreDbCaptRef - - ValidateDesc* = MethodsDesc | EphemMethodsDesc | CoreDbErrorRef + ValidateSubDesc* = CoreDbCtxRef | CoreDbTxRef # | CoreDbCaptRef # ------------------------------------------------------------------------------ # Private helpers # ------------------------------------------------------------------------------ -proc validateMethodsDesc(base: CoreDbBaseFns) = - doAssert not base.destroyFn.isNil - doAssert not base.errorPrintFn.isNil - doAssert not base.levelFn.isNil - doAssert not base.newKvtFn.isNil - doAssert not base.newCtxFn.isNil - doAssert not base.newCtxFromTxFn.isNil - doAssert not base.swapCtxFn.isNil - doAssert not base.beginFn.isNil - # doAssert not base.newCaptureFn.isNil # currently disabled - doAssert not base.persistentFn.isNil - -proc validateMethodsDesc(kvt: CoreDbKvtFns) = - doAssert not kvt.backendFn.isNil - doAssert not kvt.getFn.isNil - doAssert not kvt.lenFn.isNil - doAssert not kvt.delFn.isNil - doAssert not kvt.putFn.isNil - doAssert not kvt.hasKeyFn.isNil - doAssert not kvt.forgetFn.isNil - -proc validateMethodsDesc(ctx: CoreDbCtxFns) = - doAssert not ctx.getAccountsFn.isNil - doAssert not ctx.getColumnFn.isNil - doAssert not ctx.forgetFn.isNil - -proc validateMethodsDesc(fns: CoreDbMptFns) = - doAssert not fns.backendFn.isNil - doAssert not fns.fetchFn.isNil - doAssert not fns.deleteFn.isNil - doAssert not fns.mergeFn.isNil - doAssert not fns.hasPathFn.isNil - doAssert not fns.stateFn.isNil - -proc validateMethodsDesc(fns: CoreDbAccFns) = - doAssert not fns.backendFn.isNil - doAssert not fns.fetchFn.isNil - doAssert not fns.clearStorageFn.isNil - doAssert not fns.deleteFn.isNil - doAssert not fns.hasPathFn.isNil - doAssert not fns.mergeFn.isNil - doAssert not fns.stateFn.isNil - - doAssert not fns.slotFetchFn.isNil - doAssert not fns.slotDeleteFn.isNil - doAssert not fns.slotHasPathFn.isNil - doAssert not fns.slotMergeFn.isNil - doAssert not fns.slotStateFn.isNil - doAssert not fns.slotStateEmptyFn.isNil - -# ------------ - -proc validateMethodsDesc(e: CoreDbErrorRef) = - doAssert e.error != CoreDbErrorCode(0) - doAssert not e.isNil - doAssert not e.parent.isNil - -proc validateMethodsDesc(eph: EphemMethodsDesc) = - doAssert not eph.isNil - doAssert not eph.parent.isNil - -proc validateMethodsDesc(kvt: CoreDbKvtRef) = - doAssert not kvt.isNil - doAssert not kvt.parent.isNil - kvt.methods.validateMethodsDesc - -proc validateMethodsDesc(ctx: CoreDbCtxRef) = +proc validateSubDescRef(ctx: CoreDbCtxRef) = doAssert not ctx.isNil doAssert not ctx.parent.isNil - ctx.methods.validateMethodsDesc + doAssert not ctx.mpt.isNil + doAssert not ctx.kvt.isNil -proc validateMethodsDesc(mpt: CoreDbMptRef) = - doAssert not mpt.isNil - doAssert not mpt.parent.isNil - mpt.methods.validateMethodsDesc - -proc validateMethodsDesc(acc: CoreDbAccRef) = - doAssert not acc.isNil - doAssert not acc.parent.isNil - acc.methods.validateMethodsDesc +proc validateSubDescRef(tx: CoreDbTxRef) = + doAssert not tx.isNil + doAssert not tx.ctx.isNil + doAssert not tx.aTx.isNil + doAssert not tx.kTx.isNil when false: # currently disabled - proc validateMethodsDesc(cpt: CoreDbCaptRef) = + proc validateSubDescRef(cpt: CoreDbCaptRef) = doAssert not cpt.isNil doAssert not cpt.parent.isNil doAssert not cpt.methods.recorderFn.isNil doAssert not cpt.methods.getFlagsFn.isNil doAssert not cpt.methods.forgetFn.isNil -proc validateMethodsDesc(tx: CoreDbTxRef) = - doAssert not tx.isNil - doAssert not tx.parent.isNil - doAssert not tx.methods.levelFn.isNil - doAssert not tx.methods.commitFn.isNil - doAssert not tx.methods.rollbackFn.isNil - doAssert not tx.methods.disposeFn.isNil - -proc validateMethodsDesc(db: CoreDbRef) = - doAssert not db.isNil - doAssert db.dbType != CoreDbType(0) - db.methods.validateMethodsDesc - # ------------------------------------------------------------------------------ # Public debugging helpers # ------------------------------------------------------------------------------ -proc validate*(dsc: ValidateDesc) = - dsc.validateMethodsDesc +proc validate*(dsc: ValidateSubDesc) = + dsc.validateSubDescRef proc validate*(db: CoreDbRef) = - db.validateMethodsDesc + doAssert not db.isNil + doAssert db.dbType != CoreDbType(0) + db.defCtx.validate # ------------------------------------------------------------------------------ # End diff --git a/nimbus/db/core_db/base_iterators.nim b/nimbus/db/core_db/base_iterators.nim index c6b238047..1a9902986 100644 --- a/nimbus/db/core_db/base_iterators.nim +++ b/nimbus/db/core_db/base_iterators.nim @@ -13,10 +13,23 @@ import std/typetraits, eth/common, - ./backend/aristo_db, + ../aristo as use_ari, + ../aristo/[aristo_walk, aristo_serialise], + ../kvt as use_kvt, + ../kvt/[kvt_init/memory_only, kvt_walk], ./base/[api_tracking, base_desc], ./base +when CoreDbEnableApiJumpTable: + discard +else: + import + ../aristo/[aristo_desc, aristo_path, aristo_tx], + ../kvt/[kvt_desc, kvt_tx] + +include + ./backend/aristo_replicate + when CoreDbEnableApiTracking: import chronicles @@ -35,39 +48,43 @@ iterator pairs*(kvt: CoreDbKvtRef): (Blob, Blob) {.apiRaise.} = ## Iterator supported on memory DB (otherwise implementation dependent) ## kvt.setTrackNewApi KvtPairsIt - case kvt.parent.dbType: + case kvt.dbType: of AristoDbMemory: - for k,v in kvt.aristoKvtPairsMem(): + let p = kvt.call(forkTx, kvt.kvt, 0).valueOrApiError "kvt/pairs()" + defer: discard kvt.call(forget, p) + for (k,v) in use_kvt.MemBackendRef.walkPairs p: yield (k,v) of AristoDbVoid: - for k,v in kvt.aristoKvtPairsVoid(): + let p = kvt.call(forkTx, kvt.kvt, 0).valueOrApiError "kvt/pairs()" + defer: discard kvt.call(forget, p) + for (k,v) in use_kvt.VoidBackendRef.walkPairs p: yield (k,v) of Ooops, AristoDbRocks: - raiseAssert: "Unsupported database type: " & $kvt.parent.dbType + raiseAssert: "Unsupported database type: " & $kvt.dbType kvt.ifTrackNewApi: debug newApiTxt, api, elapsed iterator pairs*(mpt: CoreDbMptRef): (Blob, Blob) = ## Trie traversal, only supported for `CoreDbMptRef` ## mpt.setTrackNewApi MptPairsIt - case mpt.parent.dbType: + case mpt.dbType: of AristoDbMemory, AristoDbRocks, AristoDbVoid: - for k,v in mpt.aristoMptPairs(): - yield (k,v) + for (path,data) in mpt.mpt.rightPairsGeneric mpt.rootID: + yield (mpt.call(pathAsBlob, path), data) of Ooops: - raiseAssert: "Unsupported database type: " & $mpt.parent.dbType + raiseAssert: "Unsupported database type: " & $mpt.dbType mpt.ifTrackNewApi: debug newApiTxt, api, elapsed iterator slotPairs*(acc: CoreDbAccRef; accPath: Hash256): (Blob, Blob) = ## Trie traversal, only supported for `CoreDbMptRef` ## acc.setTrackNewApi AccSlotPairsIt - case acc.parent.dbType: + case acc.dbType: of AristoDbMemory, AristoDbRocks, AristoDbVoid: - for k,v in acc.aristoSlotPairs accPath: - yield (k,v) + for (path,data) in acc.mpt.rightPairsStorage accPath: + yield (acc.call(pathAsBlob, path), data) of Ooops: - raiseAssert: "Unsupported database type: " & $acc.parent.dbType + raiseAssert: "Unsupported database type: " & $acc.dbType acc.ifTrackNewApi: doAssert accPath.len == 32 debug newApiTxt, api, elapsed @@ -76,15 +93,15 @@ iterator replicate*(mpt: CoreDbMptRef): (Blob, Blob) {.apiRaise.} = ## Low level trie dump, only supported for non persistent `CoreDbMptRef` ## mpt.setTrackNewApi MptReplicateIt - case mpt.parent.dbType: + case mpt.dbType: of AristoDbMemory: - for k,v in aristoReplicateMem(mpt): + for k,v in aristoReplicate[use_ari.MemBackendRef](mpt): yield (k,v) of AristoDbVoid: - for k,v in aristoReplicateVoid(mpt): + for k,v in aristoReplicate[use_ari.VoidBackendRef](mpt): yield (k,v) of Ooops, AristoDbRocks: - raiseAssert: "Unsupported database type: " & $mpt.parent.dbType + raiseAssert: "Unsupported database type: " & $mpt.dbType mpt.ifTrackNewApi: debug newApiTxt, api, elapsed # ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/base_iterators_persistent.nim b/nimbus/db/core_db/base_iterators_persistent.nim index 0297e6edf..73d62c09a 100644 --- a/nimbus/db/core_db/base_iterators_persistent.nim +++ b/nimbus/db/core_db/base_iterators_persistent.nim @@ -13,10 +13,16 @@ import std/typetraits, eth/common, - ./backend/[aristo_db, aristo_rocksdb], + ../aristo as use_ari, + ../aristo/aristo_init/rocks_db, + ../aristo/[aristo_desc, aristo_walk/persistent, aristo_tx], + ../kvt, # needed for `aristo_replicate` ./base/[api_tracking, base_desc], ./base +include + ./backend/aristo_replicate + when CoreDbEnableApiTracking: import chronicles @@ -25,7 +31,7 @@ when CoreDbEnableApiTracking: newApiTxt = logTxt & "API" # Annotation helper(s) -{.pragma: rlpRaise, gcsafe, raises: [AristoApiRlpError].} +{.pragma: rlpRaise, gcsafe, raises: [CoreDbApiError].} # ------------------------------------------------------------------------------ # Public iterators @@ -35,18 +41,18 @@ iterator replicatePersistent*(mpt: CoreDbMptRef): (Blob, Blob) {.rlpRaise.} = ## Extended version of `replicate()` for `Aristo` persistent backend. ## mpt.setTrackNewApi MptReplicateIt - case mpt.parent.dbType: + case mpt.dbType: of AristoDbMemory: - for k,v in aristoReplicateMem(mpt): + for k,v in aristoReplicate[use_ari.MemBackendRef](mpt): yield (k,v) of AristoDbVoid: - for k,v in aristoReplicateVoid(mpt): + for k,v in aristoReplicate[use_ari.VoidBackendRef](mpt): yield (k,v) of AristoDbRocks: - for k,v in aristoReplicateRdb(mpt): - yield (k,v) + for k, v in aristoReplicate[rocks_db.RdbBackendRef](mpt): + yield (k, v) else: - raiseAssert: "Unsupported database type: " & $mpt.parent.dbType + raiseAssert: "Unsupported database type: " & $mpt.dbType mpt.ifTrackNewApi: debug newApiTxt, api, elapsed # ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/core_apps.nim b/nimbus/db/core_db/core_apps.nim index 70c449b11..a209931fc 100644 --- a/nimbus/db/core_db/core_apps.nim +++ b/nimbus/db/core_db/core_apps.nim @@ -310,15 +310,7 @@ proc getSavedStateBlockNumber*( ## Returns the block number registered when the database was last time ## updated, or `BlockNumber(0)` if there was no updata found. ## - ## This function verifies the state consistency of the database and throws - ## an assert exception if that fails. So the function will only apply to a - ## saved database state. For an an opportunistic use, the `relax` argument - ## can be set `true` so this function also returns the block number if the - ## state consistency check fails. - ## - # FIXME: This construct following will be replaced by a proper - # `CoreDb` method. - db.ctx.getColumn(CtGeneric).backend.toAristoSavedStateBlockNumber() + db.stateBlockNumber() proc getBlockHeader*( db: CoreDbRef; diff --git a/nimbus/db/core_db/memory_only.nim b/nimbus/db/core_db/memory_only.nim index 43e888917..b0922decc 100644 --- a/nimbus/db/core_db/memory_only.nim +++ b/nimbus/db/core_db/memory_only.nim @@ -24,20 +24,7 @@ export base, base_iterators, common, - core_apps, - - # see `aristo_db` - isAristo, - toAristo, - toAristoProfData, - toAristoSavedStateBlockNumber, - - # Standard interface for calculating merkle hash signatures (see `aristo`) - MerkleSignRef, - merkleSignBegin, - merkleSignAdd, - merkleSignCommit, - to + core_apps # ------------------------------------------------------------------------------ # Public constructors diff --git a/nimbus/db/kvt.nim b/nimbus/db/kvt.nim index d50c23b97..a4af8dfb8 100644 --- a/nimbus/db/kvt.nim +++ b/nimbus/db/kvt.nim @@ -23,6 +23,7 @@ import export MemBackendRef, VoidBackendRef, + finish, init import diff --git a/nimbus/db/kvt/kvt_tx.nim b/nimbus/db/kvt/kvt_tx.nim index b50cfe11e..21609cd7c 100644 --- a/nimbus/db/kvt/kvt_tx.nim +++ b/nimbus/db/kvt/kvt_tx.nim @@ -32,7 +32,7 @@ func isTop*(tx: KvtTxRef): bool = ## level transaction. tx.txFrameIsTop() -func level*(tx: KvtTxRef): int = +func txLevel*(tx: KvtTxRef): int = ## Getter, positive nesting level of transaction argument `tx` tx.txFrameLevel() diff --git a/nimbus/db/ledger/base/api_tracking.nim b/nimbus/db/ledger/base/api_tracking.nim index eb7239f8f..cfd42cde9 100644 --- a/nimbus/db/ledger/base/api_tracking.nim +++ b/nimbus/db/ledger/base/api_tracking.nim @@ -97,9 +97,6 @@ func toStr*(w: EthAddress): string = func toStr*(w: Hash256): string = w.data.oaToStr -func toStr*(w: CoreDbMptRef): string = - if w.CoreDbMptRef.isNil: "nil" else: "MptRef" - func toStr*(w: CodeBytesRef): string = if w.CodeBytesRef.isNil: "nil" else: "[" & $w.bytes.len & "]" diff --git a/tests/test_coredb.nim b/tests/test_coredb.nim index 8facd4670..f54896354 100644 --- a/tests/test_coredb.nim +++ b/tests/test_coredb.nim @@ -357,7 +357,7 @@ when isMainModule: #dbType = CdbAristoDualRocks, capture = capture, pruneHistory = true, - #profilingOk = true, + profilingOk = true, #finalDiskCleanUpOk = false, oldLogAlign = true ) diff --git a/tests/test_coredb/test_chainsync.nim b/tests/test_coredb/test_chainsync.nim index bc657f3e0..c0cc5747d 100644 --- a/tests/test_coredb/test_chainsync.nim +++ b/tests/test_coredb/test_chainsync.nim @@ -184,9 +184,9 @@ proc test_chainSync*( # API data need to be captured so it will be available after the services # have terminated. when CoreDbEnableApiProfiling: - # terminated. - (aristoProfData, kvtProfData) = com.db.toAristoProfData() - cdbProfData = com.db.dbProfData() + aristoProfData = com.db.ariApi.AristoApiProfRef.data + kvtProfData = com.db.kvtApi.KvtApiProfRef.data + cdbProfData = com.db.profTab when LedgerEnableApiProfiling: ldgProfData = com.db.ldgProfData()