diff --git a/nimbus/db/core_db/backend/aristo_db.nim b/nimbus/db/core_db/backend/aristo_db.nim index aaba198c7..2eb4aa3e8 100644 --- a/nimbus/db/core_db/backend/aristo_db.nim +++ b/nimbus/db/core_db/backend/aristo_db.nim @@ -13,11 +13,14 @@ import eth/common, results, - "../.."/[aristo, aristo/aristo_desc, aristo/aristo_init/memory_only], + "../.."/[aristo, aristo/aristo_desc, aristo/aristo_walk], "../.."/[kvt, kvt/kvt_desc, kvt/kvt_init/memory_only], ".."/[base, base/base_desc], ./aristo_db/[common_desc, handlers_aristo, handlers_kvt] +include + ./aristo_db/aristo_replicate + # Annotation helpers {.pragma: noRaise, gcsafe, raises: [].} {.pragma: rlpRaise, gcsafe, raises: [AristoApiRlpError].} @@ -39,9 +42,6 @@ type # Private helpers # ------------------------------------------------------------------------------ -template valueOrApiError[U,V](rc: Result[U,V]; info: static[string]): U = - rc.valueOr: raise (ref AristoApiRlpError)(msg: info) - func notImplemented[T]( _: typedesc[T]; db: AristoCoreDbRef; @@ -50,39 +50,6 @@ func notImplemented[T]( ## Applies only to `Aristo` methods err((VertexID(0),aristo.NotImplemented).toError(db, info)) -# ------------------------------------------------------------------------------ -# Private call back functions (too large for embedding to maintain) -# ------------------------------------------------------------------------------ - -iterator kvtPairs( - T: typedesc; - dsc: CoreDxKvtRef; - info: static[string]; - ): (Blob,Blob) = - let p = dsc.kvt.forkTop.valueOrApiError info - defer: discard p.forget() - - dsc.methods.pairsIt = iterator(): (Blob, Blob) = - for (n,k,v) in T.walkPairs p: - yield (k,v) - - -iterator mptReplicate( - T: typedesc; - dsc: CoreDxMptRef; - info: static[string]; - ): (Blob,Blob) - {.rlpRaise.} = - let p = dsc.mpt.forkTop.valueOrApiError info - defer: discard p.forget() - - let root = dsc.root - for (vid,key,vtx,node) in T.replicate p: - if key.len == 32: - yield (@key, node.encode) - elif vid == root: - yield (@(key.to(Hash256).data), node.encode) - # ------------------------------------------------------------------------------ # Private tx and base methods # ------------------------------------------------------------------------------ @@ -159,12 +126,7 @@ proc baseMethods( newKvtFn: proc(saveMode: CoreDbSaveFlags): CoreDbRc[CoreDxKvtRef] = db.kdbBase.gc() - let dsc = ? db.kdbBase.newKvtHandler(saveMode, "newKvtFn()") - when K is MemBackendRef: - dsc.methods.pairsIt = iterator(): (Blob, Blob) = - for (n,k,v) in K.kvtPairs dsc: - yield (k,v) - ok(dsc), + ok(? db.kdbBase.newKvtHandler(saveMode, "newKvtFn()")), newMptFn: proc( root: CoreDbVidRef; @@ -172,12 +134,7 @@ proc baseMethods( saveMode: CoreDbSaveFlags; ): CoreDbRc[CoreDxMptRef] = db.kdbBase.gc() - let dsc = ? db.adbBase.newMptHandler(root, saveMode, "newMptFn()") - when K is MemBackendRef: - dsc.methods.replicateIt = iterator: (Blob,Blob) {.rlpRaise.} = - for w in T.mptReplicate(dsc, "forkTop() for replicateIt()"): - yield w - ok(dsc), + ok(? db.adbBase.newMptHandler(root, saveMode, "newMptFn()")), newAccFn: proc( root: CoreDbVidRef; @@ -292,6 +249,31 @@ func toAristo*(be: CoreDbAccBackendRef): AristoDbRef = if be.parent.isAristo: return be.AristoCoreDbAccBE.adb +# ------------------------------------------------------------------------------ +# Public aristo iterators +# ------------------------------------------------------------------------------ + +iterator aristoKvtPairs*(dsc: CoreDxKvtRef): (Blob,Blob) {.rlpRaise.} = + let p = dsc.to(KvtDbRef).forkTop.valueOrApiError "aristoKvtPairs()" + defer: discard p.forget() + for (k,v) in kvt.MemBackendRef.walkPairs p: + yield (k,v) + +iterator aristoMptPairs*(dsc: CoreDxMptRef): (Blob,Blob) {.rlpRaise.} = + let mpt = dsc.to(AristoDbRef) + for (k,v) in mpt.right LeafTie(root: dsc.rootID): + yield (k.path.pathAsBlob, mpt.serialise(v).valueOr(EmptyBlob)) + +iterator aristoReplicateMem*(dsc: CoreDxMptRef): (Blob,Blob) {.rlpRaise.} = + ## Instantiation for `MemBackendRef` + for k,v in aristoReplicate[aristo.MemBackendRef](dsc): + yield (k,v) + +iterator aristoReplicateVoid*(dsc: CoreDxMptRef): (Blob,Blob) {.rlpRaise.} = + ## Instantiation for `VoidBackendRef` + for k,v in aristoReplicate[aristo.VoidBackendRef](dsc): + yield (k,v) + # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/backend/aristo_db/aristo_replicate.nim b/nimbus/db/core_db/backend/aristo_db/aristo_replicate.nim new file mode 100644 index 000000000..27198c81a --- /dev/null +++ b/nimbus/db/core_db/backend/aristo_db/aristo_replicate.nim @@ -0,0 +1,39 @@ +# Nimbus +# Copyright (c) 2023-2024 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +## Generic iterator, prototype to be included (rather than imported). Using +## an include file avoids duplicating code because the `T` argument is not +## bound to any object type. Otherwise all object types would be required +## when providing this iterator for import. +## +## This is not wanted here, because the import of a **pesistent** object +## would always require extra linking. + +template valueOrApiError[U,V](rc: Result[U,V]; info: static[string]): U = + rc.valueOr: raise (ref AristoApiRlpError)(msg: info) + +iterator aristoReplicate[T]( + dsc: CoreDxMptRef; + ): (Blob,Blob) + {.gcsafe, raises: [AristoApiRlpError].} = + ## Generic iterator used for building dedicated backend iterators. + ## + let + root = dsc.rootID + mpt = dsc.to(AristoDbRef) + p = mpt.forkTop.valueOrApiError "aristoReplicate()" + defer: discard p.forget() + for (vid,key,vtx,node) in T.replicate(p): + if key.len == 32: + yield (@key, node.encode) + elif vid == root: + yield (@(key.to(Hash256).data), node.encode) + +# End diff --git a/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim b/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim index 9ebae5b29..02fa8ac49 100644 --- a/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim +++ b/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim @@ -368,14 +368,7 @@ proc mptMethods(cMpt: AristoChildDbRef): CoreDbMptFns = cMpt.mptPersistent("persistentFn()"), forgetFn: proc(): CoreDbRc[void] = - cMpt.forget("forgetFn()"), - - pairsIt: iterator: (Blob,Blob) = - for (k,v) in cMpt.mpt.right LeafTie(root: cMpt.root): - yield (k.path.pathAsBlob, cMpt.mpt.serialise(v).valueOr(EmptyBlob)), - - replicateIt: iterator: (Blob,Blob) = - discard) + cMpt.forget("forgetFn()")) # ------------------------------------------------------------------------------ # Private account call back functions @@ -580,8 +573,8 @@ proc gc*(base: AristoBaseRef) = # --------------------- -func mpt*(dsc: CoreDxMptRef): AristoDbRef = - dsc.AristoCoreDxMptRef.ctx.mpt +func to*(dsc: CoreDxMptRef, T: type AristoDbRef): T = + AristoCoreDxMptRef(dsc).ctx.mpt func rootID*(dsc: CoreDxMptRef): VertexID = dsc.AristoCoreDxMptRef.ctx.root diff --git a/nimbus/db/core_db/backend/aristo_db/handlers_kvt.nim b/nimbus/db/core_db/backend/aristo_db/handlers_kvt.nim index ab8e3d4c9..db0c087b3 100644 --- a/nimbus/db/core_db/backend/aristo_db/handlers_kvt.nim +++ b/nimbus/db/core_db/backend/aristo_db/handlers_kvt.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2023 Status Research & Development GmbH +# Copyright (c) 2023-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -246,10 +246,7 @@ proc kvtMethods(cKvt: KvtChildDbRef): CoreDbKvtFns = cKvt.kvtPersistent("persistentFn()"), forgetFn: proc(): CoreDbRc[void] = - cKvt.forget("forgetFn()"), - - pairsIt: iterator(): (Blob, Blob) = - discard) + cKvt.forget("forgetFn()")) # ------------------------------------------------------------------------------ # Public handlers and helpers @@ -313,8 +310,8 @@ proc gc*(base: KvtBaseRef) = # --------------------- -func kvt*(dsc: CoreDxKvtRef): KvtDbRef = - dsc.KvtCoreDxKvtRef.ctx.kvt +func to*(dsc: CoreDxKvtRef; T: type KvtDbRef): T = + KvtCoreDxKvtRef(dsc).ctx.kvt func txTop*( base: KvtBaseRef; diff --git a/nimbus/db/core_db/backend/aristo_rocksdb.nim b/nimbus/db/core_db/backend/aristo_rocksdb.nim index 99032baae..2ac80c985 100644 --- a/nimbus/db/core_db/backend/aristo_rocksdb.nim +++ b/nimbus/db/core_db/backend/aristo_rocksdb.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2023 Status Research & Development GmbH +# Copyright (c) 2023-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -13,9 +13,19 @@ import eth/common, results, - "../.."/[aristo, aristo/aristo_persistent, kvt, kvt/kvt_persistent], + ../../aristo, + ../../aristo/[aristo_persistent, aristo_walk/persistent], + ../../kvt, + ../../kvt/kvt_persistent, ../base, - ./aristo_db + ./aristo_db, + ./aristo_db/handlers_aristo + +include + ./aristo_db/aristo_replicate + +# Annotation helper(s) +{.pragma: rlpRaise, gcsafe, raises: [AristoApiRlpError].} # ------------------------------------------------------------------------------ # Public constructor @@ -33,6 +43,15 @@ proc newAristoRocksDbCoreDbRef*(path: string): CoreDbRef = aristo_persistent.RdbBackendRef, path) +# ------------------------------------------------------------------------------ +# Public aristo iterators +# ------------------------------------------------------------------------------ + +iterator aristoReplicateRdb*(dsc: CoreDxMptRef): (Blob,Blob) {.rlpRaise.} = + ## Instantiation for `VoidBackendRef` + for k,v in aristoReplicate[aristo_persistent.RdbBackendRef](dsc): + yield (k,v) + # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/backend/legacy_db.nim b/nimbus/db/core_db/backend/legacy_db.nim index f6a024d51..245c39687 100644 --- a/nimbus/db/core_db/backend/legacy_db.nim +++ b/nimbus/db/core_db/backend/legacy_db.nim @@ -226,11 +226,7 @@ proc kvtMethods(db: LegacyDbRef): CoreDbKvtFns = ok(), forgetFn: proc(): CoreDbRc[void] = - ok(), - - pairsIt: iterator(): (Blob, Blob) = - for k,v in tdb.pairsInMemoryDB: - yield (k,v)) + ok()) proc mptMethods(mpt: HexaryChildDbRef; db: LegacyDbRef): CoreDbMptFns = ## Hexary trie database handlers @@ -273,17 +269,7 @@ proc mptMethods(mpt: HexaryChildDbRef; db: LegacyDbRef): CoreDbMptFns = ok(), forgetFn: proc(): CoreDbRc[void] = - ok(), - - pairsIt: iterator: (Blob,Blob) {.gcsafe, raises: [LegacyApiRlpError].} = - reraiseRlpException("pairsIt()"): - for k,v in mpt.trie.pairs(): - yield (k,v), - - replicateIt: iterator: (Blob,Blob) {.gcsafe, raises: [LegacyApiRlpError].} = - reraiseRlpException("replicateIt()"): - for k,v in mpt.trie.replicate(): - yield (k,v)) + ok()) proc accMethods(mpt: HexaryChildDbRef; db: LegacyDbRef): CoreDbAccFns = ## Hexary trie database handlers @@ -509,6 +495,30 @@ func toLegacy*(be: CoreDbAccBackendRef): HexaryTrie = if be.parent.isLegacy: return be.LegacyCoreDbAccBE.mpt +# ------------------------------------------------------------------------------ +# Public legacy iterators +# ------------------------------------------------------------------------------ + +iterator legaKvtPairs*(kvt: CoreDxKvtRef): (Blob, Blob) = + for k,v in kvt.parent.LegacyDbRef.tdb.pairsInMemoryDB: + yield (k,v) + +iterator legaMptPairs*( + mpt: CoreDxMptRef; + ): (Blob,Blob) + {.gcsafe, raises: [LegacyApiRlpError].} = + reraiseRlpException("legaMptPairs()"): + for k,v in mpt.methods.backendFn().LegacyCoreDbMptBE.mpt.pairs(): + yield (k,v) + +iterator legaReplicate*( + mpt: CoreDxMptRef; + ): (Blob,Blob) + {.gcsafe, raises: [LegacyApiRlpError].} = + reraiseRlpException("legaReplicate()"): + for k,v in mpt.methods.backendFn().LegacyCoreDbMptBE.mpt.replicate(): + yield (k,v) + # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/backend/legacy_rocksdb.nim b/nimbus/db/core_db/backend/legacy_rocksdb.nim index b45e53975..afa2e7266 100644 --- a/nimbus/db/core_db/backend/legacy_rocksdb.nim +++ b/nimbus/db/core_db/backend/legacy_rocksdb.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2023 Status Research & Development GmbH +# Copyright (c) 2023-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) diff --git a/nimbus/db/core_db/base.nim b/nimbus/db/core_db/base.nim index a6b292773..fa4c0a8cf 100644 --- a/nimbus/db/core_db/base.nim +++ b/nimbus/db/core_db/base.nim @@ -100,11 +100,6 @@ const # Private helpers # ------------------------------------------------------------------------------ -template itNotImplemented(db: CoreDbRef, name: string) = - warn logTxt & "iterator not implemented", dbType=db.dbType, meth=name - -# --------- - when EnableApiTracking: when EnableApiProfiling: {.warning: "*** Provided API profiling for CoreDB (disabled by default)".} @@ -137,14 +132,14 @@ when ProvideLegacyAPI: code const ctx {.inject,used.} = s - template setTrackLegaApi( + template setTrackLegaApi*( w: CoreDbApiTrackRef; s: static[CoreDbFnInx]; ) = w.setTrackLegaApi(s): discard - template ifTrackLegaApi(w: CoreDbApiTrackRef; code: untyped) = + template ifTrackLegaApi*(w: CoreDbApiTrackRef; code: untyped) = when EnableApiTracking: w.endLegaApiIf: when EnableApiProfiling: @@ -164,14 +159,14 @@ template setTrackNewApi( code const ctx {.inject,used.} = s -template setTrackNewApi( +template setTrackNewApi*( w: CoreDxApiTrackRef; s: static[CoreDbFnInx]; ) = w.setTrackNewApi(s): discard -template ifTrackNewApi(w: CoreDxApiTrackRef; code: untyped) = +template ifTrackNewApi*(w: CoreDxApiTrackRef; code: untyped) = when EnableApiTracking: w.endNewApiIf: when EnableApiProfiling: @@ -202,14 +197,6 @@ func toCoreDxPhkRef(mpt: CoreDxMptRef): CoreDxPhkRef = proc(k: openArray[byte]): CoreDbRc[bool] = mpt.methods.hasPathFn(k.keccakHash.data) - result.methods.pairsIt = - iterator(): (Blob, Blob) = - mpt.parent.itNotImplemented("phk/pairs()") - - result.methods.replicateIt = - iterator(): (Blob, Blob) = - mpt.parent.itNotImplemented("phk/replicate()") - when AutoValidateDescriptors: result.validate @@ -244,15 +231,8 @@ proc bless*(db: CoreDbRef; child: CoreDbVidRef): CoreDbVidRef = proc bless*(db: CoreDbRef; child: CoreDxKvtRef): CoreDxKvtRef = - ## Complete sub-module descriptor, fill in `parent` and de-actvate - ## iterator for persistent database. + ## Complete sub-module descriptor, fill in `parent` child.parent = db - - # Disable interator for non-memory instances - if db.dbType in CoreDbPersistentTypes: - child.methods.pairsIt = iterator(): (Blob, Blob) = - db.itNotImplemented "pairs/kvt" - when AutoValidateDescriptors: child.validate child @@ -540,12 +520,6 @@ proc forget*(dsc: CoreDxKvtRef): CoreDbRc[void] {.discardable.} = result = dsc.methods.forgetFn() dsc.ifTrackNewApi: debug newApiTxt, ctx, elapsed, result -iterator pairs*(kvt: CoreDxKvtRef): (Blob, Blob) {.apiRaise.} = - ## Iterator supported on memory DB (otherwise implementation dependent) - kvt.setTrackNewApi KvtPairsIt - for k,v in kvt.methods.pairsIt(): yield (k,v) - kvt.ifTrackNewApi: debug newApiTxt, ctx, elapsed - # ------------------------------------------------------------------------------ # Public Merkle Patricia Tree, hexary trie constructors # ------------------------------------------------------------------------------ @@ -833,21 +807,6 @@ proc hasPath*(phk: CoreDxPhkRef; key: openArray[byte]): CoreDbRc[bool] = result = phk.methods.hasPathFn key phk.ifTrackNewApi: debug newApiTxt, ctx, elapsed, key=key.toStr, result - -iterator pairs*(mpt: CoreDxMptRef): (Blob, Blob) {.apiRaise.} = - ## Trie traversal, only supported for `CoreDxMptRef` - ## - mpt.setTrackNewApi MptPairsIt - for k,v in mpt.methods.pairsIt(): yield (k,v) - mpt.ifTrackNewApi: debug newApiTxt, ctx, elapsed - -iterator replicate*(mpt: CoreDxMptRef): (Blob, Blob) {.apiRaise.} = - ## Low level trie dump, only supported for `CoreDxMptRef` - ## - mpt.setTrackNewApi MptReplicateIt - for k,v in mpt.methods.replicateIt(): yield (k,v) - mpt.ifTrackNewApi: debug newApiTxt, ctx, elapsed - # ------------------------------------------------------------------------------ # Public trie database methods for accounts # ------------------------------------------------------------------------------ @@ -998,11 +957,6 @@ when ProvideLegacyAPI: result = kvt.distinctBase.hasKey(key).expect $ctx kvt.ifTrackLegaApi: debug legaApiTxt, ctx, elapsed, key=key.toStr, result - iterator pairs*(kvt: CoreDbKvtRef): (Blob, Blob) {.apiRaise.} = - kvt.setTrackLegaApi LegaKvtPairsIt - for k,v in kvt.distinctBase.pairs(): yield (k,v) - kvt.ifTrackLegaApi: debug legaApiTxt, ctx, elapsed - # ---------------- proc toMpt*(phk: CoreDbPhkRef): CoreDbMptRef = @@ -1115,19 +1069,6 @@ when ProvideLegacyAPI: result = phk.distinctBase.rootVid().hash().expect $ctx phk.ifTrackLegaApi: debug legaApiTxt, ctx, elapsed, result=result.toStr - - iterator pairs*(mpt: CoreDbMptRef): (Blob, Blob) {.apiRaise.} = - ## Trie traversal, not supported for `CoreDbPhkRef` - mpt.setTrackLegaApi LegaMptPairsIt - for k,v in mpt.distinctBase.pairs(): yield (k,v) - mpt.ifTrackLegaApi: debug legaApiTxt, ctx - - iterator replicate*(mpt: CoreDbMptRef): (Blob, Blob) {.apiRaise.} = - ## Low level trie dump, not supported for `CoreDbPhkRef` - mpt.setTrackLegaApi LegaMptReplicateIt - for k,v in mpt.distinctBase.replicate(): yield (k,v) - mpt.ifTrackLegaApi: debug legaApiTxt, ctx, elapsed - # ---------------- proc getTransactionID*(db: CoreDbRef): CoreDbTxID = diff --git a/nimbus/db/core_db/base/base_desc.nim b/nimbus/db/core_db/base/base_desc.nim index c26b3322a..8b11326a1 100644 --- a/nimbus/db/core_db/base/base_desc.nim +++ b/nimbus/db/core_db/base/base_desc.nim @@ -12,8 +12,7 @@ import eth/common, - results, - ../../../errors + results # Annotation helpers {.pragma: noRaise, gcsafe, raises: [].} @@ -130,7 +129,6 @@ type CoreDbKvtPersistentFn* = proc(): CoreDbRc[void] {.noRaise.} CoreDbKvtForgetFn* = proc(): CoreDbRc[void] {.noRaise.} CoreDbKvtHasKeyFn* = proc(k: openArray[byte]): CoreDbRc[bool] {.noRaise.} - CoreDbKvtPairsIt* = iterator(): (Blob,Blob) {.apiRaise.} CoreDbKvtFns* = object ## Methods for key-value table @@ -141,7 +139,6 @@ type hasKeyFn*: CoreDbKvtHasKeyFn persistentFn*: CoreDbKvtPersistentFn forgetFn*: CoreDbKvtForgetFn - pairsIt*: CoreDbKvtPairsIt # -------------------------------------------------- @@ -163,8 +160,6 @@ type CoreDbMptIsPruningFn* = proc(): bool {.noRaise.} CoreDbMptPersistentFn* = proc(): CoreDbRc[void] {.noRaise.} CoreDbMptForgetFn* = proc(): CoreDbRc[void] {.noRaise.} - CoreDbMptPairsIt* = iterator(): (Blob,Blob) {.apiRaise.} - CoreDbMptReplicateIt* = iterator(): (Blob,Blob) {.apiRaise.} CoreDbMptFns* = object ## Methods for trie objects @@ -177,8 +172,6 @@ type isPruningFn*: CoreDbMptIsPruningFn persistentFn*: CoreDbMptPersistentFn forgetFn*: CoreDbMptForgetFn - pairsIt*: CoreDbMptPairsIt - replicateIt*: CoreDbMptReplicateIt # ---------------------------------------------------- diff --git a/nimbus/db/core_db/base/validate.nim b/nimbus/db/core_db/base/validate.nim index 8678b70c9..7dad2e780 100644 --- a/nimbus/db/core_db/base/validate.nim +++ b/nimbus/db/core_db/base/validate.nim @@ -54,7 +54,6 @@ proc validateMethodsDesc(kvt: CoreDbKvtFns) = doAssert not kvt.persistentFn.isNil doAssert not kvt.forgetFn.isNil doAssert not kvt.hasKeyFn.isNil - doAssert not kvt.pairsIt.isNil proc validateMethodsDesc(fns: CoreDbMptFns) = doAssert not fns.backendFn.isNil @@ -66,8 +65,6 @@ proc validateMethodsDesc(fns: CoreDbMptFns) = doAssert not fns.isPruningFn.isNil doAssert not fns.persistentFn.isNil doAssert not fns.forgetFn.isNil - doAssert not fns.pairsIt.isNil - doAssert not fns.replicateIt.isNil proc validateMethodsDesc(fns: CoreDbAccFns) = doAssert not fns.backendFn.isNil diff --git a/nimbus/db/core_db/base_iterators.nim b/nimbus/db/core_db/base_iterators.nim new file mode 100644 index 000000000..bdddea878 --- /dev/null +++ b/nimbus/db/core_db/base_iterators.nim @@ -0,0 +1,109 @@ +# Nimbus +# Copyright (c) 2024 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +{.push raises: [].} + +import + std/typetraits, + eth/common, + ./backend/[aristo_db, legacy_db], + ./base/[api_tracking, base_desc], + ./base + +when CoreDbEnableApiTracking: + import chronicles + +const + ProvideLegacyAPI = CoreDbProvideLegacyAPI + logTxt = "CoreDb/it " + legaApiTxt = logTxt & "legacy API" + newApiTxt = logTxt & "API" + +# Annotation helper(s) +{.pragma: apiRaise, gcsafe, raises: [CoreDbApiError].} + +# ------------------------------------------------------------------------------ +# Public iterators +# ------------------------------------------------------------------------------ + +iterator pairs*(kvt: CoreDxKvtRef): (Blob, Blob) {.apiRaise.} = + ## Iterator supported on memory DB (otherwise implementation dependent) + ## + kvt.setTrackNewApi KvtPairsIt + case kvt.parent.dbType: + of LegacyDbMemory: + for k,v in kvt.legaKvtPairs(): + yield (k,v) + of AristoDbMemory: + for k,v in kvt.aristoKvtPairs(): + yield (k,v) + else: + raiseAssert: "Unsupported database type: " & $kvt.parent.dbType + kvt.ifTrackNewApi: debug newApiTxt, ctx, elapsed + +iterator pairs*(mpt: CoreDxMptRef): (Blob, Blob) {.apiRaise.} = + ## Trie traversal, only supported for `CoreDxMptRef` (not `Phk`) + ## + mpt.setTrackNewApi MptPairsIt + case mpt.parent.dbType: + of LegacyDbMemory, LegacyDbPersistent: + for k,v in mpt.legaMptPairs(): + yield (k,v) + of AristoDbMemory, AristoDbRocks, AristoDbVoid: + for k,v in mpt.aristoMptPairs(): + yield (k,v) + else: + raiseAssert: "Unsupported database type: " & $mpt.parent.dbType + mpt.ifTrackNewApi: + let trie = mpt.methods.getTrieFn() + debug newApiTxt, ctx, elapsed, trie + +iterator replicate*(mpt: CoreDxMptRef): (Blob, Blob) {.apiRaise.} = + ## Low level trie dump, only supported for `CoreDxMptRef` (not `Phk`) + ## + mpt.setTrackNewApi MptReplicateIt + case mpt.parent.dbType: + of LegacyDbMemory, LegacyDbPersistent: + for k,v in mpt.legaReplicate(): + yield (k,v) + of AristoDbMemory: + for k,v in aristoReplicateMem(mpt): + yield (k,v) + of AristoDbVoid: + for k,v in aristoReplicateVoid(mpt): + yield (k,v) + else: + raiseAssert: "Unsupported database type: " & $mpt.parent.dbType + mpt.ifTrackNewApi: + let trie = mpt.methods.getTrieFn() + debug newApiTxt, ctx, elapsed, trie + +when ProvideLegacyAPI: + + iterator pairs*(kvt: CoreDbKvtRef): (Blob, Blob) {.apiRaise.} = + kvt.setTrackLegaApi LegaKvtPairsIt + for k,v in kvt.distinctBase.pairs(): yield (k,v) + kvt.ifTrackLegaApi: debug legaApiTxt, ctx, elapsed + + iterator pairs*(mpt: CoreDbMptRef): (Blob, Blob) {.apiRaise.} = + ## Trie traversal, not supported for `CoreDbPhkRef` + mpt.setTrackLegaApi LegaMptPairsIt + for k,v in mpt.distinctBase.pairs(): yield (k,v) + mpt.ifTrackLegaApi: debug legaApiTxt, ctx + + iterator replicate*(mpt: CoreDbMptRef): (Blob, Blob) {.apiRaise.} = + ## Low level trie dump, not supported for `CoreDbPhkRef` + mpt.setTrackLegaApi LegaMptReplicateIt + for k,v in mpt.distinctBase.replicate(): yield (k,v) + mpt.ifTrackLegaApi: debug legaApiTxt, ctx, elapsed + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/base_iterators_persistent.nim b/nimbus/db/core_db/base_iterators_persistent.nim new file mode 100644 index 000000000..c396a94cf --- /dev/null +++ b/nimbus/db/core_db/base_iterators_persistent.nim @@ -0,0 +1,69 @@ +# Nimbus +# Copyright (c) 2024 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +{.push raises: [].} + +import + std/typetraits, + eth/common, + ./backend/[aristo_db, aristo_rocksdb, legacy_db], + ./base/[api_tracking, base_desc], + ./base + +when CoreDbEnableApiTracking: + import chronicles + +const + ProvideLegacyAPI = CoreDbProvideLegacyAPI + logTxt = "CoreDb/itp " + legaApiTxt = logTxt & "legacy API" + newApiTxt = logTxt & "API" + +# Annotation helper(s) +{.pragma: rlpRaise, gcsafe, raises: [AristoApiRlpError, LegacyApiRlpError].} + +# ------------------------------------------------------------------------------ +# Public iterators +# ------------------------------------------------------------------------------ + +iterator replicatePersistent*(mpt: CoreDxMptRef): (Blob, Blob) {.rlpRaise.} = + ## Extended version of `replicate()` for `Aristo` persistent backend. + ## + mpt.setTrackNewApi MptReplicateIt + case mpt.parent.dbType: + of LegacyDbMemory, LegacyDbPersistent: + for k,v in mpt.legaReplicate(): + yield (k,v) + of AristoDbMemory: + for k,v in aristoReplicateMem(mpt): + yield (k,v) + of AristoDbVoid: + for k,v in aristoReplicateVoid(mpt): + yield (k,v) + of AristoDbRocks: + for k,v in aristoReplicateRdb(mpt): + yield (k,v) + else: + raiseAssert: "Unsupported database type: " & $mpt.parent.dbType + mpt.ifTrackNewApi: + let trie = mpt.methods.getTrieFn() + debug newApiTxt, ctx, elapsed, trie + +when ProvideLegacyAPI: + + iterator replicatePersistent*(mpt: CoreDbMptRef): (Blob, Blob) {.rlpRaise.} = + ## Low level trie dump, not supported for `CoreDbPhkRef` + mpt.setTrackLegaApi LegaMptReplicateIt + for k,v in mpt.distinctBase.replicatePersistent(): yield (k,v) + mpt.ifTrackLegaApi: debug legaApiTxt, ctx, elapsed + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/memory_only.nim b/nimbus/db/core_db/memory_only.nim index 37d707534..9a0b3cd1a 100644 --- a/nimbus/db/core_db/memory_only.nim +++ b/nimbus/db/core_db/memory_only.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2023 Status Research & Development GmbH +# Copyright (c) 2023-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -14,15 +14,19 @@ import std/options, eth/[common, trie/db], ../aristo, - ./backend/[aristo_db, legacy_db], - #./core_apps_legacy as core_apps - ./core_apps_newapi as core_apps + ./backend/[aristo_db, legacy_db] +import + #./core_apps_legacy as core_apps + ./core_apps_newapi as core_apps import ./base except bless +import + ./base_iterators export base, + base_iterators, common, core_apps, @@ -41,7 +45,7 @@ export to # ------------------------------------------------------------------------------ -# Public constructor +# Public constructors # ------------------------------------------------------------------------------ proc newCoreDbRef*( diff --git a/nimbus/db/core_db/persistent.nim b/nimbus/db/core_db/persistent.nim index 8cd0d7557..3f62fa649 100644 --- a/nimbus/db/core_db/persistent.nim +++ b/nimbus/db/core_db/persistent.nim @@ -13,6 +13,12 @@ ## applications by importing `db/code_db/memory_only` (rather than ## `db/core_db/persistent`.) ## +## The right way to use this file on a conditional mode is to import it as in +## :: +## import ./path/to/core_db +## when ..condition.. +## import ./path/to/core_db/persistent +## {.push raises: [].} import @@ -23,14 +29,25 @@ import export memory_only +# This file is currently inconsistent due to the `dbBackend == rocksdb` hack +# which will be removed, soon (must go to the test base where such a compile +# time flag induced mechanism might be useful.) +# +# The `Aristo` backend has no business with `dbBackend` and will be extended +# in future. +{.warning: "Inconsistent API file needs to be de-uglified".} + # Allow hive sim to compile with dbBackend == none when dbBackend == rocksdb: import + base_iterators_persistent, ./backend/[aristo_rocksdb, legacy_rocksdb] export + base_iterators_persistent, toRocksStoreRef + proc newCoreDbRef*( dbType: static[CoreDbType]; # Database type symbol path: string; # Storage path for database @@ -48,6 +65,8 @@ proc newCoreDbRef*( else: {.error: "Unsupported dbType for persistent newCoreDbRef()".} + else: + {.error: "Unsupported dbBackend setting for persistent newCoreDbRef()".} proc newCoreDbRef*( dbType: static[CoreDbType]; # Database type symbol @@ -65,5 +84,8 @@ proc newCoreDbRef*( else: {.error: "Unsupported dbType for persistent newCoreDbRef()" & " with qidLayout argument".} + else: + {.error: "Unsupported dbBackend setting for persistent newCoreDbRef()" & + " with qidLayout argument".} # End