diff --git a/nimbus/db/core_db/backend/aristo_db.nim b/nimbus/db/core_db/backend/aristo_db.nim new file mode 100644 index 000000000..38f18a9c0 --- /dev/null +++ b/nimbus/db/core_db/backend/aristo_db.nim @@ -0,0 +1,285 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +{.push raises: [].} + +import + eth/common, + results, + "../.."/[aristo, aristo/aristo_desc, aristo/aristo_init/memory_only], + "../.."/[kvt, kvt/kvt_desc, kvt/kvt_init/memory_only], + ".."/[base, base/base_desc], + ./aristo_db/[common_desc, handlers_aristo, handlers_kvt] + +# Annotation helpers +{.pragma: noRaise, gcsafe, raises: [].} +{.pragma: rlpRaise, gcsafe, raises: [AristoApiRlpError].} + +export + AristoApiRlpError, + AristoCoreDbKvtBE, + memory_only + +type + AristoCoreDbRef* = ref object of CoreDbRef + ## Main descriptor + kdbBase: KvtBaseRef ## Kvt subsystem + adbBase: AristoBaseRef ## Aristo subsystem + + AristoCoreDbBE = ref object of CoreDbBackendRef + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +template valueOrApiError[U,V](rc: Result[U,V]; info: static[string]): U = + rc.valueOr: raise (ref AristoApiRlpError)(msg: info) + +func notImplemented[T]( + _: typedesc[T]; + db: AristoCoreDbRef; + info: string; + ): CoreDbRc[T] {.gcsafe.} = + ## Applies only to `Aristo` methods + err((VertexID(0),aristo.NotImplemented).toError(db, info)) + +# ------------------------------------------------------------------------------ +# Private call back functions (too large for embedding to maintain) +# ------------------------------------------------------------------------------ + +iterator kvtPairs( + T: typedesc; + dsc: CoreDxKvtRef; + info: static[string]; + ): (Blob,Blob) = + let p = dsc.kvt.forkTop.valueOrApiError info + defer: discard p.forget() + + dsc.methods.pairsIt = iterator(): (Blob, Blob) = + for (n,k,v) in T.walkPairs p: + yield (k,v) + + +iterator mptReplicate( + T: typedesc; + dsc: CoreDxMptRef; + info: static[string]; + ): (Blob,Blob) + {.rlpRaise.} = + let p = dsc.mpt.forkTop.valueOrApiError info + defer: discard p.forget() + + let root = dsc.root + for (vid,key,vtx,node) in T.replicate p: + if key.len == 32: + yield (@key, node.encode) + elif vid == root: + yield (@(key.to(Hash256).data), node.encode) + +# ------------------------------------------------------------------------------ +# Private tx and base methods +# ------------------------------------------------------------------------------ + +proc txMethods( + db: AristoCoreDbRef; + aTx: AristoTxRef; + kTx: KvtTxRef; + ): CoreDbTxFns = + ## To be constructed by some `CoreDbBaseFns` function + CoreDbTxFns( + commitFn: proc(ignore: bool): CoreDbRc[void] = + const info = "commitFn()" + ? aTx.commit.toVoidRc(db, info) + ? kTx.commit.toVoidRc(db, info) + ok(), + + rollbackFn: proc(): CoreDbRc[void] = + const info = "rollbackFn()" + ? aTx.rollback.toVoidRc(db, info) + ? kTx.rollback.toVoidRc(db, info) + ok(), + + disposeFn: proc(): CoreDbRc[void] = + const info = "disposeFn()" + if aTx.isTop: ? aTx.rollback.toVoidRc(db, info) + if kTx.isTop: ? kTx.rollback.toVoidRc(db, info) + ok(), + + safeDisposeFn: proc(): CoreDbRc[void] = + const info = "safeDisposeFn()" + if aTx.isTop: ? aTx.rollback.toVoidRc(db, info) + if kTx.isTop: ? kTx.rollback.toVoidRc(db, info) + ok()) + + +proc baseMethods( + db: AristoCoreDbRef; + A: typedesc; + K: typedesc; + ): CoreDbBaseFns = + let db = db + CoreDbBaseFns( + backendFn: proc(): CoreDbBackendRef = + db.bless(AristoCoreDbBE()), + + destroyFn: proc(flush: bool) = + db.adbBase.destroy(flush) + db.kdbBase.destroy(flush), + + vidHashFn: proc(vid: CoreDbVidRef; update: bool): CoreDbRc[Hash256] = + db.adbBase.getHash(vid, update, "vidHashFn()"), + + errorPrintFn: proc(e: CoreDbErrorRef): string = + e.errorPrint(), + + legacySetupFn: proc() = + discard, + + getRootFn: proc(root: Hash256; createOk: bool): CoreDbRc[CoreDbVidRef] = + db.adbBase.getVid(root, createOk, "getRootFn()"), + + newKvtFn: proc(saveMode: CoreDbSaveFlags): CoreDbRc[CoreDxKvtRef] = + db.kdbBase.gc() + let dsc = ? db.kdbBase.newKvtHandler(saveMode, "newKvtFn()") + when K is MemBackendRef: + dsc.methods.pairsIt = iterator(): (Blob, Blob) = + for (n,k,v) in K.kvtPairs dsc: + yield (k,v) + ok(dsc), + + newMptFn: proc( + root: CoreDbVidRef; + prune: bool; + saveMode: CoreDbSaveFlags; + ): CoreDbRc[CoreDxMptRef] = + db.kdbBase.gc() + let dsc = ? db.adbBase.newMptHandler(root, prune, saveMode, "newMptFn()") + when K is MemBackendRef: + dsc.methods.replicateIt = iterator: (Blob,Blob) {.rlpRaise.} = + for w in T.mptReplicate(dsc, "forkTop() for replicateIt()"): + yield w + ok(dsc), + + newAccFn: proc( + root: CoreDbVidRef; + prune: bool; + saveMode: CoreDbSaveFlags; + ): CoreDbRc[CoreDxAccRef] = + db.kdbBase.gc() + ok(? db.adbBase.newAccHandler(prune, saveMode, "newAccFn()")), + + beginFn: proc(): CoreDbRc[CoreDxTxRef] = + const info = "beginFn()" + let + aTx = ? db.adbBase.txBegin(info) + kTx = ? db.kdbBase.txBegin(info) + ok(db.bless CoreDxTxRef(methods: db.txMethods(aTx, kTx))), + + getIdFn: proc(): CoreDbRc[CoreDxTxID] = + CoreDxTxID.notImplemented(db, "getIdFn()"), + + captureFn: proc(flags: set[CoreDbCaptFlags]): CoreDbRc[CoreDxCaptRef] = + CoreDxCaptRef.notImplemented(db, "capture()")) + +# ------------------------------------------------------------------------------ +# Private constructor helpers +# ------------------------------------------------------------------------------ + +proc create( + dbType: CoreDbType; + kdb: KvtDbRef; + K: typedesc; + adb: AristoDbRef; + A: typedesc; + ): CoreDbRef = + ## Constructor helper + + # Local extensions + var db = AristoCoreDbRef() + db.adbBase = AristoBaseRef.init(db, adb) + db.kdbBase = KvtBaseRef.init(db, kdb) + + # Base descriptor + db.dbType = dbType + db.methods = db.baseMethods(A,K) + db.bless + +proc init( + dbType: CoreDbType; + K: typedesc; + A: typedesc; + qlr: QidLayoutRef; + ): CoreDbRef = + dbType.create(KvtDbRef.init(K), K, AristoDbRef.init(A, qlr), A) + +proc init( + dbType: CoreDbType; + K: typedesc; + A: typedesc; + ): CoreDbRef = + dbType.create(KvtDbRef.init(K), K, AristoDbRef.init(A), A) + +# ------------------------------------------------------------------------------ +# Public constructor helpers +# ------------------------------------------------------------------------------ + +proc init*( + dbType: CoreDbType; + K: typedesc; + A: typedesc; + path: string; + qlr: QidLayoutRef; + ): CoreDbRef = + dbType.create( + KvtDbRef.init(K, path).expect "Kvt/RocksDB init() failed", K, + AristoDbRef.init(A, path, qlr).expect "Aristo/RocksDB init() failed", A) + +proc init*( + dbType: CoreDbType; + K: typedesc; + A: typedesc; + path: string; + ): CoreDbRef = + dbType.create( + KvtDbRef.init(K, path).expect "Kvt/RocksDB init() failed", K, + AristoDbRef.init(A, path).expect "Aristo/RocksDB init() failed", A) + +# ------------------------------------------------------------------------------ +# Public constructor +# ------------------------------------------------------------------------------ + +proc newAristoMemoryCoreDbRef*(qlr: QidLayoutRef): CoreDbRef = + AristoDbMemory.init(kvt.MemBackendRef, aristo.MemBackendRef, qlr) + +proc newAristoMemoryCoreDbRef*(): CoreDbRef = + AristoDbMemory.init(kvt.MemBackendRef, aristo.MemBackendRef) + +proc newAristoVoidCoreDbRef*(): CoreDbRef = + AristoDbVoid.init(kvt.VoidBackendRef, aristo.VoidBackendRef) + +# ------------------------------------------------------------------------------ +# Public helpers for direct backend access +# ------------------------------------------------------------------------------ + +func toAristo*(be: CoreDbKvtBackendRef): KvtDbRef = + if be.parent.isAristo: + return be.AristoCoreDbKvtBE.kdb + +func toAristo*(be: CoreDbMptBackendRef): AristoDbRef = + if be.parent.isAristo: + return be.AristoCoreDbMptBE.adb + +func toAristo*(be: CoreDbAccBackendRef): AristoDbRef = + if be.parent.isAristo: + return be.AristoCoreDbAccBE.adb + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/backend/aristo_db/common_desc.nim b/nimbus/db/core_db/backend/aristo_db/common_desc.nim new file mode 100644 index 000000000..3d1cd36dd --- /dev/null +++ b/nimbus/db/core_db/backend/aristo_db/common_desc.nim @@ -0,0 +1,53 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +{.push raises: [].} + +import + ../../../../errors, + "../../.."/[aristo, kvt], + ../../base/base_desc + +type + AristoApiRlpError* = object of CoreDbApiError + ## For re-routing exceptions in iterator closure + + AristoCoreDbError* = ref object of CoreDbErrorRef + ## Error return code + ctx*: string ## Context where the exception or error occured + case isAristo*: bool + of true: + aVid*: VertexID + aErr*: AristoError + else: + kErr*: KvtError + +# ------------------------------------------------------------------------------ +# Public helpers +# ------------------------------------------------------------------------------ + +func isAristo*(be: CoreDbRef): bool = + be.dbType in {AristoDbMemory, AristoDbRocks} + +func errorPrint*(e: CoreDbErrorRef): string = + if not e.isNil: + let e = e.AristoCoreDbError + result = if e.isAristo: "Aristo: " else: "Kvt: " + result &= "ctx=\"" & $e.ctx & "\"" & ", " + if e.isAristo: + if e.aVid.isValid: + result &= "vid=\"" & $e.aVid & "\"" & ", " + result &= "error=\"" & $e.aErr & "\"" + else: + result &= "error=\"" & $e.kErr & "\"" + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim b/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim new file mode 100644 index 000000000..d7887d3de --- /dev/null +++ b/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim @@ -0,0 +1,586 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +{.push raises: [].} + +import + chronicles, + eth/common, + results, + ../../../aristo, + ../../../aristo/aristo_desc, + ../../base, + ../../base/base_desc, + ./common_desc + +type + AristoBaseRef* = ref object + parent: CoreDbRef ## Opaque top level descriptor + adb: AristoDbRef ## Aristo MPT database + gq: seq[AristoChildDbRef] ## Garbage queue, deferred disposal + + AristoChildDbRef = ref AristoChildDbObj + AristoChildDbObj = object + ## Sub-handle for triggering destructor when it goes out of scope + base: AristoBaseRef ## Local base descriptor + root: VertexID ## State root + prune: bool ## Currently unused + mpt: AristoDbRef ## Descriptor, may be copy of `base.adb` + saveMode: CoreDbSaveFlags ## When to store/discard + + AristoCoreDxMptRef = ref object of CoreDxMptRef + ## Some extendion to recover embedded state + ctx: AristoChildDbRef ## Embedded state, typical var name: `cMpt` + + AristoCoreDbVid* = ref object of CoreDbVidRef + ## Vertex ID wrapper, optinally with *MPT* context + ctx: AristoDbRef ## Optional *MPT* context, might be `nil` + aVid: VertexID ## Refers to root vertex + createOk: bool ## Create new root vertex when appropriate + expHash: Hash256 ## Deferred validation + + AristoCoreDbMptBE* = ref object of CoreDbMptBackendRef + adb*: AristoDbRef + + AristoCoreDbAccBE* = ref object of CoreDbAccBackendRef + adb*: AristoDbRef + +logScope: + topics = "aristo-hdl" + +proc gc*(base: AristoBaseRef) {.gcsafe.} + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +template logTxt(info: static[string]): static[string] = + "CoreDb/adb " & info + +func isValid(vid: CoreDbVidRef): bool = + not vid.isNil and vid.ready + +func to*(vid: CoreDbVidRef; T: type VertexID): T = + if vid.isValid: + return vid.AristoCoreDbVid.aVid + +func createOk(vid: CoreDbVidRef): bool = + if vid.isValid: + return vid.AristoCoreDbVid.createOk + +# -------------- + +func toCoreDbAccount( + cMpt: AristoChildDbRef; + acc: AristoAccount; + ): CoreDbAccount = + let db = cMpt.base.parent + CoreDbAccount( + nonce: acc.nonce, + balance: acc.balance, + codeHash: acc.codeHash, + storageVid: db.bless AristoCoreDbVid(ctx: cMpt.mpt, aVid: acc.storageID)) + +func toPayloadRef(acc: CoreDbAccount): PayloadRef = + PayloadRef( + pType: AccountData, + account: AristoAccount( + nonce: acc.nonce, + balance: acc.balance, + storageID: acc.storageVid.to(VertexID), + codeHash: acc.codeHash)) + +# -------------- + +func toErrorImpl( + e: AristoError; + db: CoreDbRef; + info: string; + error = Unspecified; + ): CoreDbErrorRef = + db.bless(error, AristoCoreDbError( + ctx: info, + isAristo: true, + aErr: e)) + +func toErrorImpl( + e: (VertexID,AristoError); + db: CoreDbRef; + info: string; + error = Unspecified; + ): CoreDbErrorRef = + db.bless(error, AristoCoreDbError( + ctx: info, + isAristo: true, + aVid: e[0], + aErr: e[1])) + + +func toRcImpl[T]( + rc: Result[T,(VertexID,AristoError)]; + db: CoreDbRef; + info: string; + error = Unspecified; + ): CoreDbRc[T] = + if rc.isOk: + when T is void: + return ok() + else: + return ok(rc.value) + err rc.error.toErrorImpl(db, info, error) + +func toRcImpl[T]( + rc: Result[T,AristoError]; + db: CoreDbRef; + info: string; + error = Unspecified; + ): CoreDbRc[T] = + if rc.isOk: + when T is void: + return ok() + else: + return ok(rc.value) + err((VertexID(0),rc.error).toErrorImpl(db, info, error)) + + +func toVoidRcImpl[T]( + rc: Result[T,(VertexID,AristoError)]; + db: CoreDbRef; + info: string; + error = Unspecified; + ): CoreDbRc[void] = + if rc.isOk: + return ok() + err rc.error.toErrorImpl(db, info, error) + +func toVoidRcImpl[T]( + rc: Result[T,AristoError]; + db: CoreDbRef; + info: string; + error = Unspecified; + ): CoreDbRc[void] = + if rc.isOk: + return ok() + err((VertexID(0),rc.error).toErrorImpl(db, info, error)) + +# ------------------------------------------------------------------------------ +# Private call back functions (too large for keeping inline) +# ------------------------------------------------------------------------------ + +proc finish( + cMpt: AristoChildDbRef; + info: static[string]; + ): CoreDbRc[void] = + ## Hexary trie destructor to be called automagically when the argument + ## wrapper gets out of scope. + let + base = cMpt.base + db = base.parent + + result = ok() + + if cMpt.mpt != base.adb: + let rc = cMpt.mpt.forget() + if rc.isErr: + result = err(rc.error.toErrorImpl(db, info)) + cMpt.mpt = AristoDbRef(nil) # disables `=destroy` + + if cMpt.saveMode == AutoSave: + if base.adb.level == 0: + let rc = base.adb.stow(persistent = true) + if rc.isErr: + result = err(rc.error.toErrorImpl(db, info)) + +proc `=destroy`(cMpt: var AristoChildDbObj) = + ## Auto destructor + if not cMpt.mpt.isNil: + # Add to destructor batch queue unless direct reference + if cMpt.mpt != cMpt.base.adb or + cMpt.saveMode == AutoSave: + cMpt.base.gq.add AristoChildDbRef( + base: cMpt.base, + mpt: cMpt.mpt, + saveMode: cMpt.saveMode) + +# ------------------------------- + +proc mptFetch( + cMpt: AristoChildDbRef; + k: openArray[byte]; + info: static[string]; + ): CoreDbRc[Blob] = + let + db = cMpt.base.parent + rc = cMpt.mpt.fetchPayload(cMpt.root, k) + if rc.isOk: + return cMpt.mpt.serialise(rc.value).toRcImpl(db, info) + + if rc.error[1] != FetchPathNotFound: + return err(rc.error.toErrorImpl(db, info)) + + err rc.error.toErrorImpl(db, info, MptNotFound) + + +proc mptMerge( + cMpt: AristoChildDbRef; + k: openArray[byte]; + v: openArray[byte]; + info: static[string]; + ): CoreDbRc[void] = + let rc = cMpt.mpt.merge(cMpt.root, k, v) + if rc.isErr: + return err(rc.error.toErrorImpl(cMpt.base.parent, info)) + ok() + + +proc mptDelete( + cMpt: AristoChildDbRef; + k: openArray[byte]; + info: static[string]; + ): CoreDbRc[void] = + let rc = cMpt.mpt.delete(cMpt.root, k) + if rc.isErr: + return err(rc.error.toErrorImpl(cMpt.base.parent, info)) + ok() + +# ------------------------------- + +proc accFetch( + cMpt: AristoChildDbRef; + address: EthAddress; + info: static[string]; + ): CoreDbRc[CoreDbAccount] = + let + db = cMpt.base.parent + pyl = block: + let rc = cMpt.mpt.fetchPayload(cMpt.root, address.keccakHash.data) + if rc.isOk: + rc.value + elif rc.error[1] != FetchPathNotFound: + return err(rc.error.toErrorImpl(db, info)) + else: + return err(rc.error.toErrorImpl(db, info, AccNotFound)) + + if pyl.pType != AccountData: + let vePair = (pyl.account.storageID, PayloadTypeUnsupported) + return err(vePair.toErrorImpl(db, info & "/" & $pyl.pType)) + + ok cMpt.toCoreDbAccount pyl.account + + +proc accMerge( + cMpt: AristoChildDbRef; + address: EthAddress; + acc: CoreDbAccount; + info: static[string]; + ): CoreDbRc[void] = + let + key = address.keccakHash.data + val = acc.toPayloadRef() + rc = cMpt.mpt.merge(cMpt.root, key, val) + if rc.isErr: + return rc.toVoidRcImpl(cMpt.base.parent, info) + ok() + + +proc accDelete( + cMpt: AristoChildDbRef; + address: EthAddress; + info: static[string]; + ): CoreDbRc[void] = + let + key = address.keccakHash.data + rc = cMpt.mpt.delete(cMpt.root, key) + if rc.isErr: + return rc.toVoidRcImpl(cMpt.base.parent, info) + ok() + +# ------------------------------------------------------------------------------ +# Private database methods function tables +# ------------------------------------------------------------------------------ + +proc mptMethods(cMpt: AristoChildDbRef): CoreDbMptFns = + ## Hexary trie database handlers + let db = cMpt.base.parent + CoreDbMptFns( + backendFn: proc(): CoreDbMptBackendRef = + AristoCoreDbMptBE(adb: cMpt.mpt), + + fetchFn: proc(k: openArray[byte]): CoreDbRc[Blob] = + cMpt.mptFetch(k, "fetchFn()"), + + deleteFn: proc(k: openArray[byte]): CoreDbRc[void] = + cMpt.mptDelete(k, "deleteFn()"), + + mergeFn: proc(k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] = + cMpt.mptMerge(k, v, "mergeFn()"), + + hasPathFn: proc(k: openArray[byte]): CoreDbRc[bool] = + cMpt.mpt.hasPath(cMpt.root, k).toRcImpl(db, "hasPathFn()"), + + rootVidFn: proc(): CoreDbVidRef = + var w = AristoCoreDbVid(ctx: cMpt.mpt, aVid: cMpt.root) + db.bless(w), + + isPruningFn: proc(): bool = + cMpt.prune, + + destroyFn: proc(saveMode: CoreDbSaveFlags): CoreDbRc[void] = + cMpt.base.gc() + result = cMpt.finish "destroyFn()" + cMpt.mpt = AristoDbRef(nil), # Disables `=destroy()` action + + pairsIt: iterator: (Blob,Blob) = + for (k,v) in cMpt.mpt.right LeafTie(root: cMpt.root): + yield (k.path.pathAsBlob, cMpt.mpt.serialise(v).valueOr(EmptyBlob)), + + replicateIt: iterator: (Blob,Blob) {.gcsafe, raises: [AristoApiRlpError].} = + discard) + +proc accMethods(cMpt: AristoChildDbRef): CoreDbAccFns = + ## Hexary trie database handlers + let db = cMpt.base.parent + CoreDbAccFns( + backendFn: proc(): CoreDbAccBackendRef = + db.bless(AristoCoreDbAccBE(adb: cMpt.mpt)), + + fetchFn: proc(address: EthAddress): CoreDbRc[CoreDbAccount] = + cMpt.accFetch(address, "fetchFn()"), + + deleteFn: proc(address: EthAddress): CoreDbRc[void] = + cMpt.mptDelete(address, "deleteFn()"), + + mergeFn: proc(address: EthAddress; acc: CoreDbAccount): CoreDbRc[void] = + cMpt.accMerge(address, acc, "mergeFn()"), + + hasPathFn: proc(address: EthAddress): CoreDbRc[bool] = + let key = address.keccakHash.data + cMpt.mpt.hasPath(cMpt.root, key).toRcImpl(db, "hasPathFn()"), + + rootVidFn: proc(): CoreDbVidRef = + db.bless(AristoCoreDbVid(ctx: cMpt.mpt, aVid: cMpt.root)), + + isPruningFn: proc(): bool = + cMpt.prune, + + destroyFn: proc(saveMode: CoreDbSaveFlags): CoreDbRc[void] = + cMpt.base.gc() + result = cMpt.finish "destroyFn()" + cMpt.mpt = AristoDbRef(nil)) # Disables `=destroy()` action + +# ------------------------------------------------------------------------------ +# Public handlers and helpers +# ------------------------------------------------------------------------------ + +func toError*( + e: (VertexID,AristoError); + db: CoreDbRef; + info: string; + error = Unspecified; + ): CoreDbErrorRef = + e.toErrorImpl(db, info, error) + +func toVoidRc*[T]( + rc: Result[T,AristoError]; + db: CoreDbRef; + info: string; + error = Unspecified; + ): CoreDbRc[void] = + rc.toVoidRcImpl(db, info, error) + +proc gc*(base: AristoBaseRef) = + ## Run deferred destructors when it is safe. It is needed to run the + ## destructors at the same scheduler level as the API call back functions. + ## Any of the API functions can be intercepted by the `=destroy()` hook at + ## inconvenient times so that atomicity would be violated if the actual + ## destruction took place in `=destroy()`. + ## + ## Note: In practice the `db.gq` queue should not have much more than one + ## entry and mostly be empty. + const info = "gc()" + while 0 < base.gq.len: + var q: typeof base.gq + base.gq.swap q # now `=destroy()` may refill while destructing, below + for cMpt in q: + cMpt.finish(info).isOkOr: + debug logTxt info, `error`=error.errorPrint + continue # terminates `isOkOr()` + +func mpt*(dsc: CoreDxMptRef): AristoDbRef = + dsc.AristoCoreDxMptRef.ctx.mpt + +func rootID*(dsc: CoreDxMptRef): VertexID = + dsc.AristoCoreDxMptRef.ctx.root + +# --------------------- + +func txTop*( + base: AristoBaseRef; + info: static[string]; + ): CoreDbRc[AristoTxRef] = + base.adb.txTop.toRcImpl(base.parent, info) + +func txBegin*( + base: AristoBaseRef; + info: static[string]; + ): CoreDbRc[AristoTxRef] = + base.adb.txBegin.toRcImpl(base.parent, info) + + +proc getHash*( + base: AristoBaseRef; + vid: CoreDbVidRef; + update: bool; + info: static[string]; + ): CoreDbRc[Hash256] = + let + db = base.parent + aVid = vid.to(VertexID) + + if not aVid.isValid: + return ok(EMPTY_ROOT_HASH) + + let mpt = vid.AristoCoreDbVid.ctx + if update: + ? mpt.hashify.toVoidRcImpl(db, info, HashNotAvailable) + + let key = block: + let rc = mpt.getKeyRc aVid + if rc.isErr: + doAssert rc.error in {GetKeyNotFound,GetKeyTempLocked} + return err(rc.error.toErrorImpl(db, info, HashNotAvailable)) + rc.value + + ok key.to(Hash256) + + +proc getVid*( + base: AristoBaseRef; + root: Hash256; + createOk: bool; + info: static[string]; + ): CoreDbRc[CoreDbVidRef] = + let + db = base.parent + adb = base.adb + + if root == VOID_CODE_HASH: + return ok(db.bless AristoCoreDbVid()) + + block: + base.gc() # update pending changes + let rc = adb.hashify() + ? adb.hashify.toVoidRcImpl(db, info, HashNotAvailable) + + # Check whether hash is available as state root on main trie + block: + let rc = adb.getKeyRc VertexID(1) + if rc.isErr: + doAssert rc.error == GetKeyNotFound + elif rc.value == root.to(HashKey): + return ok(db.bless AristoCoreDbVid(aVid: VertexID(1), ctx: adb)) + else: + discard + + # Check whether the `root` is avalilable in backlog + block: + # .. + discard + + # Check whether the root vertex should be created + if createOk: + return ok(db.bless AristoCoreDbVid(createOk: true, expHash: root)) + + err(aristo.GenericError.toErrorImpl(db, info, RootNotFound)) + +# ------------------------------------------------------------------------------ +# Public constructors and related +# ------------------------------------------------------------------------------ + +proc newMptHandler*( + base: AristoBaseRef; + root: CoreDbVidRef; + prune: bool; + saveMode: CoreDbSaveFlags; + info: static[string]; + ): CoreDbRc[CoreDxMptRef] = + base.gc() + + var rootID = root.to(VertexID) + if not rootID.isValid: + let rc = base.adb.getKeyRc VertexID(1) + if rc.isErr and rc.error == GetKeyNotFound: + rootID = VertexID(1) + + let + db = base.parent + + (mode, mpt) = block: + if saveMode == Companion: + (saveMode, ? base.adb.forkTop.toRcImpl(db, info)) + elif base.adb.backend.isNil: + (Cached, base.adb) + else: + (saveMode, base.adb) + + cMpt = AristoChildDbRef( + base: base, + root: rootID, + prune: prune, + mpt: mpt, + saveMode: mode) + + dsc = AristoCoreDxMptRef( + ctx: cMpt, + methods: cMpt.mptMethods) + + ok(db.bless dsc) + + +proc newAccHandler*( + base: AristoBaseRef; + prune: bool; + saveMode: CoreDbSaveFlags; + info: static[string]; + ): CoreDbRc[CoreDxAccRef] = + base.gc() + + let + db = base.parent + + (mode, mpt) = block: + if saveMode == Companion: + (saveMode, ? base.adb.forkTop.toRcImpl(db, info)) + elif base.adb.backend.isNil: + (Cached, base.adb) + else: + (saveMode, base.adb) + + cMpt = AristoChildDbRef( + base: base, + root: VertexID(1), + prune: prune, + mpt: mpt, + saveMode: mode) + + ok(db.bless CoreDxAccRef(methods: cMpt.accMethods)) + + +proc destroy*(base: AristoBaseRef; flush: bool) = + base.gc() + base.adb.finish(flush) + + +func init*(T: type AristoBaseRef; db: CoreDbRef; adb: AristoDbRef): T = + T(parent: db, adb: adb) + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/backend/aristo_db/handlers_kvt.nim b/nimbus/db/core_db/backend/aristo_db/handlers_kvt.nim new file mode 100644 index 000000000..f33d352c1 --- /dev/null +++ b/nimbus/db/core_db/backend/aristo_db/handlers_kvt.nim @@ -0,0 +1,285 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +{.push raises: [].} + +import + chronicles, + eth/common, + results, + ../../../kvt, + ../../../kvt/kvt_desc, + ../../base, + ../../base/base_desc, + ./common_desc + +type + KvtBaseRef* = ref object + parent: CoreDbRef ## Opaque top level descriptor + kdb: KvtDbRef ## Key-value table + gq: seq[KvtChildDbRef] ## Garbage queue, deferred disposal + + KvtChildDbRef = ref KvtChildDbObj + KvtChildDbObj = object + ## Sub-handle for triggering destructor when it goes out of scope + base: KvtBaseRef ## Local base descriptor + kvt: KvtDbRef ## Descriptor + saveMode: CoreDbSaveFlags ## When to store/discard + + KvtCoreDxKvtRef = ref object of CoreDxKvtRef + ## Some extendion to recover embedded state + ctx: KvtChildDbRef ## Embedded state, typical var name: `cKvt` + + AristoCoreDbKvtBE* = ref object of CoreDbKvtBackendRef + kdb*: KvtDbRef + +logScope: + topics = "kvt-hdl" + +proc gc*(base: KvtBaseRef) {.gcsafe.} + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +template logTxt(info: static[string]): static[string] = + "CoreDb/kdb " & info + +func toErrorImpl( + e: KvtError; + db: CoreDbRef; + info: string; + error = Unspecified; + ): CoreDbErrorRef = + db.bless(error, AristoCoreDbError( + ctx: info, + isAristo: false, + kErr: e)) + +func toRcImpl[T]( + rc: Result[T,KvtError]; + db: CoreDbRef; + info: string; + error = Unspecified; + ): CoreDbRc[T] = + if rc.isOk: + when T is void: + return ok() + else: + return ok(rc.value) + err rc.error.toErrorImpl(db, info, error) + +# ------------------------------------------------------------------------------ +# Private call back functions (too large for keeping inline) +# ------------------------------------------------------------------------------ + +proc finish( + cKvt: KvtChildDbRef; + info: static[string]; + ): CoreDbRc[void] = + ## key-value table destructor to be called automagically when the argument + ## wrapper gets out of scope + let + base = cKvt.base + db = base.parent + + result = ok() + + if cKvt.kvt != base.kdb: + let rc = cKvt.kvt.forget() + if rc.isErr: + result = err(rc.error.toErrorImpl(db, info)) + cKvt.kvt = KvtDbRef(nil) # Disables `=destroy()` action + + if cKvt.saveMode == AutoSave: + if base.kdb.level == 0: + let rc = base.kdb.stow() + if rc.isErr: + result = err(rc.error.toErrorImpl(db, info)) + +proc `=destroy`(cKvt: var KvtChildDbObj) = + ## Auto destructor + if not cKvt.kvt.isNil: + # Add to destructor batch queue unless direct reference + if cKvt.kvt != cKvt.base.kdb or + cKvt.saveMode == AutoSave: + cKvt.base.gq.add KvtChildDbRef( + base: cKvt.base, + kvt: cKvt.kvt, + saveMode: cKvt.saveMode) + +# ------------------------------- + +proc kvtGet( + cKvt: KvtChildDbRef; + k: openArray[byte]; + info: static[string]; + ): CoreDbRc[Blob] = + ## Member of `CoreDbKvtFns` + let rc = cKvt.kvt.get(k) + if rc.isErr: + let db = cKvt.base.parent + if rc.error == GetNotFound: + return err(rc.error.toErrorImpl(db, info, KvtNotFound)) + else: + return rc.toRcImpl(db, info) + ok(rc.value) + +proc kvtPut( + cKvt: KvtChildDbRef; + k: openArray[byte]; + v: openArray[byte]; + info: static[string]; + ): CoreDbRc[void] = + let rc = cKvt.kvt.put(k,v) + if rc.isErr: + return err(rc.error.toErrorImpl(cKvt.base.parent, info)) + ok() + +proc kvtDel( + cKvt: KvtChildDbRef; + k: openArray[byte]; + info: static[string]; + ): CoreDbRc[void] = + let rc = cKvt.kvt.del k + if rc.isErr: + return err(rc.error.toErrorImpl(cKvt.base.parent, info)) + ok() + +proc kvtHasKey( + cKvt: KvtChildDbRef; + k: openArray[byte]; + info: static[string]; + ): CoreDbRc[bool] = + cKvt.kvt.hasKey(k).toRcImpl(cKvt.base.parent, info) + +# ------------------------------------------------------------------------------ +# Private database methods function table +# ------------------------------------------------------------------------------ + +proc kvtMethods(cKvt: KvtChildDbRef): CoreDbKvtFns = + ## Key-value database table handlers + let db = cKvt.base.parent + CoreDbKvtFns( + backendFn: proc(): CoreDbKvtBackendRef = + db.bless(AristoCoreDbKvtBE(kdb: cKvt.kvt)), + + getFn: proc(k: openArray[byte]): CoreDbRc[Blob] = + cKvt.kvtGet(k, "getFn()"), + + delFn: proc(k: openArray[byte]): CoreDbRc[void] = + cKvt.kvtDel(k, "delFn()"), + + putFn: proc(k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] = + cKvt.kvtPut(k, v, "putFn()"), + + hasKeyFn: proc(k: openArray[byte]): CoreDbRc[bool] = + cKvt.kvtHasKey(k, "hasKeyFn()"), + + destroyFn: proc(saveMode: CoreDbSaveFlags): CoreDbRc[void] = + cKvt.base.gc() + cKvt.finish("destroyFn()"), + + pairsIt: iterator(): (Blob, Blob) = + discard) + +# ------------------------------------------------------------------------------ +# Public handlers and helpers +# ------------------------------------------------------------------------------ + +func toVoidRc*[T]( + rc: Result[T,KvtError]; + db: CoreDbRef; + info: string; + error = Unspecified; + ): CoreDbRc[void] = + if rc.isOk: + return ok() + err rc.error.toErrorImpl(db, info, error) + +proc gc*(base: KvtBaseRef) = + ## Run deferred destructors when it is safe. It is needed to run the + ## destructors at the same scheduler level as the API call back functions. + ## Any of the API functions can be intercepted by the `=destroy()` hook at + ## inconvenient times so that atomicity would be violated if the actual + ## destruction took place in `=destroy()`. + ## + ## Note: In practice the garbage queue should not have much more than one + ## entry and mostly be empty. + const info = "gc()" + while 0 < base.gq.len: + var q: typeof base.gq + base.gq.swap q # now `=destroy()` may refill while destructing, below + for cKvt in q: + cKvt.finish(info).isOkOr: + debug logTxt info, `error`=error.errorPrint + continue # terminates `isOkOr()` + +func kvt*(dsc: CoreDxKvtRef): KvtDbRef = + dsc.KvtCoreDxKvtRef.ctx.kvt + +# --------------------- + +func txTop*( + base: KvtBaseRef; + info: static[string]; + ): CoreDbRc[KvtTxRef] = + base.kdb.txTop.toRcImpl(base.parent, info) + +func txBegin*( + base: KvtBaseRef; + info: static[string]; + ): CoreDbRc[KvtTxRef] = + base.kdb.txBegin.toRcImpl(base.parent, info) + +# ------------------------------------------------------------------------------ +# Public constructors and related +# ------------------------------------------------------------------------------ + +proc newKvtHandler*( + base: KvtBaseRef; + saveMode: CoreDbSaveFlags; + info: static[string]; + ): CoreDbRc[CoreDxKvtRef] = + base.gc() + + let + db = base.parent + + (mode, kvt) = block: + if saveMode == Companion: + (saveMode, ? base.kdb.forkTop.toRcImpl(db, info)) + elif base.kdb.backend.isNil: + (Cached, base.kdb) + else: + (saveMode, base.kdb) + + cKvt = KvtChildDbRef( + base: base, + kvt: kvt, + saveMode: saveMode) + + dsc = KvtCoreDxKvtRef( + ctx: cKvt, + methods: cKvt.kvtMethods) + + ok(db.bless dsc) + + +proc destroy*(base: KvtBaseRef; flush: bool) = + base.gc() + base.kdb.finish(flush) + +func init*(T: type KvtBaseRef; db: CoreDbRef; kdb: KvtDbRef): T = + T(parent: db, kdb: kdb) + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/backend/aristo_rocksdb.nim b/nimbus/db/core_db/backend/aristo_rocksdb.nim new file mode 100644 index 000000000..99032baae --- /dev/null +++ b/nimbus/db/core_db/backend/aristo_rocksdb.nim @@ -0,0 +1,38 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +{.push raises: [].} + +import + eth/common, + results, + "../.."/[aristo, aristo/aristo_persistent, kvt, kvt/kvt_persistent], + ../base, + ./aristo_db + +# ------------------------------------------------------------------------------ +# Public constructor +# ------------------------------------------------------------------------------ + +proc newAristoRocksDbCoreDbRef*(path: string; qlr: QidLayoutRef): CoreDbRef = + AristoDbRocks.init( + kvt_persistent.RdbBackendRef, + aristo_persistent.RdbBackendRef, + path, qlr) + +proc newAristoRocksDbCoreDbRef*(path: string): CoreDbRef = + AristoDbRocks.init( + kvt_persistent.RdbBackendRef, + aristo_persistent.RdbBackendRef, + path) + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/base/base_desc.nim b/nimbus/db/core_db/base/base_desc.nim index 1d08df9b1..eb9fd509b 100644 --- a/nimbus/db/core_db/base/base_desc.nim +++ b/nimbus/db/core_db/base/base_desc.nim @@ -24,9 +24,12 @@ type Ooops LegacyDbMemory LegacyDbPersistent + AristoDbMemory ## Memory backend emulator + AristoDbRocks ## RocksDB backend + AristoDbVoid ## No backend const - CoreDbPersistentTypes* = {LegacyDbPersistent} + CoreDbPersistentTypes* = {LegacyDbPersistent, AristoDbRocks} type CoreDbRc*[T] = Result[T,CoreDbErrorRef] diff --git a/nimbus/db/core_db/memory_only.nim b/nimbus/db/core_db/memory_only.nim index 20878b6ee..1a07504bf 100644 --- a/nimbus/db/core_db/memory_only.nim +++ b/nimbus/db/core_db/memory_only.nim @@ -14,7 +14,7 @@ import std/options, eth/[common, trie/db], ../aristo, - ./backend/legacy_db, + ./backend/[aristo_db, legacy_db], ./base, #./core_apps_legacy as core_apps ./core_apps_newapi as core_apps @@ -135,9 +135,34 @@ proc newCoreDbRef*( when dbType == LegacyDbMemory: newLegacyMemoryCoreDbRef() + elif dbType == AristoDbMemory: + newAristoMemoryCoreDbRef() + + elif dbType == AristoDbVoid: + newAristoVoidCoreDbRef() + else: {.error: "Unsupported constructor " & $dbType & ".newCoreDbRef()".} +proc newCoreDbRef*( + dbType: static[CoreDbType]; # Database type symbol + qidLayout: QidLayoutRef; # `Aristo` only + ): CoreDbRef = + ## Constructor for volatile/memory type DB + ## + ## Note: Using legacy notation `newCoreDbRef()` rather than + ## `CoreDbRef.init()` because of compiler coughing. + ## + when dbType == AristoDbMemory: + newAristoMemoryCoreDbRef(DefaultQidLayoutRef) + + elif dbType == AristoDbVoid: + newAristoVoidCoreDbRef() + + else: + {.error: "Unsupported constructor " & $dbType & ".newCoreDbRef()" & + " with qidLayout argument".} + # ------------------------------------------------------------------------------ # Public template wrappers # ------------------------------------------------------------------------------ diff --git a/nimbus/db/core_db/persistent.nim b/nimbus/db/core_db/persistent.nim index 071359445..1a908aa2c 100644 --- a/nimbus/db/core_db/persistent.nim +++ b/nimbus/db/core_db/persistent.nim @@ -17,7 +17,7 @@ import ../aristo, - ./backend/[legacy_rocksdb], + ./backend/[aristo_rocksdb, legacy_rocksdb], ./memory_only export @@ -35,7 +35,26 @@ proc newCoreDbRef*( when dbType == LegacyDbPersistent: newLegacyPersistentCoreDbRef path + elif dbType == AristoDbRocks: + newAristoRocksDbCoreDbRef path + else: {.error: "Unsupported dbType for persistent newCoreDbRef()".} +proc newCoreDbRef*( + dbType: static[CoreDbType]; # Database type symbol + path: string; # Storage path for database + qidLayout: QidLayoutRef; # Optional for `Aristo`, ignored by others + ): CoreDbRef = + ## Constructor for persistent type DB + ## + ## Note: Using legacy notation `newCoreDbRef()` rather than + ## `CoreDbRef.init()` because of compiler coughing. + when dbType == AristoDbRocks: + newAristoRocksDbCoreDbRef(path, qlr) + + else: + {.error: "Unsupported dbType for persistent newCoreDbRef()" & + " with qidLayout argument".} + # End