status-im/nimbus-eth1
2
0
Fork 0
mirror of https://github.com/status-im/nimbus-eth1.git synced 2025-02-02 07:17:19 +00:00
Code Issues Projects Releases Wiki Activity

Core db update api and fix tracer methods (#1816)

Browse Source 
* CoreDB: Re-org API

details:
  Legacy API internally uses vertex ID for root node abstraction

* Cosmetics: Move some unit test helpers to common sub-directory

* Extract constant from `accouns_cache.nim` => `constants.nim`

* Fix tracer methods

why:
  Logger dump data were wrongly dumped from the production database. This
  caused an assert exception when iterating over the persistent database
  (instead of the memory logger.) This event in turn was enabled after
  fixing another inconsistency which just set up an empty iterator. Unit
  tests failed to detect that.
This commit is contained in:
Jordan Hrycaj 2023-10-11 20:09:11 +01:00 committed by GitHub
parent 3444ffaf30
commit 786263c0b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 681 additions and 412 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
nimbus/constants.nim
View File

@ -92,4 +92,9 @@ const
# SystemAddress is where the system-transaction is sent from as per EIP-4788
SystemAddress* = hexToByteArray[20]("0xfffffffffffffffffffffffffffffffffffffffe")
RIPEMD_ADDR* = block:
proc initAddress(x: int): EthAddress {.compileTime.} =
result[19] = x.byte
initAddress(3)
# End

9
nimbus/db/accounts_cache.nim
View File

@ -81,11 +81,6 @@ const
NewlyCreated
}
ripemdAddr* = block:
proc initAddress(x: int): EthAddress {.compileTime.} =
result[19] = x.byte
initAddress(3)
when debugAccountsCache:
import
stew/byteutils
@ -105,7 +100,7 @@ proc beginSavepoint*(ac: var AccountsCache): SavePoint {.gcsafe.}
proc rawTrie*(ac: AccountsCache): AccountsTrie = ac.trie
func db(ac: AccountsCache): CoreDbRef = ac.trie.db
func kvt(ac: AccountsCache): CoreDbKvtRef = ac.db.kvt
proc kvt(ac: AccountsCache): CoreDbKvtRef = ac.db.kvt
# The AccountsCache is modeled after TrieDatabase for it's transaction style
proc init*(x: typedesc[AccountsCache], db: CoreDbRef,
@ -560,7 +555,7 @@ proc clearEmptyAccounts(ac: AccountsCache) =
# https://github.com/ethereum/EIPs/issues/716
if ac.ripemdSpecial:
ac.deleteEmptyAccount(ripemdAddr)
ac.deleteEmptyAccount(RIPEMD_ADDR)
ac.ripemdSpecial = false
proc persist*(ac: AccountsCache,

288
nimbus/db/core_db/legacy_db.nim → nimbus/db/core_db/backend/legacy_db.nim
View File

@ -14,24 +14,40 @@ import
std/options,
eth/[common, rlp, trie/db, trie/hexary],
results,
../../errors,
"."/[base, base/base_desc]
../../../errors,
".."/[base, base/base_desc]
type
LegacyApiRlpError* = object of CoreDbApiError
## For re-routing exceptions in iterator closure
LegacyDbRef* = ref object of CoreDbRef
tdb: TrieDatabaseRef # copy of descriptor reference captured with closures
# -----------
HexaryTrieRef = ref object
trie: HexaryTrie # needed for descriptor capturing with closures
LegacyDbRef* = ref object of CoreDbRef
kvt: CoreDxKvtRef ## Cache, no need to rebuild methods descriptor
tdb: TrieDatabaseRef ## Copy of descriptor reference captured with closures
LegacyDbClose* = proc() {.gcsafe, raises: [].}
## Custom destructor
HexaryChildDbRef = ref object
trie: HexaryTrie ## needed for descriptor capturing with closures
RecorderRef = ref object of RootRef
flags: set[CoreDbCaptFlags]
parent: TrieDatabaseRef
recorder: TrieDatabaseRef
appDb: CoreDbRef
logger: LegacyDbRef
appDb: LegacyDbRef
LegacyCoreDbVid* = ref object of CoreDbVidRef
vHash: Hash256 ## Hash key
LegacyCoreDbError = ref object of CoreDbErrorRef
ctx: string ## Context where the exception or error occured
name: string ## name of exception
msg: string ## Exception info
# ------------
LegacyCoreDbBE = ref object of CoreDbBackendRef
base: LegacyDbRef
@ -42,15 +58,14 @@ type
LegacyCoreDbMptBE = ref object of CoreDbMptBackendRef
mpt: HexaryTrie
LegacyCoreDbError = ref object of CoreDbErrorRef
ctx: string ## Context where the exception or error occured
name: string ## name of exception
msg: string ## Exception info
LegacyCoreDbAccBE = ref object of CoreDbAccBackendRef
mpt: HexaryTrie
proc init*(
db: LegacyDbRef;
dbType: CoreDbType;
tdb: TrieDatabaseRef;
closeDb = LegacyDbClose(nil);
): CoreDbRef
{.gcsafe.}
@ -62,11 +77,11 @@ template mapRlpException(db: LegacyDbRef; info: static[string]; code: untyped) =
try:
code
except RlpError as e:
var w = LegacyCoreDbError(
ctx: info,
name: $e.name,
msg: e.msg)
return err(db.bless w)
return err(db.bless LegacyCoreDbError(
error: RlpException,
ctx: info,
name: $e.name,
msg: e.msg))
template reraiseRlpException(info: static[string]; code: untyped) =
try:
@ -75,92 +90,91 @@ template reraiseRlpException(info: static[string]; code: untyped) =
let msg = info & ", name=\"" & $e.name & "\", msg=\"" & e.msg & "\""
raise (ref LegacyApiRlpError)(msg: msg)
# ------------------------------------------------------------------------------
# Private helpers, other functions
# ------------------------------------------------------------------------------
proc errorPrint(e: CoreDbErrorRef): string =
if not e.isNil:
let e = e.LegacyCoreDbError
result &= "ctx=\"" & $e.ctx & "\""
if e.name != "":
result &= ", name=\"" & $e.name & "\""
if e.msg != "":
result &= ", msg=\"" & $e.msg & "\""
func lvHash(vid: CoreDbVidRef): Hash256 =
if not vid.isNil and vid.ready:
return vid.LegacyCoreDbVid.vHash
EMPTY_ROOT_HASH
proc toCoreDbAccount(
data: Blob;
db: LegacyDbRef;
): CoreDbAccount
{.gcsafe, raises: [RlpError].} =
let acc = rlp.decode(data, Account)
CoreDbAccount(
nonce: acc.nonce,
balance: acc.balance,
codeHash: acc.codeHash,
storageVid: db.bless LegacyCoreDbVid(vHash: acc.storageRoot))
proc toAccount(
account: CoreDbAccount
): Account =
## Fast rewrite of `recast()` from base which reures to `vidHashFn()`
Account(
nonce: account.nonce,
balance: account.balance,
codeHash: account.codeHash,
storageRoot: account.storageVid.lvHash)
# ------------------------------------------------------------------------------
# Private mixin methods for `trieDB` (backport from capturedb/tracer sources)
# ------------------------------------------------------------------------------
proc get(db: RecorderRef, key: openArray[byte]): Blob =
## Mixin for `trieDB()`
result = db.recorder.get(key)
result = db.logger.tdb.get(key)
if result.len == 0:
result = db.parent.get(key)
if result.len != 0:
db.recorder.put(key, result)
db.logger.tdb.put(key, result)
proc put(db: RecorderRef, key, value: openArray[byte]) =
## Mixin for `trieDB()`
db.recorder.put(key, value)
db.logger.tdb.put(key, value)
if PersistPut in db.flags:
db.parent.put(key, value)
proc contains(db: RecorderRef, key: openArray[byte]): bool =
## Mixin for `trieDB()`
result = db.parent.contains(key)
doAssert(db.recorder.contains(key) == result)
doAssert(db.logger.tdb.contains(key) == result)
proc del(db: RecorderRef, key: openArray[byte]) =
## Mixin for `trieDB()`
db.recorder.del(key)
db.logger.tdb.del(key)
if PersistDel in db.flags:
db.parent.del(key)
proc newRecorderRef(
tdb: TrieDatabaseRef;
dbType: CoreDbType,
flags: set[CoreDbCaptFlags];
): RecorderRef =
## Capture constuctor, uses `mixin` values from above
result = RecorderRef(
flags: flags,
parent: tdb,
recorder: newMemoryDB())
result.appDb = LegacyDbRef().init(LegacyDbMemory, trieDB result)
logger: LegacyDbRef().init(LegacyDbMemory, newMemoryDB()).LegacyDbRef)
result.appDb = LegacyDbRef().init(dbType, trieDB result).LegacyDbRef
# ------------------------------------------------------------------------------
# Private database method function tables
# ------------------------------------------------------------------------------
proc miscMethods(db: LegacyDbRef): CoreDbMiscFns =
CoreDbMiscFns(
backendFn: proc(): CoreDbBackendRef =
db.bless(LegacyCoreDbBE(base: db)),
errorPrintFn: proc(e: CoreDbErrorRef): string =
if not e.isNil:
let e = e.LegacyCoreDbError
result &= "ctx=\"" & $e.ctx & "\"" & " , "
result &= "name=\"" & $e.name & "\"" & ", "
result &= "msg=\"" & $e.msg & "\""
discard,
legacySetupFn: proc() =
db.tdb.put(EMPTY_ROOT_HASH.data, @[0x80u8]))
proc kvtHandlers(db: LegacyDbRef): CoreDxKvtRef =
## Key-value database table handlers
let tdb = db.tdb
CoreDxKvtRef(
methods: CoreDbKvtFns(
backendFn: proc(): CoreDbKvtBackendRef =
db.bless(LegacyCoreDbKvtBE(tdb: tdb)),
getFn: proc(k: openArray[byte]): CoreDbRc[Blob] =
ok(tdb.get(k)),
delFn: proc(k: openArray[byte]): CoreDbRc[void] =
tdb.del(k)
ok(),
putFn: proc(k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] =
tdb.put(k,v)
ok(),
containsFn: proc(k: openArray[byte]): CoreDbRc[bool] =
ok(tdb.contains(k)),
pairsIt: iterator(): (Blob, Blob) =
for k,v in tdb.pairsInMemoryDB:
yield (k,v)))
proc kvtMethods(db: LegacyDbRef): CoreDbKvtFns =
## Key-value database table handlers
let tdb = db.tdb
@ -186,23 +200,23 @@ proc kvtMethods(db: LegacyDbRef): CoreDbKvtFns =
for k,v in tdb.pairsInMemoryDB:
yield (k,v))
proc mptMethods(mpt: HexaryTrieRef; db: LegacyDbRef): CoreDbMptFns =
proc mptMethods(mpt: HexaryChildDbRef; db: LegacyDbRef): CoreDbMptFns =
## Hexary trie database handlers
CoreDbMptFns(
backendFn: proc(): CoreDbMptBackendRef =
db.bless(LegacyCoreDbMptBE(mpt: mpt.trie)),
getFn: proc(k: openArray[byte]): CoreDbRc[Blob] =
fetchFn: proc(k: openArray[byte]): CoreDbRc[Blob] =
db.mapRlpException("legacy/mpt/get()"):
return ok(mpt.trie.get(k))
discard,
delFn: proc(k: openArray[byte]): CoreDbRc[void] =
deleteFn: proc(k: openArray[byte]): CoreDbRc[void] =
db.mapRlpException("legacy/mpt/del()"):
mpt.trie.del(k)
ok(),
putFn: proc(k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] =
mergeFn: proc(k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] =
db.mapRlpException("legacy/mpt/put()"):
mpt.trie.put(k,v)
ok(),
@ -212,24 +226,57 @@ proc mptMethods(mpt: HexaryTrieRef; db: LegacyDbRef): CoreDbMptFns =
return ok(mpt.trie.contains(k))
discard,
rootHashFn: proc(): CoreDbRc[Hash256] =
ok(mpt.trie.rootHash),
rootVidFn: proc(): CoreDbVidRef =
db.bless(LegacyCoreDbVid(vHash: mpt.trie.rootHash)),
isPruningFn: proc(): bool =
mpt.trie.isPruning,
pairsIt: iterator(): (Blob, Blob) {.gcsafe, raises: [CoreDbApiError].} =
pairsIt: iterator: (Blob,Blob) {.gcsafe, raises: [LegacyApiRlpError].} =
reraiseRlpException("legacy/mpt/pairs()"):
for k,v in mpt.trie.pairs():
yield (k,v)
discard,
replicateIt: iterator(): (Blob, Blob) {.gcsafe, raises: [CoreDbApiError].} =
replicateIt: iterator: (Blob,Blob) {.gcsafe, raises: [LegacyApiRlpError].} =
reraiseRlpException("legacy/mpt/replicate()"):
for k,v in mpt.trie.replicate():
yield (k,v)
discard)
proc accMethods(mpt: HexaryChildDbRef; db: LegacyDbRef): CoreDbAccFns =
## Hexary trie database handlers
CoreDbAccFns(
backendFn: proc(): CoreDbAccBackendRef =
db.bless(LegacyCoreDbAccBE(mpt: mpt.trie)),
fetchFn: proc(k: EthAddress): CoreDbRc[CoreDbAccount] =
const info = "legacy/mpt/getAccount()"
db.mapRlpException info:
return ok mpt.trie.get(k.keccakHash.data).toCoreDbAccount(db)
return err(db.bless LegacyCoreDbError(error: MptNotFound, ctx: info)),
deleteFn: proc(k: EthAddress): CoreDbRc[void] =
db.mapRlpException("legacy/mpt/del()"):
mpt.trie.del(k.keccakHash.data)
ok(),
mergeFn: proc(k: EthAddress; v: CoreDbAccount): CoreDbRc[void] =
db.mapRlpException("legacy/mpt/put()"):
mpt.trie.put(k.keccakHash.data, rlp.encode v.toAccount)
ok(),
containsFn: proc(k: EthAddress): CoreDbRc[bool] =
db.mapRlpException("legacy/mpt/put()"):
return ok(mpt.trie.contains k.keccakHash.data)
discard,
rootVidFn: proc(): CoreDbVidRef =
db.bless(LegacyCoreDbVid(vHash: mpt.trie.rootHash)),
isPruningFn: proc(): bool =
mpt.trie.isPruning)
proc txMethods(tx: DbTransaction): CoreDbTxFns =
CoreDbTxFns(
commitFn: proc(applyDeletes: bool): CoreDbRc[void] =
@ -259,37 +306,72 @@ proc cptMethods(cpt: RecorderRef): CoreDbCaptFns =
recorderFn: proc(): CoreDbRc[CoreDbRef] =
ok(cpt.appDb),
logDbFn: proc(): CoreDbRc[CoreDbRef] =
ok(cpt.logger),
getFlagsFn: proc(): set[CoreDbCaptFlags] =
cpt.flags)
# ------------------------------------------------------------------------------
# Private constructor functions table
# Private base methods (including constructors)
# ------------------------------------------------------------------------------
proc constructors(db: LegacyDbRef): CoreDbConstructorFns =
proc baseMethods(
db: LegacyDbRef;
dbType: CoreDbType;
closeDb: LegacyDbClose;
): CoreDbBaseFns =
let tdb = db.tdb
CoreDbConstructorFns(
mptFn: proc(root: Hash256): CoreDbRc[CoreDxMptRef] =
let mpt = HexaryTrieRef(trie: initHexaryTrie(tdb, root, false))
var dsc = CoreDxMptRef(methods: mpt.mptMethods(db))
ok(db.bless dsc),
CoreDbBaseFns(
backendFn: proc(): CoreDbBackendRef =
db.bless(LegacyCoreDbBE(base: db)),
legacyMptFn: proc(root: Hash256; prune: bool): CoreDbRc[CoreDxMptRef] =
let mpt = HexaryTrieRef(trie: initHexaryTrie(tdb, root, prune))
var dsc = CoreDxMptRef(methods: mpt.mptMethods(db))
ok(db.bless dsc),
destroyFn: proc(ignore: bool) =
if not closeDb.isNil:
closeDb()
discard,
vidHashFn: proc(vid: CoreDbVidRef): Result[Hash256,void] =
ok(vid.lvHash),
errorPrintFn: proc(e: CoreDbErrorRef): string =
e.errorPrint(),
legacySetupFn: proc() =
db.tdb.put(EMPTY_ROOT_HASH.data, @[0x80u8]),
getRootFn: proc(root: Hash256; createOk: bool): CoreDbRc[CoreDbVidRef] =
if root == EMPTY_CODE_HASH:
return ok(db.bless LegacyCoreDbVid(vHash: EMPTY_CODE_HASH))
# Due to the way it is used for creating a ne root node, `createOk` must
# be checked before `contains()` is run. Otherwise it might bail out in
# the assertion of the above trace/recorder mixin `contains()` function.
if createOk or tdb.contains(root.data):
return ok(db.bless LegacyCoreDbVid(vHash: root))
err(db.bless LegacyCoreDbError(error: RootNotFound, ctx: "getRoot()")),
newKvtFn: proc(): CoreDxKvtRef =
db.kvt,
newMptFn: proc(root: CoreDbVidRef, prune: bool): CoreDbRc[CoreDxMptRef] =
let mpt = HexaryChildDbRef(trie: initHexaryTrie(tdb, root.lvHash, prune))
ok(db.bless CoreDxMptRef(methods: mpt.mptMethods db)),
newAccFn: proc(root: CoreDbVidRef, prune: bool): CoreDbRc[CoreDxAccRef] =
let mpt = HexaryChildDbRef(trie: initHexaryTrie(tdb, root.lvHash, prune))
ok(db.bless CoreDxAccRef(methods: mpt.accMethods db)),
getIdFn: proc(): CoreDbRc[CoreDxTxID] =
var dsc = CoreDxTxID(methods: tdb.getTransactionID.tidMethods(tdb))
ok(db.bless dsc),
ok(db.bless CoreDxTxID(methods: tdb.getTransactionID.tidMethods(tdb))),
beginFn: proc(): CoreDbRc[CoreDxTxRef] =
var dsc = CoreDxTxRef(methods: tdb.beginTransaction.txMethods)
ok(db.bless dsc),
ok(db.bless CoreDxTxRef(methods: tdb.beginTransaction.txMethods)),
captureFn: proc(flags: set[CoreDbCaptFlags]): CoreDbRc[CoreDxCaptRef] =
var dsc = CoreDxCaptRef(methods: newRecorderRef(tdb, flags).cptMethods)
ok(db.bless dsc))
captureFn: proc(flgs: set[CoreDbCaptFlags]): CoreDbRc[CoreDxCaptRef] =
let fns = newRecorderRef(tdb, dbtype, flgs).cptMethods
ok(db.bless CoreDxCaptRef(methods: fns)))
# ------------------------------------------------------------------------------
# Public constructor helpers
@ -299,14 +381,18 @@ proc init*(
db: LegacyDbRef;
dbType: CoreDbType;
tdb: TrieDatabaseRef;
closeDb = LegacyDbClose(nil);
): CoreDbRef =
## Constructor helper
# Local extensions
db.tdb = tdb
db.init(
dbType = dbType,
dbMethods = db.miscMethods,
kvtMethods = db.kvtMethods,
newSubMod = db.constructors)
db
db.kvt = db.bless CoreDxKvtRef(methods: db.kvtMethods())
# Base descriptor
db.dbType = dbType
db.methods = db.baseMethods(dbType, closeDb)
db.bless
# ------------------------------------------------------------------------------
# Public constructor and low level data retrieval, storage & transation frame
@ -336,7 +422,11 @@ func toLegacy*(be: CoreDbKvtBackendRef): TrieDatabaseRef =
func toLegacy*(be: CoreDbMptBackendRef): HexaryTrie =
if be.parent.isLegacy:
return be.LegacyCoreDbMptBE.mpt
func toLegacy*(be: CoreDbAccBackendRef): HexaryTrie =
if be.parent.isLegacy:
return be.LegacyCoreDbAccBE.mpt
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

15
nimbus/db/core_db/legacy_rocksdb.nim → nimbus/db/core_db/backend/legacy_rocksdb.nim
View File

@ -12,13 +12,18 @@
import
eth/trie/db,
../select_backend,
"."/[base, legacy_db]
rocksdb,
"../.."/select_backend,
../base,
./legacy_db
type
LegaPersDbRef = ref object of LegacyDbRef
rdb: RocksStoreRef # for backend access with legacy mode
# No other backend supported
doAssert nimbus_db_backend == "rocksdb"
# ------------------------------------------------------------------------------
# Public constructor and low level data retrieval, storage & transation frame
# ------------------------------------------------------------------------------
@ -41,7 +46,11 @@ proc newLegacyPersistentCoreDbRef*(path: string): CoreDbRef =
except CatchableError as e:
let msg = "DB initialisation error(" & $e.name & "): " & e.msg
raise (ref ResultDefect)(msg: msg)
LegaPersDbRef(rdb: backend.rdb).init(LegacyDbPersistent, backend.trieDB)
proc done() =
backend.rdb.store.close()
LegaPersDbRef(rdb: backend.rdb).init(LegacyDbPersistent, backend.trieDB, done)
# ------------------------------------------------------------------------------
# Public helper for direct backend access

345
nimbus/db/core_db/base.nim
View File

@ -19,13 +19,19 @@ import
./base/[base_desc, validate]
export
CoreDbAccount,
CoreDbApiError,
CoreDbBackendRef,
CoreDbCaptFlags,
CoreDbErrorCode,
CoreDbErrorRef,
CoreDbAccBackendRef,
CoreDbKvtBackendRef,
CoreDbMptBackendRef,
CoreDbRef,
CoreDbType,
CoreDbVidRef,
CoreDxAccRef,
CoreDxCaptRef,
CoreDxKvtRef,
CoreDxMptRef,
@ -52,7 +58,7 @@ const
when ProvideCoreDbLegacyAPI:
type
TxWrapperApiError* = object of CoreDbApiError
## For re-routing exceptions in iterator closure
## For re-routing exception on tx/action template
CoreDbKvtRef* = distinct CoreDxKvtRef ## Let methods defect on error
CoreDbMptRef* = distinct CoreDxMptRef ## ...
@ -61,22 +67,27 @@ when ProvideCoreDbLegacyAPI:
CoreDbTxID* = distinct CoreDxTxID
CoreDbCaptRef* = distinct CoreDxCaptRef
CoreDbTrieRef* = CoreDbMptRef | CoreDbPhkRef
CoreDbTrieRefs* = CoreDbMptRef | CoreDbPhkRef
## Shortcut, *MPT* modules for (legacy API)
CoreDbChldRef* = CoreDbKvtRef | CoreDbTrieRef | CoreDbTxRef | CoreDbTxID |
CoreDbCaptRef
## Shortcut, all modules with a `parent` (for legacy API)
CoreDbChldRefs* = CoreDbKvtRef | CoreDbTrieRefs | CoreDbTxRef | CoreDbTxID |
CoreDbCaptRef
## Shortcut, all modules with a `parent` entry (for legacy API)
type
CoreDxTrieRef* = CoreDxMptRef | CoreDxPhkRef
## Shortcut, *MPT* modules
CoreDxTrieRefs = CoreDxMptRef | CoreDxPhkRef | CoreDxAccRef
## Shortcut, *MPT* descriptors
CoreDxChldRef* = CoreDxKvtRef | CoreDxTrieRef | CoreDxTxRef | CoreDxTxID |
CoreDxCaptRef |
CoreDbErrorRef |
CoreDbBackendRef | CoreDbKvtBackendRef | CoreDbMptBackendRef
## Shortcut, all modules with a `parent`
CoreDxTrieRelated = CoreDxTrieRefs | CoreDxTxRef | CoreDxTxID | CoreDxCaptRef
## Shortcut, descriptors for sub-modules running on an *MPT*
CoreDbBackends = CoreDbBackendRef | CoreDbKvtBackendRef |
CoreDbMptBackendRef | CoreDbAccBackendRef
## Shortcut, all backend descriptors.
CoreDxChldRefs = CoreDxKvtRef | CoreDxTrieRelated | CoreDbBackends |
CoreDbErrorRef
## Shortcut, all descriptors with a `parent` entry.
# ------------------------------------------------------------------------------
# Private functions: helpers
@ -96,17 +107,17 @@ func toCoreDxPhkRef(mpt: CoreDxMptRef): CoreDxPhkRef =
fromMpt: mpt,
methods: mpt.methods)
result.methods.getFn =
result.methods.fetchFn =
proc(k: openArray[byte]): CoreDbRc[Blob] =
mpt.methods.getFn(k.keccakHash.data)
mpt.methods.fetchFn(k.keccakHash.data)
result.methods.delFn =
result.methods.deleteFn =
proc(k: openArray[byte]): CoreDbRc[void] =
mpt.methods.delFn(k.keccakHash.data)
mpt.methods.deleteFn(k.keccakHash.data)
result.methods.putFn =
proc(k:openArray[byte]; v:openArray[byte]): CoreDbRc[void] =
mpt.methods.putFn(k.keccakHash.data, v)
result.methods.mergeFn =
proc(k:openArray[byte]; v: openArray[byte]): CoreDbRc[void] =
mpt.methods.mergeFn(k.keccakHash.data, v)
result.methods.containsFn =
proc(k: openArray[byte]): CoreDbRc[bool] =
@ -128,49 +139,47 @@ func parent(phk: CoreDxPhkRef): CoreDbRef =
phk.fromMpt.parent
# ------------------------------------------------------------------------------
# Public constructor
# Public constructor helper
# ------------------------------------------------------------------------------
template bless*(db: CoreDbRef; child: untyped): auto =
## Complete sub-module descriptor, fill in `parent`
proc bless*(db: CoreDbRef): CoreDbRef =
## Verify descriptor
when AutoValidateDescriptors:
db.validate
db
proc bless*(db: CoreDbRef; child: CoreDbVidRef): CoreDbVidRef =
## Complete sub-module descriptor, fill in `parent` and actvate it.
child.parent = db
child.ready = true
when AutoValidateDescriptors:
child.validate
child
proc init*(
db: CoreDbRef; # Main descriptor, locally extended
dbType: CoreDbType; # Backend symbol
dbMethods: CoreDbMiscFns; # General methods
kvtMethods: CoreDbKvtFns; # Kvt related methods
newSubMod: CoreDbConstructorFns; # Sub-module constructors
) =
## Base descriptor initaliser
db.dbType = dbType
db.methods = dbMethods
db.new = newSubMod
db.kvtRef = CoreDxKvtRef(
parent: db,
methods: kvtMethods)
proc bless*(db: CoreDbRef; child: CoreDxKvtRef): CoreDxKvtRef =
## Complete sub-module descriptor, fill in `parent` and de-actvate
## iterator for persistent database.
child.parent = db
# Disable interator for non-memory instances
if dbType in CoreDbPersistentTypes:
db.kvtRef.methods.pairsIt = iterator(): (Blob, Blob) =
if db.dbType in CoreDbPersistentTypes:
child.methods.pairsIt = iterator(): (Blob, Blob) =
db.itNotImplemented "pairs/kvt"
when AutoValidateDescriptors:
db.validate
child.validate
child
proc newCoreDbCaptRef*(db: CoreDbRef; methods: CoreDbCaptFns): CoreDxCaptRef =
## Capture constructor helper.
result = CoreDxCaptRef(
parent: db,
methods: methods)
proc bless*[T: CoreDxTrieRelated | CoreDbErrorRef | CoreDbBackends](
db: CoreDbRef;
child: T;
): auto =
## Complete sub-module descriptor, fill in `parent`.
child.parent = db
when AutoValidateDescriptors:
db.validate
child.validate
child
# ------------------------------------------------------------------------------
# Public main descriptor methods
@ -180,32 +189,87 @@ proc dbType*(db: CoreDbRef): CoreDbType =
## Getter
db.dbType
# On the persistent legacy hexary trie, this function is needed for
# bootstrapping and Genesis setup when the `purge` flag is activated.
proc compensateLegacySetup*(db: CoreDbRef) =
## On the persistent legacy hexary trie, this function is needed for
## bootstrapping and Genesis setup when the `purge` flag is activated.
## Otherwise the database backend may defect on an internal inconsistency.
db.methods.legacySetupFn()
func parent*(cld: CoreDxChldRef): CoreDbRef =
func parent*(cld: CoreDxChldRefs): CoreDbRef =
## Getter, common method for all sub-modules
cld.parent
proc backend*(db: CoreDbRef): CoreDbBackendRef =
## Getter, retrieves the *raw* backend object for special support.
result = db.methods.backendFn()
result.parent = db
proc backend*(dsc: CoreDxKvtRef | CoreDxTrieRelated | CoreDbRef): auto =
## Getter, retrieves the *raw* backend object for special/localised support.
dsc.methods.backendFn()
proc finish*(db: CoreDbRef; flush = false) =
## Database destructor. If the argument `flush` is set `false`, the database
## is left as-is and only the in-memory handlers are cleaned up.
##
## Otherwise the destructor is allowed to remove the database. This feature
## depends on the backend database. Currently, only the `AristoDbRocks` type
## backend removes the database on `true`.
db.methods.destroyFn flush
proc `$$`*(e: CoreDbErrorRef): string =
## Pretty print error symbol, note that this directive may have side effects
## as it calls a backend function.
e.parent.methods.errorPrintFn(e)
proc hash*(vid: CoreDbVidRef): Result[Hash256,void] =
## Getter (well, sort of), retrieves the hash for a `vid` argument. The
## function might fail if there is currently no hash available (e.g. on
## `Aristo`.) Note that this is different from succeeding with an
## `EMPTY_ROOT_HASH` value.
##
## The value `EMPTY_ROOT_HASH` is also returned on an empty `vid` argument
## `CoreDbVidRef(nil)`, say.
##
if not vid.isNil and vid.ready:
return vid.parent.methods.vidHashFn vid
ok EMPTY_ROOT_HASH
proc recast*(account: CoreDbAccount): Result[Account,void] =
## Convert the argument `account` to the portable Ethereum representation
## of an account. This conversion may fail if the storage root hash (see
## `hash()` above) is currently unavailable.
##
## Note that for the legacy backend, this function always succeeds.
##
ok Account(
nonce: account.nonce,
balance: account.balance,
codeHash: account.codeHash,
storageRoot: ? account.storageVid.hash)
proc getRoot*(
db: CoreDbRef;
root: Hash256;
createOk = false;
): CoreDbRc[CoreDbVidRef] =
## Find root node with argument hash `root` in database and return the
## corresponding `CoreDbVidRef` object. If the `root` arguent is set
## `EMPTY_CODE_HASH`, this function always succeeds, otherwise it fails
## unless a root node with the corresponding hash exists.
##
## This function is intended to open a virtual accounts trie database as in:
## ::
## proc openAccountLedger(db: CoreDbRef, rootHash: Hash256): CoreDxMptRef =
## let root = db.getRoot(rootHash).isOkOr:
## # some error handling
## return
## db.newAccMpt root
##
db.methods.getRootFn(root, createOk)
# ------------------------------------------------------------------------------
# Public key-value table methods
# ------------------------------------------------------------------------------
func toKvt*(db: CoreDbRef): CoreDxKvtRef =
proc newKvt*(db: CoreDbRef): CoreDxKvtRef =
## Getter (pseudo constructor)
db.kvtRef
db.methods.newKvtFn()
proc get*(kvt: CoreDxKvtRef; key: openArray[byte]): CoreDbRc[Blob] =
kvt.methods.getFn key
@ -228,74 +292,67 @@ iterator pairs*(kvt: CoreDxKvtRef): (Blob, Blob) {.apiRaise.} =
for k,v in kvt.methods.pairsIt():
yield (k,v)
proc backend*(kvt: CoreDxKvtRef): CoreDbKvtBackendRef =
## Getter, retrieves the *raw* backend object for special support.
result = kvt.methods.backendFn()
result.parent = kvt.parent
# ------------------------------------------------------------------------------
# Public Merkle Patricia Tree, hexary trie constructors
# ------------------------------------------------------------------------------
proc newMpt*(db: CoreDbRef; root=EMPTY_ROOT_HASH): CoreDbRc[CoreDxMptRef] =
## Constructor
db.new.mptFn root
proc newMpt*(db: CoreDbRef; root: CoreDbVidRef; prune = true): CoreDxMptRef =
## Constructor, will defect on failure (note that the legacy backend
## always succeeds)
db.methods.newMptFn(root, prune).valueOr: raiseAssert $$error
proc newAccMpt*(db: CoreDbRef; root: CoreDbVidRef; prune = true): CoreDxAccRef =
## Similar to `newMpt()` for handling accounts. Although this sub-trie can
## be emulated by means of `newMpt(..).toPhk()`, it is recommended using
## this constructor which implies its own subset of methods to handle that
## trie.
db.methods.newAccFn(root, prune).valueOr: raiseAssert $$error
proc toMpt*(phk: CoreDxPhkRef): CoreDxMptRef =
## Replaces the pre-hashed argument trie `phk` by the non pre-hashed *MPT*.
## Note that this does not apply to an accounts trie that was created by
## `newAccMpt()`.
phk.fromMpt
proc toPhk*(mpt: CoreDxMptRef): CoreDxPhkRef =
## Replaces argument `mpt` by a pre-hashed *MPT*.
## Note that this does not apply to an accounts trie that was created by
## `newAaccMpt()`.
mpt.toCoreDxPhkRef
# ------------------------------------------------------------------------------
# Public hexary trie legacy constructors
# Public common methods for all hexary trie databases (`mpt`, `phk`, or `acc`)
# ------------------------------------------------------------------------------
proc newMptPrune*(
db: CoreDbRef;
root = EMPTY_ROOT_HASH;
prune = true;
): CoreDbRc[CoreDxMptRef] =
## Constructor, `HexaryTrie` compliant
db.new.legacyMptFn(root, prune)
proc newPhkPrune*(
db: CoreDbRef;
root = EMPTY_ROOT_HASH;
prune = true;
): CoreDbRc[CoreDxPhkRef] =
## Constructor, `SecureHexaryTrie` compliant
ok (? db.new.legacyMptFn(root, prune)).toCoreDxPhkRef
# ------------------------------------------------------------------------------
# Public hexary trie database methods (`mpt` or `phk`)
# ------------------------------------------------------------------------------
proc isPruning*(trie: CoreDxTrieRef): bool =
proc isPruning*(dsc: CoreDxTrieRefs | CoreDxAccRef): bool =
## Getter
trie.methods.isPruningFn()
dsc.methods.isPruningFn()
proc get*(trie: CoreDxTrieRef; key: openArray[byte]): CoreDbRc[Blob] =
trie.methods.getFn(key)
proc rootVid*(dsc: CoreDxTrieRefs | CoreDxAccRef): CoreDbVidRef =
## Getter, result is not `nil`
dsc.methods.rootVidFn()
proc del*(trie: CoreDxTrieRef; key: openArray[byte]): CoreDbRc[void] =
trie.methods.delFn key
# ------------------------------------------------------------------------------
# Public generic hexary trie database methods (`mpt` or `phk`)
# ------------------------------------------------------------------------------
proc put*(
trie: CoreDxTrieRef;
proc fetch*(trie: CoreDxTrieRefs; key: openArray[byte]): CoreDbRc[Blob] =
## Fetch data from the argument `trie`
trie.methods.fetchFn(key)
proc delete*(trie: CoreDxTrieRefs; key: openArray[byte]): CoreDbRc[void] =
trie.methods.deleteFn key
proc merge*(
trie: CoreDxTrieRefs;
key: openArray[byte];
value: openArray[byte];
): CoreDbRc[void] =
trie.methods.putFn(key, value)
trie.methods.mergeFn(key, value)
proc contains*(trie: CoreDxTrieRef; key: openArray[byte]): CoreDbRc[bool] =
proc contains*(trie: CoreDxTrieRefs; key: openArray[byte]): CoreDbRc[bool] =
trie.methods.containsFn key
proc rootHash*(trie: CoreDxTrieRef): CoreDbRc[Hash256] =
trie.methods.rootHashFn()
iterator pairs*(mpt: CoreDxMptRef): (Blob, Blob) {.apiRaise.} =
## Trie traversal, only supported for `CoreDxMptRef`
for k,v in mpt.methods.pairsIt():
@ -306,10 +363,26 @@ iterator replicate*(mpt: CoreDxMptRef): (Blob, Blob) {.apiRaise.} =
for k,v in mpt.methods.replicateIt():
yield (k,v)
proc backend*(trie: CoreDxTrieRef): CoreDbMptBackendRef =
## Getter, retrieves the *raw* backend object for special support.
result = trie.methods.backendFn()
result.parent = trie.parent
# ------------------------------------------------------------------------------
# Public trie database methods for accounts
# ------------------------------------------------------------------------------
proc fetch*(acc: CoreDxAccRef; address: EthAddress): CoreDbRc[CoreDbAccount] =
## Fetch data from the argument `trie`
acc.methods.fetchFn address
proc delete*(acc: CoreDxAccRef; address: EthAddress): CoreDbRc[void] =
acc.methods.deleteFn address
proc merge*(
acc: CoreDxAccRef;
address: EthAddress;
account: CoreDbAccount;
): CoreDbRc[void] =
acc.methods.mergeFn(address, account)
proc contains*(acc: CoreDxAccRef; address: EthAddress): CoreDbRc[bool] =
acc.methods.containsFn address
# ------------------------------------------------------------------------------
# Public transaction related methods
@ -317,7 +390,7 @@ proc backend*(trie: CoreDxTrieRef): CoreDbMptBackendRef =
proc toTransactionID*(db: CoreDbRef): CoreDbRc[CoreDxTxID] =
## Getter, current transaction state
db.new.getIdFn()
db.methods.getIdFn()
proc shortTimeReadOnly*(
id: CoreDxTxID;
@ -329,7 +402,7 @@ proc shortTimeReadOnly*(
proc newTransaction*(db: CoreDbRef): CoreDbRc[CoreDxTxRef] =
## Constructor
db.new.beginFn()
db.methods.beginFn()
proc commit*(tx: CoreDxTxRef, applyDeletes = true): CoreDbRc[void] =
tx.methods.commitFn applyDeletes
@ -352,12 +425,15 @@ proc newCapture*(
flags: set[CoreDbCaptFlags] = {};
): CoreDbRc[CoreDxCaptRef] =
## Constructor
db.new.captureFn flags
db.methods.captureFn flags
proc recorder*(db: CoreDxCaptRef): CoreDbRc[CoreDbRef] =
## Getter
db.methods.recorderFn()
proc logDb*(db: CoreDxCaptRef): CoreDbRc[CoreDbRef] =
db.methods.logDbFn()
proc flags*(db: CoreDxCaptRef): set[CoreDbCaptFlags] =
## Getter
db.methods.getFlagsFn()
@ -368,15 +444,18 @@ proc flags*(db: CoreDxCaptRef): set[CoreDbCaptFlags] =
when ProvideCoreDbLegacyAPI:
func parent*(cld: CoreDbChldRef): CoreDbRef =
func parent*(cld: CoreDbChldRefs): CoreDbRef =
## Getter, common method for all sub-modules
cld.distinctBase.parent()
proc backend*(dsc: CoreDbChldRefs): auto =
dsc.distinctBase.backend
# ----------------
func kvt*(db: CoreDbRef): CoreDbKvtRef =
proc kvt*(db: CoreDbRef): CoreDbKvtRef =
## Legacy pseudo constructor, see `toKvt()` for production constructor
db.toKvt.CoreDbKvtRef
db.newKvt().CoreDbKvtRef
proc get*(kvt: CoreDbKvtRef; key: openArray[byte]): Blob =
kvt.distinctBase.get(key).expect "kvt/get()"
@ -394,22 +473,17 @@ when ProvideCoreDbLegacyAPI:
for k,v in kvt.distinctBase.pairs():
yield (k,v)
proc backend*(kvt: CoreDbKvtRef): CoreDbKvtBackendRef =
kvt.distinctBase.backend
# ----------------
proc toMpt*(phk: CoreDbPhkRef): CoreDbMptRef =
phk.distinctBase.toMpt.CoreDbMptRef
proc mpt*(db: CoreDbRef; root=EMPTY_ROOT_HASH): CoreDbMptRef =
db.newMpt(root).expect("db/mpt()").CoreDbMptRef
proc mptPrune*(db: CoreDbRef; root: Hash256; prune = true): CoreDbMptRef =
db.newMptPrune(root, prune).expect("db/mptPrune()").CoreDbMptRef
let vid = db.getRoot(root, createOk=true).expect "mpt/getRoot()"
db.newMpt(vid, prune).CoreDbMptRef
proc mptPrune*(db: CoreDbRef; prune = true): CoreDbMptRef =
db.newMptPrune(EMPTY_ROOT_HASH, prune).expect("db/mptPrune()").CoreDbMptRef
db.newMpt(CoreDbVidRef(nil), prune).CoreDbMptRef
# ----------------
@ -417,48 +491,42 @@ when ProvideCoreDbLegacyAPI:
mpt.distinctBase.toPhk.CoreDbPhkRef
proc phkPrune*(db: CoreDbRef; root: Hash256; prune = true): CoreDbPhkRef =
db.newPhkPrune(root, prune).expect("db/phkPrune()").CoreDbPhkRef
let vid = db.getRoot(root, createOk=true).expect "phk/getRoot()"
db.newMpt(vid, prune).toCoreDxPhkRef.CoreDbPhkRef
proc phkPrune*(db: CoreDbRef; prune = true): CoreDbPhkRef =
db.newPhkPrune(EMPTY_ROOT_HASH, prune).expect("db/phkPrune()").CoreDbPhkRef
db.newMpt(CoreDbVidRef(nil), prune).toCoreDxPhkRef.CoreDbPhkRef
# ----------------
proc isPruning*(trie: CoreDbTrieRef): bool =
proc isPruning*(trie: CoreDbTrieRefs): bool =
trie.distinctBase.isPruning()
proc get*(trie: CoreDbTrieRef; key: openArray[byte]): Blob =
trie.distinctBase.get(key).expect "trie/get()"
proc get*(trie: CoreDbTrieRefs; key: openArray[byte]): Blob =
trie.distinctBase.fetch(key).expect "trie/get()"
proc del*(trie: CoreDbTrieRef; key: openArray[byte]) =
trie.distinctBase.del(key).expect "trie/del()"
proc del*(trie: CoreDbTrieRefs; key: openArray[byte]) =
trie.distinctBase.delete(key).expect "trie/del()"
proc put*(
trie: CoreDbTrieRef;
key: openArray[byte];
value: openArray[byte];
) =
trie.distinctBase.put(key, value).expect "trie/put()"
proc put*(trie: CoreDbTrieRefs; key: openArray[byte]; val: openArray[byte]) =
trie.distinctBase.merge(key, val).expect "trie/put()"
proc contains*(trie: CoreDbTrieRef; key: openArray[byte]): bool =
proc contains*(trie: CoreDbTrieRefs; key: openArray[byte]): bool =
trie.distinctBase.contains(key).expect "trie/contains()"
proc rootHash*(trie: CoreDbTrieRef): Hash256 =
trie.distinctBase.rootHash().expect "trie/rootHash()"
proc rootHash*(trie: CoreDbTrieRefs): Hash256 =
trie.distinctBase.rootVid().hash().expect "trie/rootHash()"
iterator pairs*(mpt: CoreDbMptRef): (Blob, Blob) {.apiRaise.} =
## Trie traversal, only supported for `CoreDbMptRef`
## Trie traversal, not supported for `CoreDbPhkRef`
for k,v in mpt.distinctBase.pairs():
yield (k,v)
iterator replicate*(mpt: CoreDbMptRef): (Blob, Blob) {.apiRaise.} =
## Low level trie dump, only supported for `CoreDbMptRef`
## Low level trie dump, not supported for `CoreDbPhkRef`
for k,v in mpt.distinctBase.replicate():
yield (k,v)
proc backend*(trie: CoreDbTrieRef): CoreDbMptBackendRef =
trie.distinctBase.backend
# ----------------
proc getTransactionID*(db: CoreDbRef): CoreDbTxID =
@ -512,6 +580,9 @@ when ProvideCoreDbLegacyAPI:
proc recorder*(db: CoreDbCaptRef): CoreDbRef =
db.distinctBase.recorder().expect("db/recorder()")
proc logDb*(db: CoreDbCaptRef): CoreDbRef =
db.distinctBase.logDb().expect("db/logDb()")
proc flags*(db: CoreDbCaptRef): set[CoreDbCaptFlags] =
db.distinctBase.flags()

177
nimbus/db/core_db/base/base_desc.nim
View File

@ -24,57 +24,76 @@ type
Ooops
LegacyDbMemory
LegacyDbPersistent
# AristoDbMemory
# AristoDbPersistent
AristoDbMemory ## Memory backend emulator
AristoDbRocks ## RocksDB backend
AristoDbVoid ## No backend (to be prefered over `XxxDbMemory`)
const
CoreDbPersistentTypes* = {LegacyDbPersistent}
CoreDbPersistentTypes* = {LegacyDbPersistent, AristoDbRocks}
type
CoreDbRc*[T] = Result[T,CoreDbErrorRef]
CoreDbAccount* = object
## Generic account representation referencing an *MPT* sub-trie
nonce*: AccountNonce ## Some `uint64` type
balance*: UInt256
storageVid*: CoreDbVidRef ## Implies storage root sub-MPT
codeHash*: Hash256
CoreDbErrorCode* = enum
Unspecified = 0
RlpException
KvtNotFound
MptNotFound
RootNotFound
CoreDbCaptFlags* {.pure.} = enum
PersistPut
PersistDel
# --------------------------------------------------
# Constructors
# --------------------------------------------------
CoreDbNewMptFn* =
proc(root: Hash256): CoreDbRc[CoreDxMptRef] {.noRaise.}
CoreDbNewLegaMptFn* =
proc(root: Hash256; prune: bool): CoreDbRc[CoreDxMptRef] {.noRaise.}
CoreDbNewTxGetIdFn* = proc(): CoreDbRc[CoreDxTxID] {.noRaise.}
CoreDbNewTxBeginFn* = proc(): CoreDbRc[CoreDxTxRef] {.noRaise.}
CoreDbNewCaptFn* =
proc(flgs: set[CoreDbCaptFlags]): CoreDbRc[CoreDxCaptRef] {.noRaise.}
CoreDbConstructorFns* = object
## Constructors
# Hexary trie
mptFn*: CoreDbNewMptFn
legacyMptFn*: CoreDbNewLegaMptFn # Legacy handler, should go away
# Transactions
getIdFn*: CoreDbNewTxGetIdFn
beginFn*: CoreDbNewTxBeginFn
# capture/tracer
captureFn*: CoreDbNewCaptFn
# --------------------------------------------------
# Sub-descriptor: Misc methods for main descriptor
# --------------------------------------------------
CoreDbBackendFn* = proc(): CoreDbBackendRef {.noRaise.}
CoreDbErrorPrintFn* = proc(e: CoreDbErrorRef): string {.noRaise.}
CoreDbInitLegaSetupFn* = proc() {.noRaise.}
CoreDbBaseBackendFn* = proc(): CoreDbBackendRef {.noRaise.}
CoreDbBaseDestroyFn* = proc(flush = true) {.noRaise.}
CoreDbBaseVidHashFn* =
proc(vid: CoreDbVidRef): Result[Hash256,void] {.noRaise.}
CoreDbBaseErrorPrintFn* = proc(e: CoreDbErrorRef): string {.noRaise.}
CoreDbBaseInitLegaSetupFn* = proc() {.noRaise.}
CoreDbBaseRootFn* =
proc(root: Hash256; createOk: bool): CoreDbRc[CoreDbVidRef] {.noRaise.}
CoreDbBaseKvtFn* = proc(): CoreDxKvtRef {.noRaise.}
CoreDbBaseMptFn* =
proc(root: CoreDbVidRef; prune: bool): CoreDbRc[CoreDxMptRef] {.noRaise.}
CoreDbBaseAccFn* =
proc(root: CoreDbVidRef; prune: bool): CoreDbRc[CoreDxAccRef] {.noRaise.}
CoreDbBaseTxGetIdFn* = proc(): CoreDbRc[CoreDxTxID] {.noRaise.}
CoreDbBaseTxBeginFn* = proc(): CoreDbRc[CoreDxTxRef] {.noRaise.}
CoreDbBaseCaptFn* =
proc(flgs: set[CoreDbCaptFlags]): CoreDbRc[CoreDxCaptRef] {.noRaise.}
CoreDbMiscFns* = object
backendFn*: CoreDbBackendFn
errorPrintFn*: CoreDbErrorPrintFn
legacySetupFn*: CoreDbInitLegaSetupFn
CoreDbBaseFns* = object
backendFn*: CoreDbBaseBackendFn
destroyFn*: CoreDbBaseDestroyFn
vidHashFn*: CoreDbBaseVidHashFn
errorPrintFn*: CoreDbBaseErrorPrintFn
legacySetupFn*: CoreDbBaseInitLegaSetupFn
getRootFn*: CoreDbBaseRootFn
# Kvt constructor
newKvtFn*: CoreDbBaseKvtFn
# Hexary trie constructors
newMptFn*: CoreDbBaseMptFn
newAccFn*: CoreDbBaseAccFn
# Transactions constructors
getIdFn*: CoreDbBaseTxGetIdFn
beginFn*: CoreDbBaseTxBeginFn
# capture/tracer constructors
captureFn*: CoreDbBaseCaptFn
# --------------------------------------------------
@ -99,32 +118,59 @@ type
# --------------------------------------------------
# Sub-descriptor: Mpt/hexary trie methods
# Sub-descriptor: generic Mpt/hexary trie methods
# --------------------------------------------------
CoreDbMptBackendFn* = proc(): CoreDbMptBackendRef {.noRaise.}
CoreDbMptGetFn* =
CoreDbMptFetchFn* =
proc(k: openArray[byte]): CoreDbRc[Blob] {.noRaise.}
CoreDbMptDelFn* =
CoreDbMptFetchAccountFn* =
proc(k: openArray[byte]): CoreDbRc[CoreDbAccount] {.noRaise.}
CoreDbMptDeleteFn* =
proc(k: openArray[byte]): CoreDbRc[void] {.noRaise.}
CoreDbMptPutFn* =
proc(k: openArray[byte]; v: openArray[byte]): CoreDbRc[void ] {.noRaise.}
CoreDbMptMergeFn* =
proc(k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] {.noRaise.}
CoreDbMptMergeAccountFn* =
proc(k: openArray[byte]; v: CoreDbAccount): CoreDbRc[void] {.noRaise.}
CoreDbMptContainsFn* = proc(k: openArray[byte]): CoreDbRc[bool] {.noRaise.}
CoreDbMptRootHashFn* = proc(): CoreDbRc[Hash256] {.noRaise.}
CoreDbMptRootVidFn* = proc(): CoreDbVidRef {.noRaise.}
CoreDbMptIsPruningFn* = proc(): bool {.noRaise.}
CoreDbMptPairsIt* = iterator(): (Blob,Blob) {.apiRaise.}
CoreDbMptReplicateIt* = iterator(): (Blob,Blob) {.apiRaise.}
CoreDbMptFns* = object
## Methods for trie objects
backendFn*: CoreDbMptBackendFn
getFn*: CoreDbMptGetFn
delFn*: CoreDbMptDelFn
putFn*: CoreDbMptPutFn
containsFn*: CoreDbMptContainsFn
rootHashFn*: CoreDbMptRootHashFn
pairsIt*: CoreDbMptPairsIt
replicateIt*: CoreDbMptReplicateIt
isPruningFn*: CoreDbMptIsPruningFn # Legacy handler, should go away
backendFn*: CoreDbMptBackendFn
fetchFn*: CoreDbMptFetchFn
deleteFn*: CoreDbMptDeleteFn
mergeFn*: CoreDbMptMergeFn
containsFn*: CoreDbMptContainsFn
rootVidFn*: CoreDbMptRootVidFn
pairsIt*: CoreDbMptPairsIt
replicateIt*: CoreDbMptReplicateIt
isPruningFn*: CoreDbMptIsPruningFn
# ----------------------------------------------------
# Sub-descriptor: Mpt/hexary trie methods for accounts
# ------------------------------------------------------
CoreDbAccBackendFn* = proc(): CoreDbAccBackendRef {.noRaise.}
CoreDbAccFetchFn* = proc(k: EthAddress): CoreDbRc[CoreDbAccount] {.noRaise.}
CoreDbAccDeleteFn* = proc(k: EthAddress): CoreDbRc[void] {.noRaise.}
CoreDbAccMergeFn* =
proc(k: EthAddress; v: CoreDbAccount): CoreDbRc[void] {.noRaise.}
CoreDbAccContainsFn* = proc(k: EthAddress): CoreDbRc[bool] {.noRaise.}
CoreDbAccRootVidFn* = proc(): CoreDbVidRef {.noRaise.}
CoreDbAccIsPruningFn* = proc(): bool {.noRaise.}
CoreDbAccFns* = object
## Methods for trie objects
backendFn*: CoreDbAccBackendFn
fetchFn*: CoreDbAccFetchFn
deleteFn*: CoreDbAccDeleteFn
mergeFn*: CoreDbAccMergeFn
containsFn*: CoreDbAccContainsFn
rootVidFn*: CoreDbAccRootVidFn
isPruningFn*: CoreDbAccIsPruningFn
# --------------------------------------------------
@ -156,10 +202,12 @@ type
# Sub-descriptor: capture recorder methods
# --------------------------------------------------
CoreDbCaptRecorderFn* = proc(): CoreDbRc[CoreDbRef] {.noRaise.}
CoreDbCaptLogDbFn* = proc(): CoreDbRc[CoreDbRef] {.noRaise.}
CoreDbCaptFlagsFn* = proc(): set[CoreDbCaptFlags] {.noRaise.}
CoreDbCaptFns* = object
recorderFn*: CoreDbCaptRecorderFn
logDbFn*: CoreDbCaptLogDbFn
getFlagsFn*: CoreDbCaptFlagsFn
# --------------------------------------------------
@ -168,12 +216,11 @@ type
CoreDbRef* = ref object of RootRef
## Database descriptor
dbType*: CoreDbType
kvtRef*: CoreDxKvtRef
new*: CoreDbConstructorFns
methods*: CoreDbMiscFns
methods*: CoreDbBaseFns
CoreDbErrorRef* = ref object of RootRef
## Generic error object
error*: CoreDbErrorCode
parent*: CoreDbRef
CoreDbBackendRef* = ref object of RootRef
@ -188,6 +235,10 @@ type
## Backend wrapper for direct backend access
parent*: CoreDbRef
CoreDbAccBackendRef* = ref object of RootRef
## Backend wrapper for direct backend access
parent*: CoreDbRef
CoreDxKvtRef* = ref object
## Statically initialised Key-Value pair table living in `CoreDbRef`
parent*: CoreDbRef
@ -199,10 +250,22 @@ type
parent*: CoreDbRef
methods*: CoreDbMptFns
CoreDxAccRef* = ref object
## Similar to `CoreDxKvtRef`, only dealing with `CoreDbAccount` data
## rather than `Blob` values.
parent*: CoreDbRef
methods*: CoreDbAccFns
CoreDbVidRef* = ref object of RootRef
## Generic state root: `Hash256` for legacy, `VertexID` for Aristo. This
## object makes only sense in the context od an *MPT*.
parent*: CoreDbRef
ready*: bool ## Must be set `true` to enable
CoreDxPhkRef* = ref object
## Similar to `CoreDbMptRef` but with pre-hashed keys. That is, any
## argument key for `put()`, `get()` etc. will be hashed first before
## being applied.
## argument key for `merge()`, `fetch()` etc. will be hashed first
## before being applied.
fromMpt*: CoreDxMptRef
methods*: CoreDbMptFns

68
nimbus/db/core_db/base/validate.nim
View File

@ -14,22 +14,34 @@ import
type
EphemMethodsDesc =
CoreDbBackendRef | CoreDbKvtBackendRef | CoreDbMptBackendRef
CoreDbBackendRef | CoreDbKvtBackendRef | CoreDbMptBackendRef |
CoreDbAccBackendRef | CoreDbVidRef
MethodsDesc =
CoreDxKvtRef |
CoreDxMptRef | CoreDxPhkRef |
CoreDxTxRef | CoreDxTxID |
CoreDxKvtRef |
CoreDxMptRef | CoreDxPhkRef | CoreDxAccRef |
CoreDxTxRef | CoreDxTxID |
CoreDxCaptRef
ValidateDesc* = MethodsDesc | EphemMethodsDesc | CoreDbErrorRef
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
proc validateMethodsDesc(msc: CoreDbMiscFns) =
doAssert not msc.backendFn.isNil
doAssert not msc.errorPrintFn.isNil
doAssert not msc.legacySetupFn.isNil
proc validateMethodsDesc(base: CoreDbBaseFns) =
doAssert not base.backendFn.isNil
doAssert not base.destroyFn.isNil
doAssert not base.vidHashFn.isNil
doAssert not base.errorPrintFn.isNil
doAssert not base.legacySetupFn.isNil
doAssert not base.getRootFn.isNil
doAssert not base.newKvtFn.isNil
doAssert not base.newMptFn.isNil
doAssert not base.newAccFn.isNil
doAssert not base.getIdFn.isNil
doAssert not base.beginFn.isNil
doAssert not base.captureFn.isNil
proc validateMethodsDesc(kvt: CoreDbKvtFns) =
doAssert not kvt.backendFn.isNil
@ -41,24 +53,31 @@ proc validateMethodsDesc(kvt: CoreDbKvtFns) =
proc validateMethodsDesc(fns: CoreDbMptFns) =
doAssert not fns.backendFn.isNil
doAssert not fns.getFn.isNil
doAssert not fns.delFn.isNil
doAssert not fns.putFn.isNil
doAssert not fns.fetchFn.isNil
doAssert not fns.deleteFn.isNil
doAssert not fns.mergeFn.isNil
doAssert not fns.containsFn.isNil
doAssert not fns.rootHashFn.isNil
doAssert not fns.rootVidFn.isNil
doAssert not fns.isPruningFn.isNil
doAssert not fns.pairsIt.isNil
doAssert not fns.replicateIt.isNil
proc validateConstructors(new: CoreDbConstructorFns) =
doAssert not new.mptFn.isNil
doAssert not new.legacyMptFn.isNil
doAssert not new.getIdFn.isNil
doAssert not new.beginFn.isNil
doAssert not new.captureFn.isNil
proc validateMethodsDesc(fns: CoreDbAccFns) =
doAssert not fns.backendFn.isNil
doAssert not fns.fetchFn.isNil
doAssert not fns.deleteFn.isNil
doAssert not fns.mergeFn.isNil
doAssert not fns.containsFn.isNil
doAssert not fns.rootVidFn.isNil
doAssert not fns.isPruningFn.isNil
# ------------
proc validateMethodsDesc(vid: CoreDbVidRef) =
doAssert not vid.isNil
doAssert not vid.parent.isNil
doAssert vid.ready == true
proc validateMethodsDesc(e: CoreDbErrorRef) =
doAssert not e.isNil
doAssert not e.parent.isNil
@ -66,7 +85,7 @@ proc validateMethodsDesc(e: CoreDbErrorRef) =
proc validateMethodsDesc(eph: EphemMethodsDesc) =
doAssert not eph.isNil
doAssert not eph.parent.isNil
proc validateMethodsDesc(kvt: CoreDxKvtRef) =
doAssert not kvt.isNil
doAssert not kvt.parent.isNil
@ -77,6 +96,11 @@ proc validateMethodsDesc(mpt: CoreDxMptRef) =
doAssert not mpt.parent.isNil
mpt.methods.validateMethodsDesc
proc validateMethodsDesc(acc: CoreDxAccRef) =
doAssert not acc.isNil
doAssert not acc.parent.isNil
acc.methods.validateMethodsDesc
proc validateMethodsDesc(phk: CoreDxPhkRef) =
doAssert not phk.isNil
doAssert not phk.fromMpt.isNil
@ -104,16 +128,14 @@ proc validateMethodsDesc(id: CoreDxTxID) =
proc validateMethodsDesc(db: CoreDbRef) =
doAssert not db.isNil
doAssert db.dbType != CoreDbType(0)
db.kvtRef.validateMethodsDesc
db.new.validateConstructors
db.methods.validateMethodsDesc
# ------------------------------------------------------------------------------
# Public debugging helpers
# ------------------------------------------------------------------------------
proc validate*(desc: MethodsDesc | EphemMethodsDesc | CoreDbErrorRef) =
desc.validateMethodsDesc
proc validate*(dsc: ValidateDesc) =
dsc.validateMethodsDesc
proc validate*(db: CoreDbRef) =
db.validateMethodsDesc

26
nimbus/db/core_db/memory_only.nim
View File

@ -13,14 +13,18 @@
import
std/options,
eth/[common, trie/db],
"."/[base, core_apps, legacy_db]
./backend/[legacy_db],
"."/[base, core_apps]
export
common,
core_apps,
# Not all symbols from the object sources will be exported by default
CoreDbAccount,
CoreDbApiError,
CoreDbCaptFlags,
CoreDbErrorCode,
CoreDbErrorRef,
CoreDbCaptRef,
CoreDbKvtRef,
@ -30,6 +34,8 @@ export
CoreDbTxID,
CoreDbTxRef,
CoreDbType,
CoreDbVidRef,
CoreDxAccRef,
CoreDxCaptRef,
CoreDxKvtRef,
CoreDxMptRef,
@ -44,26 +50,34 @@ export
contains,
dbType,
del,
delete,
dispose,
fetch,
finish,
get,
getRoot,
getTransactionID,
hash,
isLegacy,
isPruning,
kvt,
logDb,
merge,
mptPrune,
newAccMpt,
newCapture,
newMpt,
newMptPrune,
newPhkPrune,
newTransaction,
pairs,
parent,
phkPrune,
put,
recast,
recorder,
replicate,
rollback,
rootHash,
rootVid,
safeDispose,
setTransactionID,
toLegacy,
@ -86,7 +100,9 @@ proc newCoreDbRef*(
##
db.newLegacyPersistentCoreDbRef()
proc newCoreDbRef*(dbType: static[CoreDbType]): CoreDbRef =
proc newCoreDbRef*(
dbType: static[CoreDbType]; # Database type symbol
): CoreDbRef =
## Constructor for volatile/memory type DB
##
## Note: Using legacy notation `newCoreDbRef()` rather than
@ -96,7 +112,7 @@ proc newCoreDbRef*(dbType: static[CoreDbType]): CoreDbRef =
newLegacyMemoryCoreDbRef()
else:
{.error: "Unsupported dbType for memory newCoreDbRef()".}
{.error: "Unsupported dbType for memory-only newCoreDbRef()".}
# ------------------------------------------------------------------------------
# Public template wrappers

17
nimbus/db/core_db/persistent.nim
View File

@ -5,7 +5,7 @@
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
## This module automatically pulls in the persistent backend library at the
## This module automatically pulls in the persistent backend libraries at the
## linking stage (e.g. `rocksdb`) which can be avoided for pure memory DB
## applications by importing `db/code_db/memory_only` (rather than
## `db/core_db/persistent`.)
@ -13,12 +13,18 @@
{.push raises: [].}
import
"."/[memory_only, legacy_rocksdb]
../aristo,
./backend/[legacy_rocksdb],
./memory_only
export
memory_only
memory_only,
toRocksStoreRef
proc newCoreDbRef*(dbType: static[CoreDbType]; path: string): CoreDbRef =
proc newCoreDbRef*(
dbType: static[CoreDbType]; # Database type symbol
path: string; # Storage path for database
): CoreDbRef =
## Constructor for persistent type DB
##
## Note: Using legacy notation `newCoreDbRef()` rather than
@ -26,6 +32,9 @@ proc newCoreDbRef*(dbType: static[CoreDbType]; path: string): CoreDbRef =
when dbType == LegacyDbPersistent:
newLegacyPersistentCoreDbRef path
elif dbType == AristoDbRocks:
newAristoRocksDbCoreDbRef path
else:
{.error: "Unsupported dbType for persistent newCoreDbRef()".}

2
nimbus/db/state_db.nim
View File

@ -63,7 +63,7 @@ proc pruneTrie*(db: AccountStateDB): bool =
func db*(db: AccountStateDB): CoreDbRef =
db.trie.db
func kvt*(db: AccountStateDB): CoreDbKvtRef =
proc kvt*(db: AccountStateDB): CoreDbKvtRef =
db.trie.db.kvt
proc rootHash*(db: AccountStateDB): KeccakHash =

2
nimbus/evm/interpreter_dispatch.nim
View File

@ -121,7 +121,7 @@ proc afterExecCall(c: Computation) =
## also see: https://github.com/ethereum/EIPs/issues/716
if c.isError or c.fork >= FkByzantium:
if c.msg.contractAddress == ripemdAddr:
if c.msg.contractAddress == RIPEMD_ADDR:
# Special case to account for geth+parity bug
c.vmState.stateDB.ripemdSpecial()

2
nimbus/sync/handlers/snap.nim
View File

@ -17,7 +17,7 @@ import
eth/[common, p2p, trie/db, trie/nibbles],
stew/[byteutils, interval_set],
../../core/chain,
../../db/core_db/legacy_db,
../../db/core_db,
../snap/[constants, range_desc],
../snap/worker/db/[hexary_desc, hexary_error, hexary_paths,
snapdb_persistent, hexary_range],

2
nimbus/sync/snap/worker/db/snapdb_desc.nim
View File

@ -14,7 +14,7 @@ import
std/tables,
chronicles,
eth/[common, p2p, trie/nibbles],
../../../../db/core_db/legacy_rocksdb,
../../../../db/core_db/persistent,
../../../../db/[core_db, select_backend, storage_types],
../../../protocol,
../../range_desc,

3
nimbus/sync/snap/worker/db/snapdb_persistent.nim
View File

@ -14,7 +14,8 @@ import
std/[algorithm, tables],
chronicles,
eth/[common, trie/db],
../../../../db/[core_db, core_db/legacy_db, kvstore_rocksdb],
results,
../../../../db/[core_db, kvstore_rocksdb],
../../range_desc,
"."/[hexary_desc, hexary_error, rocky_bulk_load, snapdb_desc]

2
nimbus/tracer.nim
View File

@ -76,7 +76,7 @@ proc dumpMemoryDB*(node: JsonNode, db: CoreDbRef) =
node["state"] = n
proc dumpMemoryDB*(node: JsonNode, capture: CoreDbCaptRef) =
node.dumpMemoryDB capture.recorder
node.dumpMemoryDB capture.logDb
const
senderName = "sender"

50
tests/replay/xcheck.nim Normal file
View File

@ -0,0 +1,50 @@
# Nimbus - Types, data structures and shared utilities used in network sync
#
# Copyright (c) 2018-2021 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or
# distributed except according to those terms.
import
unittest2
# ------------------------------------------------------------------------------
# Public workflow helpers
# ------------------------------------------------------------------------------
template xCheck*(expr: untyped): untyped =
## Note: this check will invoke `expr` twice
if not (expr):
check expr
return
template xCheck*(expr: untyped; ifFalse: untyped): untyped =
## Note: this check will invoke `expr` twice
if not (expr):
ifFalse
check expr
return
template xCheckRc*(expr: untyped): untyped =
if rc.isErr:
xCheck(expr)
template xCheckRc*(expr: untyped; ifFalse: untyped): untyped =
if rc.isErr:
xCheck(expr, ifFalse)
template xCheckErr*(expr: untyped): untyped =
if rc.isOk:
xCheck(expr)
template xCheckErr*(expr: untyped; ifFalse: untyped): untyped =
if rc.isOk:
xCheck(expr, ifFalse)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

1
tests/test_aristo/test_backend.nim
View File

@ -28,6 +28,7 @@ import
aristo_persistent,
aristo_transcode,
aristo_vid],
../replay/xcheck,
./test_helpers
const

1
tests/test_aristo/test_filter.nim
View File

@ -23,6 +23,7 @@ import
../../nimbus/db/aristo,
../../nimbus/db/aristo/aristo_desc/desc_backend,
../../nimbus/db/aristo/aristo_filter/[filter_fifos, filter_scheduler],
../replay/xcheck,
./test_helpers
type

33
tests/test_aristo/test_helpers.nim
View File

@ -192,39 +192,6 @@ proc mapRootVid*(
leafTie: LeafTie(root: toVid, path: it.leafTie.path),
payload: it.payload))
# ------------------------------------------------------------------------------
# Public workflow helpers
# ------------------------------------------------------------------------------
template xCheck*(expr: untyped): untyped =
## Note: this check will invoke `expr` twice
if not (expr):
check expr
return
template xCheck*(expr: untyped; ifFalse: untyped): untyped =
## Note: this check will invoke `expr` twice
if not (expr):
ifFalse
check expr
return
template xCheckRc*(expr: untyped): untyped =
if rc.isErr:
xCheck(expr)
template xCheckRc*(expr: untyped; ifFalse: untyped): untyped =
if rc.isErr:
xCheck(expr, ifFalse)
template xCheckErr*(expr: untyped): untyped =
if rc.isOk:
xCheck(expr)
template xCheckErr*(expr: untyped; ifFalse: untyped): untyped =
if rc.isOk:
xCheck(expr, ifFalse)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

1
tests/test_aristo/test_misc.nim
View File

@ -22,6 +22,7 @@ import
../../nimbus/db/aristo/[
aristo_debug, aristo_desc, aristo_transcode, aristo_vid],
../../nimbus/db/aristo/aristo_filter/filter_scheduler,
../replay/xcheck,
./test_helpers
type

1
tests/test_aristo/test_tx.nim
View File

@ -21,6 +21,7 @@ import
aristo_check, aristo_debug, aristo_delete, aristo_desc, aristo_get,
aristo_merge],
../../nimbus/db/[aristo, aristo/aristo_init/persistent],
../replay/xcheck,
./test_helpers
type

33
tests/test_coredb/test_helpers.nim
View File

@ -39,39 +39,6 @@ proc say*(noisy = false; pfx = "***"; args: varargs[string, `$`]) =
# Public helpers
# ------------------------------------------------------------------------------
# ------------------------------------------------------------------------------
# Public workflow helpers
# ------------------------------------------------------------------------------
template xCheck*(expr: untyped): untyped =
## Note: this check will invoke `expr` twice
if not (expr):
check expr
return
template xCheck*(expr: untyped; ifFalse: untyped): untyped =
## Note: this check will invoke `expr` twice
if not (expr):
ifFalse
check expr
return
template xCheckRc*(expr: untyped): untyped =
if rc.isErr:
xCheck(expr)
template xCheckRc*(expr: untyped; ifFalse: untyped): untyped =
if rc.isErr:
xCheck(expr, ifFalse)
template xCheckErr*(expr: untyped): untyped =
if rc.isOk:
xCheck(expr)
template xCheckErr*(expr: untyped; ifFalse: untyped): untyped =
if rc.isOk:
xCheck(expr, ifFalse)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

2
tests/test_coredb/test_legacy.nim
View File

@ -15,7 +15,7 @@ import
results,
unittest2,
../../nimbus/[core/chain],
../replay/undump_blocks,
../replay/[undump_blocks, xcheck],
./test_helpers
# ------------------------------------------------------------------------------

2
tests/test_rocksdb_timing.nim
View File

@ -17,7 +17,7 @@ import
eth/[common, p2p],
rocksdb,
unittest2,
../nimbus/db/core_db/[legacy_rocksdb, persistent],
../nimbus/db/core_db/persistent,
../nimbus/core/chain,
../nimbus/sync/snap/range_desc,
../nimbus/sync/snap/worker/db/hexary_desc,

2
tests/test_rocksdb_timing/test_db_timing.nim
View File

@ -18,7 +18,7 @@ import
unittest2,
../../nimbus/core/chain,
../../nimbus/db/core_db,
../../nimbus/db/core_db/legacy_rocksdb,
../../nimbus/db/core_db/persistent,
../../nimbus/sync/snap/range_desc,
../../nimbus/sync/snap/worker/db/[hexary_desc, rocky_bulk_load],
../../nimbus/utils/prettify,

2
tests/test_sync_snap.nim
View File

@ -18,7 +18,7 @@ import
rocksdb,
unittest2,
../nimbus/db/[core_db, kvstore_rocksdb],
../nimbus/db/core_db/[legacy_rocksdb, persistent],
../nimbus/db/core_db/persistent,
../nimbus/core/chain,
../nimbus/sync/types,
../nimbus/sync/snap/range_desc,

2
tests/test_sync_snap/test_syncdb.nim
View File

@ -18,7 +18,7 @@ import
unittest2,
../../nimbus/common as nimbus_common,
../../nimbus/core/chain,
../../nimbus/db/[core_db/legacy_db, storage_types],
../../nimbus/db/storage_types,
../../nimbus/sync/snap/worker/db/snapdb_desc,
../replay/[pp, undump_blocks, undump_kvp],
./test_helpers