From 3198ad1bbd9ef3c686ed000ac8dccfa82cd9d81f Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Wed, 25 Oct 2023 15:03:09 +0100 Subject: [PATCH] Fix default pruning for ledger and update core db and ledger logging (#1861) * Make sure that storage tries are not pruned (by default) on the new Ledger API why: Pruning might kill some unwanted entries from storage tries ending up with an unstable database leading to crashes. * Implement `CoreDb` and `LedgerRef` API tracing details: + Locally enabled at compile time via constants `ProvideCoreDbLegacyAPI` and `EnableApiTracking` in either `base.nim` source + If enabled it can be selectively turned on/off via public switches in the `CoreDb` descriptor. * Allow suppressing opportunistic `ifNecessaryGetXxx()` functions why: Better troubleshooting when the system crashes (assertions will then most probably happen outside an `async` function.) --- nimbus/db/core_db/backend/legacy_db.nim | 15 +- nimbus/db/core_db/base.nim | 442 ++++++++++++++---- nimbus/db/core_db/base/base_desc.nim | 11 +- nimbus/db/core_db/memory_only.nim | 4 +- nimbus/db/core_db/persistent.nim | 3 - nimbus/db/distinct_tries.nim | 55 ++- nimbus/db/ledger.nim | 21 + nimbus/db/ledger/accounts_cache.nim | 1 + nimbus/db/ledger/accounts_ledger.nim | 6 +- nimbus/db/ledger/backend/accounts_cache.nim | 21 +- nimbus/db/ledger/backend/accounts_ledger.nim | 24 +- nimbus/db/ledger/base.nim | 153 ++++-- nimbus/db/ledger/base/base_desc.nim | 1 + nimbus/db/ledger/distinct_ledgers.nim | 60 ++- nimbus/db/state_db.nim | 2 +- nimbus/evm/async/operations.nim | 7 + tests/replay/pp_light.nim | 7 +- tests/test_coredb.nim | 13 +- tests/test_coredb/coredb_test_xx.nim | 6 +- .../{test_legacy.nim => test_chainsync.nim} | 9 +- 20 files changed, 677 insertions(+), 184 deletions(-) rename tests/test_coredb/{test_legacy.nim => test_chainsync.nim} (95%) diff --git a/nimbus/db/core_db/backend/legacy_db.nim b/nimbus/db/core_db/backend/legacy_db.nim index dc83f61f5..5e6506b3f 100644 --- a/nimbus/db/core_db/backend/legacy_db.nim +++ b/nimbus/db/core_db/backend/legacy_db.nim @@ -208,8 +208,7 @@ proc mptMethods(mpt: HexaryChildDbRef; db: LegacyDbRef): CoreDbMptFns = fetchFn: proc(k: openArray[byte]): CoreDbRc[Blob] = db.mapRlpException("legacy/mpt/get()"): - return ok(mpt.trie.get(k)) - , + return ok(mpt.trie.get(k)), deleteFn: proc(k: openArray[byte]): CoreDbRc[void] = db.mapRlpException("legacy/mpt/del()"): @@ -223,8 +222,7 @@ proc mptMethods(mpt: HexaryChildDbRef; db: LegacyDbRef): CoreDbMptFns = containsFn: proc(k: openArray[byte]): CoreDbRc[bool] = db.mapRlpException("legacy/mpt/put()"): - return ok(mpt.trie.contains(k)) - , + return ok(mpt.trie.contains(k)), rootVidFn: proc(): CoreDbVidRef = db.bless(LegacyCoreDbVid(vHash: mpt.trie.rootHash)), @@ -252,8 +250,7 @@ proc accMethods(mpt: HexaryChildDbRef; db: LegacyDbRef): CoreDbAccFns = fetchFn: proc(k: EthAddress): CoreDbRc[CoreDbAccount] = const info = "legacy/mpt/getAccount()" db.mapRlpException info: - return ok mpt.trie.get(k.keccakHash.data).toCoreDbAccount(db) - , + return ok mpt.trie.get(k.keccakHash.data).toCoreDbAccount(db), deleteFn: proc(k: EthAddress): CoreDbRc[void] = db.mapRlpException("legacy/mpt/del()"): @@ -267,8 +264,7 @@ proc accMethods(mpt: HexaryChildDbRef; db: LegacyDbRef): CoreDbAccFns = containsFn: proc(k: EthAddress): CoreDbRc[bool] = db.mapRlpException("legacy/mpt/put()"): - return ok(mpt.trie.contains k.keccakHash.data) - , + return ok(mpt.trie.contains k.keccakHash.data), rootVidFn: proc(): CoreDbVidRef = db.bless(LegacyCoreDbVid(vHash: mpt.trie.rootHash)), @@ -327,8 +323,7 @@ proc baseMethods( destroyFn: proc(ignore: bool) = if not closeDb.isNil: - closeDb() - , + closeDb(), vidHashFn: proc(vid: CoreDbVidRef): Result[Hash256,void] = ok(vid.lvHash), diff --git a/nimbus/db/core_db/base.nim b/nimbus/db/core_db/base.nim index ed90bb168..1b92a82be 100644 --- a/nimbus/db/core_db/base.nim +++ b/nimbus/db/core_db/base.nim @@ -11,7 +11,7 @@ {.push raises: [].} import - std/[options, typetraits], + std/typetraits, chronicles, eth/common, results, @@ -39,9 +39,6 @@ export CoreDxTxID, CoreDxTxRef -logScope: - topics = "core_db-base" - when defined(release): const AutoValidateDescriptors = false else: @@ -50,6 +47,11 @@ else: const ProvideCoreDbLegacyAPI = true + EnableApiTracking = true and false + ## When enabled, functions using this tracking facility need to import + ## `chronicles`, as well. Tracking is enabled by setting the `trackLegaApi` + ## and/or the `trackNewApi` flags to `true`. + # Annotation helpers {.pragma: noRaise, gcsafe, raises: [].} {.pragma: apiRaise, gcsafe, raises: [CoreDbApiError].} @@ -85,12 +87,12 @@ type CoreDbMptBackendRef | CoreDbAccBackendRef ## Shortcut, all backend descriptors. - CoreDxChldRefs = CoreDxKvtRef | CoreDxTrieRelated | CoreDbBackends | - CoreDbErrorRef + CoreDxChldRefs = CoreDxKvtRef | CoreDxTrieRelated | CoreDbVidRef | + CoreDbBackends | CoreDbErrorRef ## Shortcut, all descriptors with a `parent` entry. # ------------------------------------------------------------------------------ -# Private functions: helpers +# Private helpers # ------------------------------------------------------------------------------ template logTxt(info: static[string]): static[string] = @@ -101,6 +103,99 @@ template itNotImplemented(db: CoreDbRef, name: string) = # --------- +when EnableApiTracking: + import std/[sequtils, strutils], stew/byteutils + + template newApiTxt(info: static[string]): static[string] = + logTxt "new API " & info + + template legaApiTxt(info: static[string]): static[string] = + logTxt "legacy API " & info + + func getParent(w: CoreDxChldRefs): auto = + ## Avoida inifinite call to `parent()` in `ifTrack*Api()` tmplates + w.parent + + template setTrackLegaApiOnly(w: CoreDbChldRefs|CoreDbRef) = + when typeof(w) is CoreDbRef: + let db = w + else: + let db = w.distinctBase.getParent + let save = db.trackNewApi + # Prevent from cascaded logging + db.trackNewApi = false + defer: db.trackNewApi = save + + template ifTrackLegaApi(w: CoreDbChldRefs|CoreDbRef; code: untyped) = + block: + when typeof(w) is CoreDbRef: + let db = w + else: + let db = w.distinctBase.getParent + if db.trackLegaApi: + code + + template ifTrackNewApi(w: CoreDxChldRefs|CoreDbRef; code: untyped) = + block: + when typeof(w) is CoreDbRef: + let db = w + else: + if w.isNil: break + let db = w.getParent + if db.trackNewApi: + code + + proc oaToStr(w: openArray[byte]): string = + w.toHex.toLowerAscii + + proc toStr(w: Hash256): string = + if w == EMPTY_ROOT_HASH: "EMPTY_ROOT_HASH" else: w.data.oaToStr + + proc toStr(p: CoreDbVidRef): string = + if p.isNil: "vidRef(nil)" else: "vidRef" + + proc toStr(w: Blob): string = + if 0 < w.len and w.len < 5: "<" & w.oaToStr & ">" + else: "Blob[" & $w.len & "]" + + proc toStr(w: openArray[byte]): string = + w.oaToStr + + proc toStr(w: set[CoreDbCaptFlags]): string = + "Flags[" & $w.len & "]" + + proc toStr(rc: CoreDbRc[bool]): string = + if rc.isOk: "ok(" & $rc.value & ")" else: "err(..)" + + proc toStr(rc: CoreDbRc[void]): string = + if rc.isOk: "ok()" else:"err()" + + proc toStr(rc: CoreDbRc[Blob]): string = + if rc.isOk: "ok(Blob[" & $rc.value.len & "])" else: "err(..)" + + proc toStr(rc: Result[Hash256,void]): string = + if rc.isOk: "ok(" & rc.value.toStr & ")" else: "err()" + + proc toStr(rc: Result[Account,void]): string = + if rc.isOk: "ok(Account)" else: "err()" + + proc toStr[T](rc: CoreDbRc[T]; ifOk: static[string]): string = + if rc.isOk: "ok(" & ifOk & ")" else: "err(..)" + + proc toStr(rc: CoreDbRc[CoreDbRef]): string = rc.toStr "dbRef" + proc toStr(rc: CoreDbRc[CoreDbVidRef]): string = rc.toStr "vidRef" + proc toStr(rc: CoreDbRc[CoreDbAccount]): string = rc.toStr "accRef" + proc toStr(rc: CoreDbRc[CoreDxTxID]): string = rc.toStr "txId" + proc toStr(rc: CoreDbRc[CoreDxTxRef]): string = rc.toStr "txRef" + proc toStr(rc: CoreDbRc[CoreDxCaptRef]): string = rc.toStr "captRef" + +else: + template setTrackLegaApiOnly(w: CoreDbChldRefs|CoreDbRef) = discard + template ifTrackLegaApi(w: CoreDbChldRefs|CoreDbRef; code: untyped) = discard + template ifTrackNewApi(w: CoreDxChldRefs|CoreDbRef; code: untyped) = discard + +# --------- + func toCoreDxPhkRef(mpt: CoreDxMptRef): CoreDxPhkRef = ## MPT => pre-hashed MPT (aka PHK) result = CoreDxPhkRef( @@ -125,11 +220,11 @@ func toCoreDxPhkRef(mpt: CoreDxMptRef): CoreDxPhkRef = result.methods.pairsIt = iterator(): (Blob, Blob) {.apiRaise.} = - mpt.parent.itNotImplemented("pairs/phk") + mpt.parent.itNotImplemented("phk/pairs()") result.methods.replicateIt = iterator(): (Blob, Blob) {.apiRaise.} = - mpt.parent.itNotImplemented("replicate/phk") + mpt.parent.itNotImplemented("phk/replicate()") when AutoValidateDescriptors: result.validate @@ -146,6 +241,7 @@ proc bless*(db: CoreDbRef): CoreDbRef = ## Verify descriptor when AutoValidateDescriptors: db.validate + db.ifTrackNewApi: info newApiTxt "CoreDbRef.init()", dbType=db.dbType db proc bless*(db: CoreDbRef; child: CoreDbVidRef): CoreDbVidRef = @@ -187,21 +283,24 @@ proc bless*[T: CoreDxTrieRelated | CoreDbErrorRef | CoreDbBackends]( proc dbType*(db: CoreDbRef): CoreDbType = ## Getter - db.dbType + result = db.dbType + db.ifTrackNewApi: info newApiTxt "dbType()", result proc compensateLegacySetup*(db: CoreDbRef) = ## On the persistent legacy hexary trie, this function is needed for ## bootstrapping and Genesis setup when the `purge` flag is activated. ## Otherwise the database backend may defect on an internal inconsistency. db.methods.legacySetupFn() + db.ifTrackNewApi: info newApiTxt "compensateLegacySetup()" -func parent*(cld: CoreDxChldRefs): CoreDbRef = +proc parent*(cld: CoreDxChldRefs): CoreDbRef = ## Getter, common method for all sub-modules - cld.parent + result = cld.parent proc backend*(dsc: CoreDxKvtRef | CoreDxTrieRelated | CoreDbRef): auto = ## Getter, retrieves the *raw* backend object for special/localised support. - dsc.methods.backendFn() + result = dsc.methods.backendFn() + dsc.ifTrackNewApi: info newApiTxt "backend()" proc finish*(db: CoreDbRef; flush = false) = ## Database destructor. If the argument `flush` is set `false`, the database @@ -211,11 +310,13 @@ proc finish*(db: CoreDbRef; flush = false) = ## depends on the backend database. Currently, only the `AristoDbRocks` type ## backend removes the database on `true`. db.methods.destroyFn flush + db.ifTrackNewApi: info newApiTxt "finish()" proc `$$`*(e: CoreDbErrorRef): string = ## Pretty print error symbol, note that this directive may have side effects ## as it calls a backend function. - e.parent.methods.errorPrintFn(e) + result = e.parent.methods.errorPrintFn(e) + e.ifTrackNewApi: info newApiTxt "$$()", result proc hash*(vid: CoreDbVidRef): Result[Hash256,void] = ## Getter (well, sort of), retrieves the hash for a `vid` argument. The @@ -226,9 +327,17 @@ proc hash*(vid: CoreDbVidRef): Result[Hash256,void] = ## The value `EMPTY_ROOT_HASH` is also returned on an empty `vid` argument ## `CoreDbVidRef(nil)`, say. ## - if not vid.isNil and vid.ready: - return vid.parent.methods.vidHashFn vid - ok EMPTY_ROOT_HASH + result = block: + if not vid.isNil and vid.ready: + vid.parent.methods.vidHashFn vid + else: + ok EMPTY_ROOT_HASH + # Note: tracker will be silent if `vid` is NIL + vid.ifTrackNewApi: info newApiTxt "hash()", result=result.toStr + +proc hashOrEmpty*(vid: CoreDbVidRef): Hash256 = + ## Convenience wrapper, returns `EMPTY_ROOT_HASH` where `hash()` would fail. + vid.hash.valueOr: EMPTY_ROOT_HASH proc recast*(account: CoreDbAccount): Result[Account,void] = ## Convert the argument `account` to the portable Ethereum representation @@ -237,11 +346,18 @@ proc recast*(account: CoreDbAccount): Result[Account,void] = ## ## Note that for the legacy backend, this function always succeeds. ## - ok Account( - nonce: account.nonce, - balance: account.balance, - codeHash: account.codeHash, - storageRoot: ? account.storageVid.hash) + let vid = account.storageVid + result = block: + let rc = vid.hash + if rc.isOk: + ok Account( + nonce: account.nonce, + balance: account.balance, + codeHash: account.codeHash, + storageRoot: rc.value) + else: + err() + vid.ifTrackNewApi: info newApiTxt "recast()", result=result.toStr proc getRoot*( db: CoreDbRef; @@ -261,7 +377,9 @@ proc getRoot*( ## return ## db.newAccMpt root ## - db.methods.getRootFn(root, createOk) + result = db.methods.getRootFn(root, createOk) + db.ifTrackNewApi: + info newApiTxt "getRoot()", root=root.toStr, result=result.toStr # ------------------------------------------------------------------------------ # Public key-value table methods @@ -269,28 +387,38 @@ proc getRoot*( proc newKvt*(db: CoreDbRef): CoreDxKvtRef = ## Getter (pseudo constructor) - db.methods.newKvtFn() + result = db.methods.newKvtFn() + db.ifTrackNewApi: info newApiTxt "newKvt()" proc get*(kvt: CoreDxKvtRef; key: openArray[byte]): CoreDbRc[Blob] = - kvt.methods.getFn key + result = kvt.methods.getFn key + kvt.ifTrackNewApi: + info newApiTxt "kvt/get()", key=key.toStr, result=result.toStr proc del*(kvt: CoreDxKvtRef; key: openArray[byte]): CoreDbRc[void] = - kvt.methods.delFn key + result = kvt.methods.delFn key + kvt.ifTrackNewApi: + info newApiTxt "kvt/del()", key=key.toStr, result=result.toStr proc put*( kvt: CoreDxKvtRef; key: openArray[byte]; - value: openArray[byte]; + val: openArray[byte]; ): CoreDbRc[void] = - kvt.methods.putFn(key, value) + result = kvt.methods.putFn(key, val) + kvt.ifTrackNewApi: info newApiTxt "kvt/put()", + key=key.toStr, val=val.toSeq.toStr, result=result.toStr proc contains*(kvt: CoreDxKvtRef; key: openArray[byte]): CoreDbRc[bool] = - kvt.methods.containsFn key + result = kvt.methods.containsFn key + kvt.ifTrackNewApi: + info newApiTxt "kvt/contains()", key=key.toStr, result=result.toStr iterator pairs*(kvt: CoreDxKvtRef): (Blob, Blob) {.apiRaise.} = ## Iterator supported on memory DB (otherwise implementation dependent) for k,v in kvt.methods.pairsIt(): yield (k,v) + kvt.ifTrackNewApi: info newApiTxt "kvt/pairs()" # ------------------------------------------------------------------------------ # Public Merkle Patricia Tree, hexary trie constructors @@ -299,26 +427,30 @@ iterator pairs*(kvt: CoreDxKvtRef): (Blob, Blob) {.apiRaise.} = proc newMpt*(db: CoreDbRef; root: CoreDbVidRef; prune = true): CoreDxMptRef = ## Constructor, will defect on failure (note that the legacy backend ## always succeeds) - db.methods.newMptFn(root, prune).valueOr: raiseAssert $$error + result = db.methods.newMptFn(root, prune).valueOr: raiseAssert $$error + db.ifTrackNewApi: info newApiTxt "newMpt", root=root.toStr, prune proc newAccMpt*(db: CoreDbRef; root: CoreDbVidRef; prune = true): CoreDxAccRef = ## Similar to `newMpt()` for handling accounts. Although this sub-trie can ## be emulated by means of `newMpt(..).toPhk()`, it is recommended using ## this constructor which implies its own subset of methods to handle that ## trie. - db.methods.newAccFn(root, prune).valueOr: raiseAssert $$error + result = db.methods.newAccFn(root, prune).valueOr: raiseAssert $$error + db.ifTrackNewApi: info newApiTxt "newAccMpt", root=root.toStr, prune proc toMpt*(phk: CoreDxPhkRef): CoreDxMptRef = ## Replaces the pre-hashed argument trie `phk` by the non pre-hashed *MPT*. ## Note that this does not apply to an accounts trie that was created by ## `newAccMpt()`. - phk.fromMpt + result = phk.fromMpt + phk.ifTrackNewApi: info newApiTxt "phk/toMpt()" proc toPhk*(mpt: CoreDxMptRef): CoreDxPhkRef = ## Replaces argument `mpt` by a pre-hashed *MPT*. ## Note that this does not apply to an accounts trie that was created by ## `newAaccMpt()`. - mpt.toCoreDxPhkRef + result = mpt.toCoreDxPhkRef + mpt.ifTrackNewApi: info newApiTxt "mpt/toPhk()" # ------------------------------------------------------------------------------ # Public common methods for all hexary trie databases (`mpt`, `phk`, or `acc`) @@ -326,11 +458,13 @@ proc toPhk*(mpt: CoreDxMptRef): CoreDxPhkRef = proc isPruning*(dsc: CoreDxTrieRefs | CoreDxAccRef): bool = ## Getter - dsc.methods.isPruningFn() + result = dsc.methods.isPruningFn() + dsc.ifTrackNewApi: info newApiTxt "isPruning()", result proc rootVid*(dsc: CoreDxTrieRefs | CoreDxAccRef): CoreDbVidRef = ## Getter, result is not `nil` - dsc.methods.rootVidFn() + result = dsc.methods.rootVidFn() + dsc.ifTrackNewApi: info newApiTxt "rootVid()", result=result.toStr # ------------------------------------------------------------------------------ # Public generic hexary trie database methods (`mpt` or `phk`) @@ -338,30 +472,44 @@ proc rootVid*(dsc: CoreDxTrieRefs | CoreDxAccRef): CoreDbVidRef = proc fetch*(trie: CoreDxTrieRefs; key: openArray[byte]): CoreDbRc[Blob] = ## Fetch data from the argument `trie` - trie.methods.fetchFn(key) + result = trie.methods.fetchFn(key) + trie.ifTrackNewApi: + info newApiTxt "trie/fetch()", key=key.toStr, result=result.toStr proc delete*(trie: CoreDxTrieRefs; key: openArray[byte]): CoreDbRc[void] = - trie.methods.deleteFn key + result = trie.methods.deleteFn key + trie.ifTrackNewApi: + info newApiTxt "trie/delete()", key=key.toStr, result=result.toStr proc merge*( trie: CoreDxTrieRefs; key: openArray[byte]; - value: openArray[byte]; + val: openArray[byte]; ): CoreDbRc[void] = - trie.methods.mergeFn(key, value) + when trie is CoreDbMptRef: + const info = "mpt/merge()" + else: + const info = "phk/merge()" + result = trie.methods.mergeFn(key, val) + trie.ifTrackNewApi: info newApiTxt info, + key=key.toStr, val=val.toSeq.toStr, result=result.toStr proc contains*(trie: CoreDxTrieRefs; key: openArray[byte]): CoreDbRc[bool] = - trie.methods.containsFn key + result = trie.methods.containsFn key + trie.ifTrackNewApi: + info newApiTxt "trie/contains()", key=key.toStr, result=result.toStr iterator pairs*(mpt: CoreDxMptRef): (Blob, Blob) {.apiRaise.} = ## Trie traversal, only supported for `CoreDxMptRef` for k,v in mpt.methods.pairsIt(): yield (k,v) + mpt.ifTrackNewApi: info newApiTxt "mpt/pairs()" iterator replicate*(mpt: CoreDxMptRef): (Blob, Blob) {.apiRaise.} = ## Low level trie dump, only supported for `CoreDxMptRef` for k,v in mpt.methods.replicateIt(): yield (k,v) + mpt.ifTrackNewApi: info newApiTxt "mpt/replicate()" # ------------------------------------------------------------------------------ # Public trie database methods for accounts @@ -369,20 +517,28 @@ iterator replicate*(mpt: CoreDxMptRef): (Blob, Blob) {.apiRaise.} = proc fetch*(acc: CoreDxAccRef; address: EthAddress): CoreDbRc[CoreDbAccount] = ## Fetch data from the argument `trie` - acc.methods.fetchFn address + result = acc.methods.fetchFn address + acc.ifTrackNewApi: + info newApiTxt "acc/fetch()", address=address.toStr, result=result.toStr proc delete*(acc: CoreDxAccRef; address: EthAddress): CoreDbRc[void] = - acc.methods.deleteFn address + result = acc.methods.deleteFn address + acc.ifTrackNewApi: + info newApiTxt "acc/delete()", address=address.toStr, result=result.toStr proc merge*( acc: CoreDxAccRef; address: EthAddress; account: CoreDbAccount; ): CoreDbRc[void] = - acc.methods.mergeFn(address, account) + result = acc.methods.mergeFn(address, account) + acc.ifTrackNewApi: + info newApiTxt "acc/merge()", address=address.toStr, result=result.toStr proc contains*(acc: CoreDxAccRef; address: EthAddress): CoreDbRc[bool] = - acc.methods.containsFn address + result = acc.methods.containsFn address + acc.ifTrackNewApi: + info newApiTxt "acc/contains()", address=address.toStr, result=result.toStr # ------------------------------------------------------------------------------ # Public transaction related methods @@ -390,31 +546,37 @@ proc contains*(acc: CoreDxAccRef; address: EthAddress): CoreDbRc[bool] = proc toTransactionID*(db: CoreDbRef): CoreDbRc[CoreDxTxID] = ## Getter, current transaction state - db.methods.getIdFn() + result = db.methods.getIdFn() + db.ifTrackNewApi: info newApiTxt "toTransactionID()", result=result.toStr proc shortTimeReadOnly*( id: CoreDxTxID; action: proc() {.noRaise.}; ): CoreDbRc[void] = ## Run `action()` in an earlier transaction environment. - id.methods.roWrapperFn action - + result = id.methods.roWrapperFn action + id.ifTrackNewApi: info newApiTxt "shortTimeReadOnly()", result=result.toStr proc newTransaction*(db: CoreDbRef): CoreDbRc[CoreDxTxRef] = ## Constructor - db.methods.beginFn() + result = db.methods.beginFn() + db.ifTrackNewApi: info newApiTxt "newTransaction()", result=result.toStr proc commit*(tx: CoreDxTxRef, applyDeletes = true): CoreDbRc[void] = - tx.methods.commitFn applyDeletes + result = tx.methods.commitFn applyDeletes + tx.ifTrackNewApi: info newApiTxt "tx/commit()", result=result.toStr proc rollback*(tx: CoreDxTxRef): CoreDbRc[void] = - tx.methods.rollbackFn() + result = tx.methods.rollbackFn() + tx.ifTrackNewApi: info newApiTxt "tx/rollback()", result=result.toStr proc dispose*(tx: CoreDxTxRef): CoreDbRc[void] = - tx.methods.disposeFn() + result = tx.methods.disposeFn() + tx.ifTrackNewApi: info newApiTxt "tx/dispose()", result=result.toStr proc safeDispose*(tx: CoreDxTxRef): CoreDbRc[void] = - tx.methods.safeDisposeFn() + result = tx.methods.safeDisposeFn() + tx.ifTrackNewApi: info newApiTxt "tx/safeDispose()", result=result.toStr # ------------------------------------------------------------------------------ # Public tracer methods @@ -425,18 +587,22 @@ proc newCapture*( flags: set[CoreDbCaptFlags] = {}; ): CoreDbRc[CoreDxCaptRef] = ## Constructor - db.methods.captureFn flags + result = db.methods.captureFn flags + db.ifTrackNewApi: info newApiTxt "db/capture()", result=result.toStr -proc recorder*(db: CoreDxCaptRef): CoreDbRc[CoreDbRef] = +proc recorder*(cp: CoreDxCaptRef): CoreDbRc[CoreDbRef] = ## Getter - db.methods.recorderFn() + result = cp.methods.recorderFn() + cp.ifTrackNewApi: info newApiTxt "capt/recorder()", result=result.toStr -proc logDb*(db: CoreDxCaptRef): CoreDbRc[CoreDbRef] = - db.methods.logDbFn() +proc logDb*(cp: CoreDxCaptRef): CoreDbRc[CoreDbRef] = + result = cp.methods.logDbFn() + cp.ifTrackNewApi: info newApiTxt "capt/logDb()", result=result.toStr -proc flags*(db: CoreDxCaptRef): set[CoreDbCaptFlags] = +proc flags*(cp: CoreDxCaptRef): set[CoreDbCaptFlags] = ## Getter - db.methods.getFlagsFn() + result = cp.methods.getFlagsFn() + cp.ifTrackNewApi: info newApiTxt "capt/flags()", result=result.toStr # ------------------------------------------------------------------------------ # Public methods, legacy API @@ -444,43 +610,68 @@ proc flags*(db: CoreDxCaptRef): set[CoreDbCaptFlags] = when ProvideCoreDbLegacyAPI: - func parent*(cld: CoreDbChldRefs): CoreDbRef = + proc parent*(cld: CoreDbChldRefs): CoreDbRef = ## Getter, common method for all sub-modules - cld.distinctBase.parent() + result = cld.distinctBase.parent proc backend*(dsc: CoreDbChldRefs): auto = - dsc.distinctBase.backend + dsc.setTrackLegaApiOnly + result = dsc.distinctBase.backend + dsc.ifTrackLegaApi: info legaApiTxt "parent()" # ---------------- proc kvt*(db: CoreDbRef): CoreDbKvtRef = ## Legacy pseudo constructor, see `toKvt()` for production constructor - db.newKvt().CoreDbKvtRef + db.setTrackLegaApiOnly + result = db.newKvt().CoreDbKvtRef + db.ifTrackLegaApi: info legaApiTxt "kvt()" proc get*(kvt: CoreDbKvtRef; key: openArray[byte]): Blob = - kvt.distinctBase.get(key).expect "kvt/get()" + kvt.setTrackLegaApiOnly + const info = "kvt/get()" + result = kvt.distinctBase.get(key).expect info + kvt.ifTrackLegaApi: + info legaApiTxt info, key=key.toStr, result=result.toStr proc del*(kvt: CoreDbKvtRef; key: openArray[byte]): void = - kvt.distinctBase.del(key).expect "kvt/del()" + kvt.setTrackLegaApiOnly + const info = "kvt/del()" + kvt.distinctBase.del(key).expect info + kvt.ifTrackLegaApi: info legaApiTxt info, key=key.toStr - proc put*(db: CoreDbKvtRef; key: openArray[byte]; value: openArray[byte]) = - db.distinctBase.put(key, value).expect "kvt/put()" + proc put*(kvt: CoreDbKvtRef; key: openArray[byte]; val: openArray[byte]) = + kvt.setTrackLegaApiOnly + const info = "kvt/put()" + kvt.distinctBase.put(key, val).expect info + kvt.ifTrackLegaApi: + info legaApiTxt info, key=key.toStr, val=val.toSeq.toStr proc contains*(kvt: CoreDbKvtRef; key: openArray[byte]): bool = - kvt.distinctBase.contains(key).expect "kvt/contains()" + kvt.setTrackLegaApiOnly + const info = "kvt/contains()" + result = kvt.distinctBase.contains(key).expect info + kvt.ifTrackLegaApi: info legaApiTxt info, key=key.toStr, result iterator pairs*(kvt: CoreDbKvtRef): (Blob, Blob) {.apiRaise.} = + kvt.setTrackLegaApiOnly for k,v in kvt.distinctBase.pairs(): yield (k,v) + kvt.ifTrackLegaApi: info legaApiTxt "kvt/pairs()" # ---------------- proc toMpt*(phk: CoreDbPhkRef): CoreDbMptRef = - phk.distinctBase.toMpt.CoreDbMptRef + phk.setTrackLegaApiOnly + result = phk.distinctBase.toMpt.CoreDbMptRef + phk.ifTrackLegaApi: info legaApiTxt "phk/toMpt()" proc mptPrune*(db: CoreDbRef; root: Hash256; prune = true): CoreDbMptRef = - let vid = db.getRoot(root, createOk=true).expect "mpt/getRoot()" - db.newMpt(vid, prune).CoreDbMptRef + db.setTrackLegaApiOnly + const info = "mptPrune()" + let vid = db.getRoot(root, createOk=true).expect info + result = db.newMpt(vid, prune).CoreDbMptRef + db.ifTrackLegaApi: info legaApiTxt info, root=root.toStr, prune proc mptPrune*(db: CoreDbRef; prune = true): CoreDbMptRef = db.newMpt(CoreDbVidRef(nil), prune).CoreDbMptRef @@ -488,11 +679,16 @@ when ProvideCoreDbLegacyAPI: # ---------------- proc toPhk*(mpt: CoreDbMptRef): CoreDbPhkRef = - mpt.distinctBase.toPhk.CoreDbPhkRef + mpt.setTrackLegaApiOnly + result = mpt.distinctBase.toPhk.CoreDbPhkRef + mpt.ifTrackLegaApi: info legaApiTxt "mpt/toMpt()" proc phkPrune*(db: CoreDbRef; root: Hash256; prune = true): CoreDbPhkRef = - let vid = db.getRoot(root, createOk=true).expect "phk/getRoot()" - db.newMpt(vid, prune).toCoreDxPhkRef.CoreDbPhkRef + db.setTrackLegaApiOnly + const info = "phkPrune()" + let vid = db.getRoot(root, createOk=true).expect info + result = db.newMpt(vid, prune).toCoreDxPhkRef.CoreDbPhkRef + db.ifTrackLegaApi: info legaApiTxt info, root=root.toStr, prune proc phkPrune*(db: CoreDbRef; prune = true): CoreDbPhkRef = db.newMpt(CoreDbVidRef(nil), prune).toCoreDxPhkRef.CoreDbPhkRef @@ -500,42 +696,73 @@ when ProvideCoreDbLegacyAPI: # ---------------- proc isPruning*(trie: CoreDbTrieRefs): bool = - trie.distinctBase.isPruning() + trie.setTrackLegaApiOnly + result = trie.distinctBase.isPruning() + trie.ifTrackLegaApi: info legaApiTxt "trie/isPruning()", result proc get*(trie: CoreDbTrieRefs; key: openArray[byte]): Blob = - trie.distinctBase.fetch(key).expect "trie/get()" + trie.setTrackLegaApiOnly + const info = "trie/get()" + result = trie.distinctBase.fetch(key).expect "trie/get()" + trie.ifTrackLegaApi: + info legaApiTxt info, key=key.toStr, result=result.toStr proc del*(trie: CoreDbTrieRefs; key: openArray[byte]) = - trie.distinctBase.delete(key).expect "trie/del()" + trie.setTrackLegaApiOnly + const info = "trie/del()" + trie.distinctBase.delete(key).expect info + trie.ifTrackLegaApi: info legaApiTxt info, key=key.toStr proc put*(trie: CoreDbTrieRefs; key: openArray[byte]; val: openArray[byte]) = - trie.distinctBase.merge(key, val).expect "trie/put()" + trie.setTrackLegaApiOnly + when trie is CoreDbMptRef: + const info = "mpt/put()" + else: + const info = "phk/put()" + trie.distinctBase.merge(key, val).expect info + trie.ifTrackLegaApi: + info legaApiTxt info, key=key.toStr, val=val.toSeq.toStr proc contains*(trie: CoreDbTrieRefs; key: openArray[byte]): bool = - trie.distinctBase.contains(key).expect "trie/contains()" + trie.setTrackLegaApiOnly + const info = "trie/contains()" + result = trie.distinctBase.contains(key).expect info + trie.ifTrackLegaApi: info legaApiTxt info, key=key.toStr, result proc rootHash*(trie: CoreDbTrieRefs): Hash256 = - trie.distinctBase.rootVid().hash().expect "trie/rootHash()" + trie.setTrackLegaApiOnly + const info = "trie/rootHash()" + result = trie.distinctBase.rootVid().hash().expect info + trie.ifTrackLegaApi: info legaApiTxt info, result=result.toStr iterator pairs*(mpt: CoreDbMptRef): (Blob, Blob) {.apiRaise.} = ## Trie traversal, not supported for `CoreDbPhkRef` + mpt.setTrackLegaApiOnly for k,v in mpt.distinctBase.pairs(): yield (k,v) + mpt.ifTrackLegaApi: info legaApiTxt "mpt/pairs()" iterator replicate*(mpt: CoreDbMptRef): (Blob, Blob) {.apiRaise.} = ## Low level trie dump, not supported for `CoreDbPhkRef` + mpt.setTrackLegaApiOnly for k,v in mpt.distinctBase.replicate(): yield (k,v) + mpt.ifTrackLegaApi: info legaApiTxt "mpt/replicate()" # ---------------- proc getTransactionID*(db: CoreDbRef): CoreDbTxID = - (db.toTransactionID().expect "getTransactionID()").CoreDbTxID + db.setTrackLegaApiOnly + const info = "getTransactionID()" + result = (db.toTransactionID().expect info).CoreDbTxID + db.ifTrackLegaApi: info legaApiTxt info proc shortTimeReadOnly*( id: CoreDbTxID; action: proc() {.catchRaise.}; ) {.catchRaise.} = + id.setTrackLegaApiOnly + const info = "txId/shortTimeReadOnly()" var oops = none(ref CatchableError) proc safeFn() = try: @@ -544,7 +771,7 @@ when ProvideCoreDbLegacyAPI: oops = some(e) # Action has finished now - id.distinctBase.shortTimeReadOnly(safeFn).expect "txId/shortTimeReadOnly()" + id.distinctBase.shortTimeReadOnly(safeFn).expect info # Delayed exception if oops.isSome: @@ -553,21 +780,37 @@ when ProvideCoreDbLegacyAPI: msg = "delayed and reraised" & ", name=\"" & $e.name & "\", msg=\"" & e.msg & "\"" raise (ref TxWrapperApiError)(msg: msg) + id.ifTrackLegaApi: info legaApiTxt info proc beginTransaction*(db: CoreDbRef): CoreDbTxRef = - (db.distinctBase.newTransaction().expect "newTransaction()").CoreDbTxRef + db.setTrackLegaApiOnly + const info = "newTransaction()" + result = (db.distinctBase.newTransaction().expect info).CoreDbTxRef + db.ifTrackLegaApi: info legaApiTxt info proc commit*(tx: CoreDbTxRef, applyDeletes = true) = - tx.distinctBase.commit(applyDeletes).expect "tx/commit()" + tx.setTrackLegaApiOnly + const info = "tx/commit()" + tx.distinctBase.commit(applyDeletes).expect info + tx.ifTrackLegaApi: info legaApiTxt info proc rollback*(tx: CoreDbTxRef) = - tx.distinctBase.rollback().expect "tx/rollback()" + tx.setTrackLegaApiOnly + const info = "tx/rollback()" + tx.distinctBase.rollback().expect info + tx.ifTrackLegaApi: info legaApiTxt info proc dispose*(tx: CoreDbTxRef) = - tx.distinctBase.dispose().expect "tx/dispose()" + tx.setTrackLegaApiOnly + const info = "tx/dispose()" + tx.distinctBase.dispose().expect info + tx.ifTrackLegaApi: info legaApiTxt info proc safeDispose*(tx: CoreDbTxRef) = - tx.distinctBase.safeDispose().expect "tx/safeDispose()" + tx.setTrackLegaApiOnly + const info = "tx/safeDispose()" + tx.distinctBase.safeDispose().expect info + tx.ifTrackLegaApi: info legaApiTxt info # ---------------- @@ -575,16 +818,27 @@ when ProvideCoreDbLegacyAPI: db: CoreDbRef; flags: set[CoreDbCaptFlags] = {}; ): CoreDbCaptRef = - db.newCapture(flags).expect("db/capture()").CoreDbCaptRef + db.setTrackLegaApiOnly + const info = "db/capture()" + result = db.newCapture(flags).expect(info).CoreDbCaptRef + db.ifTrackLegaApi: info legaApiTxt info - proc recorder*(db: CoreDbCaptRef): CoreDbRef = - db.distinctBase.recorder().expect("db/recorder()") + proc recorder*(cp: CoreDbCaptRef): CoreDbRef = + cp.setTrackLegaApiOnly + const info = "capt/recorder()" + result = cp.distinctBase.recorder().expect info + cp.ifTrackLegaApi: info legaApiTxt info - proc logDb*(db: CoreDbCaptRef): CoreDbRef = - db.distinctBase.logDb().expect("db/logDb()") + proc logDb*(cp: CoreDbCaptRef): CoreDbRef = + cp.setTrackLegaApiOnly + const info = "capt/logDb()" + result = cp.distinctBase.logDb().expect info + cp.ifTrackLegaApi: info legaApiTxt info - proc flags*(db: CoreDbCaptRef): set[CoreDbCaptFlags] = - db.distinctBase.flags() + proc flags*(cp: CoreDbCaptRef): set[CoreDbCaptFlags] = + cp.setTrackLegaApiOnly + result = cp.distinctBase.flags() + cp.ifTrackLegaApi: info legaApiTxt "capt/flags()", result=result.toStr # ------------------------------------------------------------------------------ # End diff --git a/nimbus/db/core_db/base/base_desc.nim b/nimbus/db/core_db/base/base_desc.nim index 14b0d5b80..6fe98f157 100644 --- a/nimbus/db/core_db/base/base_desc.nim +++ b/nimbus/db/core_db/base/base_desc.nim @@ -24,12 +24,9 @@ type Ooops LegacyDbMemory LegacyDbPersistent - AristoDbMemory ## Memory backend emulator - AristoDbRocks ## RocksDB backend - AristoDbVoid ## No backend (to be prefered over `XxxDbMemory`) const - CoreDbPersistentTypes* = {LegacyDbPersistent, AristoDbRocks} + CoreDbPersistentTypes* = {LegacyDbPersistent} type CoreDbRc*[T] = Result[T,CoreDbErrorRef] @@ -215,7 +212,11 @@ type # -------------------------------------------------- CoreDbRef* = ref object of RootRef ## Database descriptor - dbType*: CoreDbType + dbType*: CoreDbType ## Type of database backend + trackLegaApi*: bool ## Debugging support + trackNewApi*: bool ## Debugging support + trackLedgerApi*: bool ## Debugging suggestion for subsequent ledger + localDbOnly*: bool ## Debugging, suggestion to ignore async fetch methods*: CoreDbBaseFns CoreDbErrorRef* = ref object of RootRef diff --git a/nimbus/db/core_db/memory_only.nim b/nimbus/db/core_db/memory_only.nim index c5417580f..0825b6250 100644 --- a/nimbus/db/core_db/memory_only.nim +++ b/nimbus/db/core_db/memory_only.nim @@ -42,13 +42,13 @@ export CoreDxPhkRef, CoreDxTxID, CoreDxTxRef, + `$$`, backend, beginTransaction, capture, commit, compensateLegacySetup, contains, - dbType, del, delete, dispose, @@ -58,6 +58,7 @@ export getRoot, getTransactionID, hash, + hashOrEmpty, isLegacy, isPruning, kvt, @@ -66,6 +67,7 @@ export mptPrune, newAccMpt, newCapture, + newKvt, newMpt, newTransaction, pairs, diff --git a/nimbus/db/core_db/persistent.nim b/nimbus/db/core_db/persistent.nim index e3333893d..51f8de6cb 100644 --- a/nimbus/db/core_db/persistent.nim +++ b/nimbus/db/core_db/persistent.nim @@ -32,9 +32,6 @@ proc newCoreDbRef*( when dbType == LegacyDbPersistent: newLegacyPersistentCoreDbRef path - elif dbType == AristoDbRocks: - newAristoRocksDbCoreDbRef path - else: {.error: "Unsupported dbType for persistent newCoreDbRef()".} diff --git a/nimbus/db/distinct_tries.nim b/nimbus/db/distinct_tries.nim index 131603e92..fb7ebca7c 100644 --- a/nimbus/db/distinct_tries.nim +++ b/nimbus/db/distinct_tries.nim @@ -11,8 +11,10 @@ {.push raises: [].} import + std/[algorithm, sequtils, strutils, tables], eth/[common, trie/hexary], - ./core_db + chronicles, + "."/[core_db, storage_types] type DB = CoreDbRef @@ -20,11 +22,49 @@ type StorageTrie* = distinct CoreDbPhkRef DistinctTrie* = AccountsTrie | StorageTrie +# ------------------------------------------------------------------------------ +# Private helper +# ------------------------------------------------------------------------------ + func toBase(t: DistinctTrie): CoreDbPhkRef = ## Note that `CoreDbPhkRef` is a distinct variant of `CoreDxPhkRef` for ## the legacy API. t.CoreDbPhkRef +# ------------------------------------------------------------------------------ +# Public debugging helpers +# ------------------------------------------------------------------------------ + +proc toSvp*(sl: StorageTrie): seq[(UInt256,UInt256)] = + ## Dump as slot id-value pair sequence + let + db = sl.toBase.parent + save = db.trackLegaApi + db.trackLegaApi = false + defer: db.trackLegaApi = save + let kvt = db.kvt + var kvp: Table[UInt256,UInt256] + try: + for (slotHash,val) in sl.toBase.toMpt.pairs: + if slotHash.len == 0: + kvp[high UInt256] = high UInt256 + else: + let slotRlp = kvt.get(slotHashToSlotKey(slotHash).toOpenArray) + if slotRlp.len == 0: + kvp[high UInt256] = high UInt256 + else: + kvp[rlp.decode(slotRlp,UInt256)] = rlp.decode(val,UInt256) + except CatchableError as e: + raiseAssert "Ooops(" & $e.name & "): " & e.msg + kvp.keys.toSeq.sorted.mapIt((it,kvp.getOrDefault(it,high UInt256))) + +proc toStr*(w: seq[(UInt256,UInt256)]): string = + "[" & w.mapIt("(" & it[0].toHex & "," & it[1].toHex & ")").join(", ") & "]" + +# ------------------------------------------------------------------------------ +# Public helpers +# ------------------------------------------------------------------------------ + # I don't understand why "borrow" doesn't work here. --Adam proc rootHash* (t: DistinctTrie): KeccakHash = t.toBase.rootHash() proc rootHashHex*(t: DistinctTrie): string = $t.toBase.rootHash() @@ -33,6 +73,9 @@ proc isPruning* (t: DistinctTrie): bool = t.toBase.isPruning() proc mpt* (t: DistinctTrie): CoreDbMptRef = t.toBase.toMpt() func phk* (t: DistinctTrie): CoreDbPhkRef = t.toBase +# ------------------------------------------------------------------------------ +# Public functions: accounts trie +# ------------------------------------------------------------------------------ template initAccountsTrie*(db: DB, rootHash: KeccakHash, isPruning = true): AccountsTrie = AccountsTrie(db.phkPrune(rootHash, isPruning)) @@ -56,9 +99,11 @@ proc putAccountBytes*(trie: var AccountsTrie, address: EthAddress, value: openAr proc delAccountBytes*(trie: var AccountsTrie, address: EthAddress) = CoreDbPhkRef(trie).del(address) +# ------------------------------------------------------------------------------ +# Public functions: storage trie +# ------------------------------------------------------------------------------ - -template initStorageTrie*(db: DB, rootHash: KeccakHash, isPruning = true): StorageTrie = +proc initStorageTrie*(db: DB, rootHash: KeccakHash, isPruning = true): StorageTrie = StorageTrie(db.phkPrune(rootHash, isPruning)) template initStorageTrie*(db: DB, isPruning = true): StorageTrie = @@ -96,3 +141,7 @@ proc storageTrieForAccount*(trie: AccountsTrie, account: Account, isPruning = tr # it will create virtual container for each account. # see nim-eth#9 initStorageTrie(trie.db, account.storageRoot, isPruning) + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/ledger.nim b/nimbus/db/ledger.nim index 577f66abf..e2793aee7 100644 --- a/nimbus/db/ledger.nim +++ b/nimbus/db/ledger.nim @@ -28,9 +28,30 @@ import export AccountsCache, AccountsLedgerRef, + LedgerType, base, init +# ------------------------------------------------------------------------------ +# Public constructor +# ------------------------------------------------------------------------------ + +proc init*( + ldgType: LedgerType; + db: CoreDbRef; + root: Hash256; + pruneTrie: bool; + ): LedgerRef = + case ldgType: + of LegacyAccountsCache: + result = AccountsCache.init(db, root, pruneTrie) + + of LedgerCache: + result = AccountsLedgerRef.init(db, root, pruneTrie) + + else: + raiseAssert: "Missing ledger type label" + # ------------------------------------------------------------------------------ # Public iterators # ------------------------------------------------------------------------------ diff --git a/nimbus/db/ledger/accounts_cache.nim b/nimbus/db/ledger/accounts_cache.nim index b022d39d1..45d354965 100644 --- a/nimbus/db/ledger/accounts_cache.nim +++ b/nimbus/db/ledger/accounts_cache.nim @@ -10,6 +10,7 @@ import std/[tables, hashes, sets], + chronicles, eth/[common, rlp], ../../../stateless/multi_keys, ../../constants, diff --git a/nimbus/db/ledger/accounts_ledger.nim b/nimbus/db/ledger/accounts_ledger.nim index fe3f27469..27e70c853 100644 --- a/nimbus/db/ledger/accounts_ledger.nim +++ b/nimbus/db/ledger/accounts_ledger.nim @@ -18,6 +18,7 @@ import std/[tables, hashes, sets], + chronicles, eth/[common, rlp], results, ../../../stateless/multi_keys, @@ -54,7 +55,6 @@ type codeTouched*: bool AccountsLedgerRef* = ref object - kvt: CoreDbKvtRef # Legacy API is god enough here ledger: AccountLedger savePoint: LedgerSavePoint witnessCache: Table[EthAddress, WitnessData] @@ -114,6 +114,9 @@ proc beginSavepoint*(ac: AccountsLedgerRef): LedgerSavePoint {.gcsafe.} # take this out once those are gone. proc rawTrie*(ac: AccountsLedgerRef): AccountLedger = ac.ledger +proc db(ac: AccountsLedgerRef): CoreDbRef = ac.ledger.db +proc kvt(ac: AccountsLedgerRef): CoreDbKvtRef = ac.db.kvt + func newCoreDbAccount: CoreDbAccount = CoreDbAccount( nonce: emptyAcc.nonce, @@ -131,7 +134,6 @@ template noRlpException(info: static[string]; code: untyped) = proc init*(x: typedesc[AccountsLedgerRef], db: CoreDbRef, root: KeccakHash, pruneTrie = true): AccountsLedgerRef = new result - result.kvt = db.kvt result.ledger = AccountLedger.init(db, root, pruneTrie) result.witnessCache = initTable[EthAddress, WitnessData]() discard result.beginSavepoint diff --git a/nimbus/db/ledger/backend/accounts_cache.nim b/nimbus/db/ledger/backend/accounts_cache.nim index 4f1e574f2..c82a6e334 100644 --- a/nimbus/db/ledger/backend/accounts_cache.nim +++ b/nimbus/db/ledger/backend/accounts_cache.nim @@ -183,6 +183,19 @@ proc ledgerExtras(lc: impl.AccountsCache): LedgerExtras = rawRootHashFn: proc(): Hash256 = lc.rawTrie.rootHash()) + +proc newLegacyAccountsCache( + db: CoreDbRef; + root: Hash256; + pruneTrie: bool): LedgerRef = + ## Constructor + let lc = impl.AccountsCache.init(db, root, pruneTrie) + wrp.AccountsCache( + ldgType: LegacyAccountsCache, + ac: lc, + extras: lc.ledgerExtras(), + methods: lc.ledgerMethods()).bless db + # ------------------------------------------------------------------------------ # Public iterators # ------------------------------------------------------------------------------ @@ -224,13 +237,7 @@ proc init*( db: CoreDbRef; root: Hash256; pruneTrie: bool): LedgerRef = - let lc = impl.AccountsCache.init(db, root, pruneTrie) - result = T( - ldgType: LegacyAccountsCache, - ac: lc, - extras: lc.ledgerExtras(), - methods: lc.ledgerMethods()) - result.validate + db.newLegacyAccountsCache(root, pruneTrie) # ------------------------------------------------------------------------------ # End diff --git a/nimbus/db/ledger/backend/accounts_ledger.nim b/nimbus/db/ledger/backend/accounts_ledger.nim index b267f08ae..eb8d022ed 100644 --- a/nimbus/db/ledger/backend/accounts_ledger.nim +++ b/nimbus/db/ledger/backend/accounts_ledger.nim @@ -138,10 +138,10 @@ proc ledgerMethods(lc: impl.AccountsLedgerRef): LedgerFns = safeDisposeFn: proc(sp: LedgerSpRef) = lc.safeDispose(sp.savePoint), - selfDestruct6780Fn: proc(eAddr: EthAddress) = + selfDestructFn: proc(eAddr: EthAddress) = lc.selfDestruct(eAddr), - selfDestructFn: proc(eAddr: EthAddress) = + selfDestruct6780Fn: proc(eAddr: EthAddress) = lc.selfDestruct6780(eAddr), selfDestructLenFn: proc(): int = @@ -170,6 +170,18 @@ proc ledgerExtras(lc: impl.AccountsLedgerRef): LedgerExtras = rawRootHashFn: proc(): Hash256 = lc.rawTrie.rootHash()) + +proc newAccountsLedgerRef( + db: CoreDbRef; + root: Hash256; + pruneTrie: bool): LedgerRef = + let lc = impl.AccountsLedgerRef.init(db, root, pruneTrie) + wrp.AccountsLedgerRef( + ldgType: LedgerCache, + ac: lc, + extras: lc.ledgerExtras(), + methods: lc.ledgerMethods()).bless db + # ------------------------------------------------------------------------------ # Public iterators # ------------------------------------------------------------------------------ @@ -210,13 +222,7 @@ proc init*( db: CoreDbRef; root: Hash256; pruneTrie: bool): LedgerRef = - let lc = impl.AccountsLedgerRef.init(db, root, pruneTrie) - result = T( - ldgType: LedgerCache, - ac: lc, - extras: lc.ledgerExtras(), - methods: lc.ledgerMethods()) - result.validate + db.newAccountsLedgerRef(root, pruneTrie) # ------------------------------------------------------------------------------ # End diff --git a/nimbus/db/ledger/base.nim b/nimbus/db/ledger/base.nim index 5191de726..d39940deb 100644 --- a/nimbus/db/ledger/base.nim +++ b/nimbus/db/ledger/base.nim @@ -14,7 +14,9 @@ import eth/common, + chronicles, ../../../stateless/multi_keys, + ../core_db, ./base/[base_desc, validate] type @@ -30,16 +32,57 @@ when defined(release): else: const AutoValidateDescriptors = true +const + EnableApiTracking = true and false + ## When enabled, API functions are logged. Tracking is enabled by setting + ## the `trackApi` flag to `true`. + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +when EnableApiTracking: + import std/strutils, stew/byteutils + + template apiTxt(info: static[string]): static[string] = + "Ledger API " & info + + template ifTrackApi(ldg: LedgerRef; code: untyped) = + when EnableApiTracking: + if ldg.trackApi: + code + + proc oaToStr(w: openArray[byte]): string = + w.toHex.toLowerAscii + + proc toStr(w: EthAddress): string = + w.oaToStr + + proc toStr(w: Hash256): string = + w.data.oaToStr + + proc toStr(w: Blob): string = + if 0 < w.len and w.len < 5: "<" & w.oaToStr & ">" + else: "Blob[" & $w.len & "]" + + proc toStr(w: seq[Log]): string = + "Logs[" & $w.len & "]" + +else: + template ifTrackApi(ldg: LedgerRef; code: untyped) = discard + # ------------------------------------------------------------------------------ # Public constructor helper # ------------------------------------------------------------------------------ -when AutoValidateDescriptors: - proc validate*(ldg: LedgerRef) = - validate.validate(ldg) -else: - template validate*(ldg: LedgerRef) = - discard +proc bless*(ldg: LedgerRef; db: CoreDbRef): LedgerRef = + when AutoValidateDescriptors: + ldg.validate() + when EnableApiTracking: + ldg.trackApi = db.trackLedgerApi + if ldg.trackApi: + info apiTxt "LedgerRef.init()", ldgType=ldg.ldgType + ldg # ------------------------------------------------------------------------------ # Public methods @@ -47,145 +90,197 @@ else: proc accessList*(ldg: LedgerRef, eAddr: EthAddress) = ldg.methods.accessListFn(eAddr) + ldg.ifTrackApi: info apiTxt "accessList()", eAddr=eAddr.toStr proc accessList*(ldg: LedgerRef, eAddr: EthAddress, slot: UInt256) = ldg.methods.accessList2Fn(eAddr, slot) + ldg.ifTrackApi: info apiTxt "accessList()", eAddr=eAddr.toStr, slot proc accountExists*(ldg: LedgerRef, eAddr: EthAddress): bool = - ldg.methods.accountExistsFn(eAddr) + result = ldg.methods.accountExistsFn(eAddr) + ldg.ifTrackApi: info apiTxt "accountExists()", eAddr=eAddr.toStr, result proc addBalance*(ldg: LedgerRef, eAddr: EthAddress, delta: UInt256) = ldg.methods.addBalanceFn(eAddr, delta) + ldg.ifTrackApi: info apiTxt "addBalance()", eAddr=eAddr.toStr, delta proc addLogEntry*(ldg: LedgerRef, log: Log) = ldg.methods.addLogEntryFn(log) + ldg.ifTrackApi: info apiTxt "addLogEntry()" proc beginSavepoint*(ldg: LedgerRef): LedgerSpRef = - ldg.methods.beginSavepointFn() + result = ldg.methods.beginSavepointFn() + ldg.ifTrackApi: info apiTxt "beginSavepoint()" proc clearStorage*(ldg: LedgerRef, eAddr: EthAddress) = ldg.methods.clearStorageFn(eAddr) + ldg.ifTrackApi: info apiTxt "clearStorage()", eAddr=eAddr.toStr proc clearTransientStorage*(ldg: LedgerRef) = ldg.methods.clearTransientStorageFn() + ldg.ifTrackApi: info apiTxt "clearTransientStorage()" proc collectWitnessData*(ldg: LedgerRef) = ldg.methods.collectWitnessDataFn() + ldg.ifTrackApi: info apiTxt "collectWitnessData()" proc commit*(ldg: LedgerRef, sp: LedgerSpRef) = ldg.methods.commitFn(sp) + ldg.ifTrackApi: info apiTxt "commit()" proc deleteAccount*(ldg: LedgerRef, eAddr: EthAddress) = ldg.methods.deleteAccountFn(eAddr) + ldg.ifTrackApi: info apiTxt "deleteAccount()", eAddr=eAddr.toStr proc dispose*(ldg: LedgerRef, sp: LedgerSpRef) = ldg.methods.disposeFn(sp) + ldg.ifTrackApi: info apiTxt "dispose()" proc getAndClearLogEntries*(ldg: LedgerRef): seq[Log] = - ldg.methods.getAndClearLogEntriesFn() + result = ldg.methods.getAndClearLogEntriesFn() + ldg.ifTrackApi: info apiTxt "getAndClearLogEntries()" proc getBalance*(ldg: LedgerRef, eAddr: EthAddress): UInt256 = - ldg.methods.getBalanceFn(eAddr) + result = ldg.methods.getBalanceFn(eAddr) + ldg.ifTrackApi: info apiTxt "getBalance()", eAddr=eAddr.toStr, result proc getCode*(ldg: LedgerRef, eAddr: EthAddress): Blob = - ldg.methods.getCodeFn(eAddr) + result = ldg.methods.getCodeFn(eAddr) + ldg.ifTrackApi: + info apiTxt "getCode()", eAddr=eAddr.toStr, result=result.toStr proc getCodeHash*(ldg: LedgerRef, eAddr: EthAddress): Hash256 = - ldg.methods.getCodeHashFn(eAddr) + result = ldg.methods.getCodeHashFn(eAddr) + ldg.ifTrackApi: + info apiTxt "getCodeHash()", eAddr=eAddr.toStr, result=result.toStr proc getCodeSize*(ldg: LedgerRef, eAddr: EthAddress): int = - ldg.methods.getCodeSizeFn(eAddr) + result = ldg.methods.getCodeSizeFn(eAddr) + ldg.ifTrackApi: info apiTxt "getCodeSize()", eAddr=eAddr.toStr, result proc getCommittedStorage*(ldg: LedgerRef, eAddr: EthAddress, slot: UInt256): UInt256 = - ldg.methods.getCommittedStorageFn(eAddr, slot) + result = ldg.methods.getCommittedStorageFn(eAddr, slot) + ldg.ifTrackApi: + info apiTxt "getCommittedStorage()", eAddr=eAddr.toStr, slot, result proc getNonce*(ldg: LedgerRef, eAddr: EthAddress): AccountNonce = - ldg.methods.getNonceFn(eAddr) + result = ldg.methods.getNonceFn(eAddr) + ldg.ifTrackApi: info apiTxt "getNonce()", eAddr=eAddr.toStr, result proc getStorage*(ldg: LedgerRef, eAddr: EthAddress, slot: UInt256): UInt256 = - ldg.methods.getStorageFn(eAddr, slot) + result = ldg.methods.getStorageFn(eAddr, slot) + ldg.ifTrackApi: info apiTxt "getStorage()", eAddr=eAddr.toStr, slot, result proc getStorageRoot*(ldg: LedgerRef, eAddr: EthAddress): Hash256 = - ldg.methods.getStorageRootFn(eAddr) + result = ldg.methods.getStorageRootFn(eAddr) + ldg.ifTrackApi: + info apiTxt "getStorageRoot()", eAddr=eAddr.toStr, result=result.toStr proc getTransientStorage*(ldg: LedgerRef, eAddr: EthAddress, slot: UInt256): UInt256 = - ldg.methods.getTransientStorageFn(eAddr, slot) + result = ldg.methods.getTransientStorageFn(eAddr, slot) + ldg.ifTrackApi: + info apiTxt "getTransientStorage()", eAddr=eAddr.toStr, slot, result proc hasCodeOrNonce*(ldg: LedgerRef, eAddr: EthAddress): bool = - ldg.methods.hasCodeOrNonceFn(eAddr) + result = ldg.methods.hasCodeOrNonceFn(eAddr) + ldg.ifTrackApi: info apiTxt "hasCodeOrNonce()", eAddr=eAddr.toStr, result proc inAccessList*(ldg: LedgerRef, eAddr: EthAddress): bool = - ldg.methods.inAccessListFn(eAddr) + result = ldg.methods.inAccessListFn(eAddr) + ldg.ifTrackApi: info apiTxt "inAccessList()", eAddr=eAddr.toStr, result proc inAccessList*(ldg: LedgerRef, eAddr: EthAddress, slot: UInt256): bool = - ldg.methods.inAccessList2Fn(eAddr, slot) + result = ldg.methods.inAccessList2Fn(eAddr, slot) + ldg.ifTrackApi: info apiTxt "inAccessList()", eAddr=eAddr.toStr, slot, result proc incNonce*(ldg: LedgerRef, eAddr: EthAddress) = ldg.methods.incNonceFn(eAddr) + ldg.ifTrackApi: info apiTxt "incNonce()", eAddr=eAddr.toStr proc isDeadAccount*(ldg: LedgerRef, eAddr: EthAddress): bool = - ldg.methods.isDeadAccountFn(eAddr) + result = ldg.methods.isDeadAccountFn(eAddr) + ldg.ifTrackApi: info apiTxt "isDeadAccount()", eAddr=eAddr.toStr, result proc isEmptyAccount*(ldg: LedgerRef, eAddr: EthAddress): bool = - ldg.methods.isEmptyAccountFn(eAddr) + result = ldg.methods.isEmptyAccountFn(eAddr) + ldg.ifTrackApi: info apiTxt "isEmptyAccount()", eAddr=eAddr.toStr, result proc isTopLevelClean*(ldg: LedgerRef): bool = - ldg.methods.isTopLevelCleanFn() + result = ldg.methods.isTopLevelCleanFn() + ldg.ifTrackApi: info apiTxt "isTopLevelClean()", result proc logEntries*(ldg: LedgerRef): seq[Log] = - ldg.methods.logEntriesFn() + result = ldg.methods.logEntriesFn() + ldg.ifTrackApi: info apiTxt "logEntries()", result=result.toStr proc makeMultiKeys*(ldg: LedgerRef): MultikeysRef = - ldg.methods.makeMultiKeysFn() + result = ldg.methods.makeMultiKeysFn() + ldg.ifTrackApi: info apiTxt "makeMultiKeys()" proc persist*(ldg: LedgerRef, clearEmptyAccount = false, clearCache = true) = ldg.methods.persistFn(clearEmptyAccount, clearCache) + ldg.ifTrackApi: info apiTxt "persist()", clearEmptyAccount, clearCache proc ripemdSpecial*(ldg: LedgerRef) = ldg.methods.ripemdSpecialFn() + ldg.ifTrackApi: info apiTxt "ripemdSpecial()" proc rollback*(ldg: LedgerRef, sp: LedgerSpRef) = ldg.methods.rollbackFn(sp) + ldg.ifTrackApi: info apiTxt "rollback()" proc rootHash*(ldg: LedgerRef): Hash256 = - ldg.methods.rootHashFn() + result = ldg.methods.rootHashFn() + ldg.ifTrackApi: info apiTxt "rootHash()", result=result.toStr proc safeDispose*(ldg: LedgerRef, sp: LedgerSpRef) = ldg.methods.safeDisposeFn(sp) + ldg.ifTrackApi: info apiTxt "safeDispose()" proc selfDestruct*(ldg: LedgerRef, eAddr: EthAddress) = ldg.methods.selfDestructFn(eAddr) + ldg.ifTrackApi: info apiTxt "selfDestruct()" proc selfDestruct6780*(ldg: LedgerRef, eAddr: EthAddress) = ldg.methods.selfDestruct6780Fn(eAddr) + ldg.ifTrackApi: info apiTxt "selfDestruct6780()" proc selfDestructLen*(ldg: LedgerRef): int = - ldg.methods.selfDestructLenFn() + result = ldg.methods.selfDestructLenFn() + ldg.ifTrackApi: info apiTxt "selfDestructLen()", result proc setBalance*(ldg: LedgerRef, eAddr: EthAddress, balance: UInt256) = ldg.methods.setBalanceFn(eAddr, balance) + ldg.ifTrackApi: info apiTxt "setBalance()", eAddr=eAddr.toStr, balance proc setCode*(ldg: LedgerRef, eAddr: EthAddress, code: Blob) = ldg.methods.setCodeFn(eAddr, code) + ldg.ifTrackApi: info apiTxt "setCode()", eAddr=eAddr.toStr, code=code.toStr proc setNonce*(ldg: LedgerRef, eAddr: EthAddress, nonce: AccountNonce) = ldg.methods.setNonceFn(eAddr, nonce) + ldg.ifTrackApi: info apiTxt "setNonce()", eAddr=eAddr.toStr, nonce proc setStorage*(ldg: LedgerRef, eAddr: EthAddress, slot, val: UInt256) = ldg.methods.setStorageFn(eAddr, slot, val) + ldg.ifTrackApi: info apiTxt "setStorage()", eAddr=eAddr.toStr, slot, val proc setTransientStorage*(ldg: LedgerRef, eAddr: EthAddress, slot, val: UInt256) = ldg.methods.setTransientStorageFn(eAddr, slot, val) + ldg.ifTrackApi: + info apiTxt "setTransientStorage()", eAddr=eAddr.toStr, slot, val proc subBalance*(ldg: LedgerRef, eAddr: EthAddress, delta: UInt256) = ldg.methods.subBalanceFn(eAddr, delta) + ldg.ifTrackApi: info apiTxt "setTransientStorage()", eAddr=eAddr.toStr, delta # ------------------------------------------------------------------------------ # Public methods, extensions to go away # ------------------------------------------------------------------------------ proc rawRootHash*(ldg: LedgerRef): Hash256 = - ldg.extras.rawRootHashFn() + result = ldg.extras.rawRootHashFn() + ldg.ifTrackApi: info apiTxt "rawRootHash()", result=result.toStr # ------------------------------------------------------------------------------ # Public virtual read-only methods diff --git a/nimbus/db/ledger/base/base_desc.nim b/nimbus/db/ledger/base/base_desc.nim index bf3fc3deb..8a306496f 100644 --- a/nimbus/db/ledger/base/base_desc.nim +++ b/nimbus/db/ledger/base/base_desc.nim @@ -29,6 +29,7 @@ type LedgerRef* = ref object of RootRef ## Root object with closures ldgType*: LedgerType ## For debugging + trackApi*: bool ## For debugging extras*: LedgerExtras ## Support might go away methods*: LedgerFns diff --git a/nimbus/db/ledger/distinct_ledgers.nim b/nimbus/db/ledger/distinct_ledgers.nim index 281fb87ab..8d6e14a48 100644 --- a/nimbus/db/ledger/distinct_ledgers.nim +++ b/nimbus/db/ledger/distinct_ledgers.nim @@ -10,21 +10,55 @@ {.push raises: [].} -## Re-write of `distinct_tries.nim` to be imported into `accounts_cache.nim` +## Re-write of `distinct_tries.nim` to be imported into `accounts_ledger.nim` ## for using new database API. ## import - std/typetraits, + std/[algorithm, sequtils, strutils, tables, typetraits], + chronicles, eth/common, results, - ../core_db + ".."/[core_db, storage_types] type AccountLedger* = distinct CoreDxAccRef StorageLedger* = distinct CoreDxPhkRef SomeLedger* = AccountLedger | StorageLedger +# ------------------------------------------------------------------------------ +# Public debugging helpers +# ------------------------------------------------------------------------------ + +proc toSvp*(sl: StorageLedger): seq[(UInt256,UInt256)] = + ## Dump as slot id-value pair sequence + let + db = sl.distinctBase.parent + save = db.trackNewApi + db.trackNewApi = false + defer: db.trackNewApi = save + let kvt = db.newKvt + var kvp: Table[UInt256,UInt256] + try: + for (slotHash,val) in sl.distinctBase.toMpt.pairs: + let rc = kvt.get(slotHashToSlotKey(slotHash).toOpenArray) + if rc.isErr: + warn "StorageLedger.dump()", slotHash, error=($$rc.error) + else: + kvp[rlp.decode(rc.value,UInt256)] = rlp.decode(val,UInt256) + except CatchableError as e: + raiseAssert "Ooops(" & $e.name & "): " & e.msg + kvp.keys.toSeq.sorted.mapIt((it,kvp.getOrDefault(it,high UInt256))) + +proc toStr*(w: seq[(UInt256,UInt256)]): string = + "[" & w.mapIt("(" & it[0].toHex & "," & it[1].toHex & ")").join(", ") & "]" + +# ------------------------------------------------------------------------------ +# Public helpers +# ------------------------------------------------------------------------------ + +proc db*(t: SomeLedger): CoreDbRef = + t.distinctBase.parent proc rootHash*(t: SomeLedger): Hash256 = t.distinctBase.rootVid().hash().expect "SomeLedger/rootHash()" @@ -32,6 +66,9 @@ proc rootHash*(t: SomeLedger): Hash256 = proc rootVid*(t: SomeLedger): CoreDbVidRef = t.distinctBase.rootVid +# ------------------------------------------------------------------------------ +# Public functions: accounts ledger +# ------------------------------------------------------------------------------ proc init*( T: type AccountLedger; @@ -60,17 +97,26 @@ proc merge*(al: AccountLedger; eAddr: EthAddress; account: CoreDbAccount) = proc delete*(al: AccountLedger, eAddr: EthAddress) = al.distinctBase.delete(eAddr).expect "AccountLedger/delete()" +# ------------------------------------------------------------------------------ +# Public functions: storage ledger +# ------------------------------------------------------------------------------ proc init*( T: type StorageLedger; al: AccountLedger; account: CoreDbAccount; - isPruning = true; + isPruning = false; ): T = + ## Storage trie constructor. + ## + ## Note that the argument `isPruning` should be left `false` on the legacy + ## `CoreDb` backend. Otherwise, pruning might kill some unwanted entries from + ## storage tries ending up with an unstable database leading to crashes (see + ## https://github.com/status-im/nimbus-eth1/issues/932.) al.distinctBase.parent.newMpt(account.storageVid, isPruning).toPhk.T -proc init*(T: type StorageLedger; db: CoreDbRef, isPruning = true): T = - db.newMpt(CoreDbVidRef(nil), isPruning).toPhk.T +#proc init*(T: type StorageLedger; db: CoreDbRef, isPruning = false): T = +# db.newMpt(CoreDbVidRef(nil), isPruning).toPhk.T proc fetch*(sl: StorageLedger, slot: UInt256): Result[Blob,void] = sl.distinctBase.fetch(slot.toBytesBE).mapErr proc(ign: CoreDbErrorRef)=discard @@ -90,4 +136,6 @@ iterator storage*( for (key,val) in al.distinctBase.parent.newMpt(account.storageVid).pairs: yield (key,val) +# ------------------------------------------------------------------------------ # End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/state_db.nim b/nimbus/db/state_db.nim index c2daca0f0..05f51d589 100644 --- a/nimbus/db/state_db.nim +++ b/nimbus/db/state_db.nim @@ -60,7 +60,7 @@ type proc pruneTrie*(db: AccountStateDB): bool = db.trie.isPruning -func db*(db: AccountStateDB): CoreDbRef = +proc db*(db: AccountStateDB): CoreDbRef = db.trie.db proc kvt*(db: AccountStateDB): CoreDbKvtRef = diff --git a/nimbus/evm/async/operations.nim b/nimbus/evm/async/operations.nim index beed74c87..773c83bc0 100644 --- a/nimbus/evm/async/operations.nim +++ b/nimbus/evm/async/operations.nim @@ -10,18 +10,23 @@ import proc ifNecessaryGetAccount*(vmState: BaseVMState, address: EthAddress): Future[void] {.async.} = + if vmState.com.db.localDbOnly: return await vmState.asyncFactory.ifNecessaryGetAccount(vmState.com.db, vmState.parent.blockNumber, vmState.parent.stateRoot, address, vmState.stateDB.rawTrie.rootHash) proc ifNecessaryGetCode*(vmState: BaseVMState, address: EthAddress): Future[void] {.async.} = + if vmState.com.db.localDbOnly: return await vmState.asyncFactory.ifNecessaryGetCode(vmState.com.db, vmState.parent.blockNumber, vmState.parent.stateRoot, address, vmState.stateDB.rawTrie.rootHash) proc ifNecessaryGetSlots*(vmState: BaseVMState, address: EthAddress, slots: seq[UInt256]): Future[void] {.async.} = + if vmState.com.db.localDbOnly: return await vmState.asyncFactory.ifNecessaryGetSlots(vmState.com.db, vmState.parent.blockNumber, vmState.parent.stateRoot, address, slots, vmState.stateDB.rawTrie.rootHash) proc ifNecessaryGetSlot*(vmState: BaseVMState, address: EthAddress, slot: UInt256): Future[void] {.async.} = + if vmState.com.db.localDbOnly: return await ifNecessaryGetSlots(vmState, address, @[slot]) proc ifNecessaryGetBlockHeaderByNumber*(vmState: BaseVMState, blockNumber: BlockNumber): Future[void] {.async.} = + if vmState.com.db.localDbOnly: return await vmState.asyncFactory.ifNecessaryGetBlockHeaderByNumber(vmState.com.db, blockNumber) #[ @@ -38,9 +43,11 @@ proc fetchAndPopulateNodes*(vmState: BaseVMState, paths: seq[seq[seq[byte]]], no # Sometimes it's convenient to be able to do multiple at once. proc ifNecessaryGetAccounts*(vmState: BaseVMState, addresses: seq[EthAddress]): Future[void] {.async.} = + if vmState.com.db.localDbOnly: return for address in addresses: await ifNecessaryGetAccount(vmState, address) proc ifNecessaryGetCodeForAccounts*(vmState: BaseVMState, addresses: seq[EthAddress]): Future[void] {.async.} = + if vmState.com.db.localDbOnly: return for address in addresses: await ifNecessaryGetCode(vmState, address) diff --git a/tests/replay/pp_light.nim b/tests/replay/pp_light.nim index f12aef03f..8ddf9e5e4 100644 --- a/tests/replay/pp_light.nim +++ b/tests/replay/pp_light.nim @@ -16,6 +16,7 @@ import std/[sequtils, strformat, strutils, tables, times], eth/common, + stew/byteutils, ../../nimbus/constants export @@ -106,7 +107,7 @@ proc pp*(q: openArray[int]; itemsPerLine: int; lineSep: string): string = proc pp*(a: MDigest[256]; collapse = true): string = if not collapse: - a.data.mapIt(it.toHex(2)).join.toLowerAscii + a.data.toHex.toLowerAscii elif a == ZERO_HASH256: "ZERO_HASH256" elif a == EMPTY_ROOT_HASH: @@ -118,7 +119,7 @@ proc pp*(a: MDigest[256]; collapse = true): string = elif a == ZERO_HASH256: "ZERO_HASH256" else: - a.data.mapIt(it.toHex(2)).join[56 .. 63].toLowerAscii + a.data.toHex.join[56 .. 63].toLowerAscii proc pp*(a: openArray[MDigest[256]]; collapse = true): string = "@[" & a.toSeq.mapIt(it.pp).join(" ") & "]" @@ -132,7 +133,7 @@ proc pp*(q: openArray[byte]; noHash = false): string = for n in 0..31: a[n] = q[n] MDigest[256](data: a).pp else: - q.toSeq.mapIt(it.toHex(2)).join.toLowerAscii.pp(hex = true) + q.toHex.toLowerAscii.pp(hex = true) # ------------------------------------------------------------------------------ # Elapsed time pretty printer diff --git a/tests/test_coredb.nim b/tests/test_coredb.nim index 6bc6cb70f..f6eb9f2f0 100644 --- a/tests/test_coredb.nim +++ b/tests/test_coredb.nim @@ -17,9 +17,10 @@ import eth/common, results, unittest2, - ../../nimbus/[db/core_db/persistent, core/chain], + ../../nimbus/db/[core_db/persistent, ledger], + ../../nimbus/core/chain, ./replay/pp, - ./test_coredb/[coredb_test_xx, test_legacy] + ./test_coredb/[coredb_test_xx, test_chainsync] const baseDir = [".", "..", ".."/"..", $DirSep] @@ -122,14 +123,14 @@ proc legacyRunner( suite "Legacy DB: test Core API interfaces"& &", capture={fileInfo}, {sayPersistent}": - test &"Legaci API, {numBlocksInfo} blocks": + test &"Ledger API, {numBlocksInfo} blocks": let com = openLegacyDB(persistent, dbDir, capture.network) defer: com.db.finish(flush = true) if persistent: dbDir.flushDbDir - check noisy.testChainSyncLegacyApi(filePath, com, numBlocks) + check noisy.testChainSync(filePath, com, numBlocks) # ------------------------------------------------------------------------------ # Main function(s) @@ -149,8 +150,8 @@ when isMainModule: # dumps `bulkTest2`, `bulkTest3`, .. from the `nimbus-eth1-blobs` package. # For specs see `tests/test_coredb/bulk_test_xx.nim`. var testList = @[bulkTest0] # This test is superseded by `bulkTest1` and `2` - # testList = @[failSample0] - when true: # and false: + testList = @[failSample0] + when true and false: testList = @[bulkTest2, bulkTest3] for n,capture in testList: diff --git a/tests/test_coredb/coredb_test_xx.nim b/tests/test_coredb/coredb_test_xx.nim index d48b4b136..7566ef5b3 100644 --- a/tests/test_coredb/coredb_test_xx.nim +++ b/tests/test_coredb/coredb_test_xx.nim @@ -46,9 +46,9 @@ const failSample0* = CaptureSpecs( name: "fail-goerli", - network: bulkTest0.network, - file: bulkTest0.file, - numBlocks: 18004) + network: bulkTest2.network, + file: bulkTest2.file, + numBlocks: 147_621) failSample1* = CaptureSpecs( name: "fail-main", diff --git a/tests/test_coredb/test_legacy.nim b/tests/test_coredb/test_chainsync.nim similarity index 95% rename from tests/test_coredb/test_legacy.nim rename to tests/test_coredb/test_chainsync.nim index 7d1b4d986..b91dcf488 100644 --- a/tests/test_coredb/test_legacy.nim +++ b/tests/test_coredb/test_chainsync.nim @@ -37,7 +37,7 @@ proc setErrorLevel {.used.} = # Public test function # ------------------------------------------------------------------------------ -proc test_chainSyncLegacyApi*( +proc test_chainSync*( noisy: bool; filePath: string; com: CommonRef; @@ -88,7 +88,12 @@ proc test_chainSyncLegacyApi*( xCheck runPersistBlocks1Rc == ValidationResult.OK dotsOrSpace = " " - if noisy: setTraceLevel() + if noisy: + setTraceLevel() + com.db.trackLegaApi = true + com.db.trackNewApi = true + com.db.trackLedgerApi = true + com.db.localDbOnly = true if lastOneExtra: let headers0 = headers9[0..0]