Fix default pruning for ledger and update core db and ledger logging (#1861)

* Make sure that storage tries are not pruned (by default) on the new Ledger API

why:
  Pruning might kill some unwanted entries from storage tries ending up with an unstable database
  leading to crashes.

* Implement `CoreDb` and `LedgerRef` API tracing

details:
+ Locally enabled at compile time via constants `ProvideCoreDbLegacyAPI`
  and `EnableApiTracking` in either `base.nim` source
+ If enabled it can be selectively turned on/off via public switches in
  the `CoreDb` descriptor.

* Allow suppressing opportunistic `ifNecessaryGetXxx()` functions

why:
  Better troubleshooting when the system crashes (assertions will then
  most probably happen outside an `async` function.)
This commit is contained in:
Jordan Hrycaj 2023-10-25 15:03:09 +01:00 committed by GitHub
parent 2a76975f6a
commit 3198ad1bbd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 677 additions and 184 deletions

View File

@ -208,8 +208,7 @@ proc mptMethods(mpt: HexaryChildDbRef; db: LegacyDbRef): CoreDbMptFns =
fetchFn: proc(k: openArray[byte]): CoreDbRc[Blob] =
db.mapRlpException("legacy/mpt/get()"):
return ok(mpt.trie.get(k))
,
return ok(mpt.trie.get(k)),
deleteFn: proc(k: openArray[byte]): CoreDbRc[void] =
db.mapRlpException("legacy/mpt/del()"):
@ -223,8 +222,7 @@ proc mptMethods(mpt: HexaryChildDbRef; db: LegacyDbRef): CoreDbMptFns =
containsFn: proc(k: openArray[byte]): CoreDbRc[bool] =
db.mapRlpException("legacy/mpt/put()"):
return ok(mpt.trie.contains(k))
,
return ok(mpt.trie.contains(k)),
rootVidFn: proc(): CoreDbVidRef =
db.bless(LegacyCoreDbVid(vHash: mpt.trie.rootHash)),
@ -252,8 +250,7 @@ proc accMethods(mpt: HexaryChildDbRef; db: LegacyDbRef): CoreDbAccFns =
fetchFn: proc(k: EthAddress): CoreDbRc[CoreDbAccount] =
const info = "legacy/mpt/getAccount()"
db.mapRlpException info:
return ok mpt.trie.get(k.keccakHash.data).toCoreDbAccount(db)
,
return ok mpt.trie.get(k.keccakHash.data).toCoreDbAccount(db),
deleteFn: proc(k: EthAddress): CoreDbRc[void] =
db.mapRlpException("legacy/mpt/del()"):
@ -267,8 +264,7 @@ proc accMethods(mpt: HexaryChildDbRef; db: LegacyDbRef): CoreDbAccFns =
containsFn: proc(k: EthAddress): CoreDbRc[bool] =
db.mapRlpException("legacy/mpt/put()"):
return ok(mpt.trie.contains k.keccakHash.data)
,
return ok(mpt.trie.contains k.keccakHash.data),
rootVidFn: proc(): CoreDbVidRef =
db.bless(LegacyCoreDbVid(vHash: mpt.trie.rootHash)),
@ -327,8 +323,7 @@ proc baseMethods(
destroyFn: proc(ignore: bool) =
if not closeDb.isNil:
closeDb()
,
closeDb(),
vidHashFn: proc(vid: CoreDbVidRef): Result[Hash256,void] =
ok(vid.lvHash),

View File

@ -11,7 +11,7 @@
{.push raises: [].}
import
std/[options, typetraits],
std/typetraits,
chronicles,
eth/common,
results,
@ -39,9 +39,6 @@ export
CoreDxTxID,
CoreDxTxRef
logScope:
topics = "core_db-base"
when defined(release):
const AutoValidateDescriptors = false
else:
@ -50,6 +47,11 @@ else:
const
ProvideCoreDbLegacyAPI = true
EnableApiTracking = true and false
## When enabled, functions using this tracking facility need to import
## `chronicles`, as well. Tracking is enabled by setting the `trackLegaApi`
## and/or the `trackNewApi` flags to `true`.
# Annotation helpers
{.pragma: noRaise, gcsafe, raises: [].}
{.pragma: apiRaise, gcsafe, raises: [CoreDbApiError].}
@ -85,12 +87,12 @@ type
CoreDbMptBackendRef | CoreDbAccBackendRef
## Shortcut, all backend descriptors.
CoreDxChldRefs = CoreDxKvtRef | CoreDxTrieRelated | CoreDbBackends |
CoreDbErrorRef
CoreDxChldRefs = CoreDxKvtRef | CoreDxTrieRelated | CoreDbVidRef |
CoreDbBackends | CoreDbErrorRef
## Shortcut, all descriptors with a `parent` entry.
# ------------------------------------------------------------------------------
# Private functions: helpers
# Private helpers
# ------------------------------------------------------------------------------
template logTxt(info: static[string]): static[string] =
@ -101,6 +103,99 @@ template itNotImplemented(db: CoreDbRef, name: string) =
# ---------
when EnableApiTracking:
import std/[sequtils, strutils], stew/byteutils
template newApiTxt(info: static[string]): static[string] =
logTxt "new API " & info
template legaApiTxt(info: static[string]): static[string] =
logTxt "legacy API " & info
func getParent(w: CoreDxChldRefs): auto =
## Avoida inifinite call to `parent()` in `ifTrack*Api()` tmplates
w.parent
template setTrackLegaApiOnly(w: CoreDbChldRefs|CoreDbRef) =
when typeof(w) is CoreDbRef:
let db = w
else:
let db = w.distinctBase.getParent
let save = db.trackNewApi
# Prevent from cascaded logging
db.trackNewApi = false
defer: db.trackNewApi = save
template ifTrackLegaApi(w: CoreDbChldRefs|CoreDbRef; code: untyped) =
block:
when typeof(w) is CoreDbRef:
let db = w
else:
let db = w.distinctBase.getParent
if db.trackLegaApi:
code
template ifTrackNewApi(w: CoreDxChldRefs|CoreDbRef; code: untyped) =
block:
when typeof(w) is CoreDbRef:
let db = w
else:
if w.isNil: break
let db = w.getParent
if db.trackNewApi:
code
proc oaToStr(w: openArray[byte]): string =
w.toHex.toLowerAscii
proc toStr(w: Hash256): string =
if w == EMPTY_ROOT_HASH: "EMPTY_ROOT_HASH" else: w.data.oaToStr
proc toStr(p: CoreDbVidRef): string =
if p.isNil: "vidRef(nil)" else: "vidRef"
proc toStr(w: Blob): string =
if 0 < w.len and w.len < 5: "<" & w.oaToStr & ">"
else: "Blob[" & $w.len & "]"
proc toStr(w: openArray[byte]): string =
w.oaToStr
proc toStr(w: set[CoreDbCaptFlags]): string =
"Flags[" & $w.len & "]"
proc toStr(rc: CoreDbRc[bool]): string =
if rc.isOk: "ok(" & $rc.value & ")" else: "err(..)"
proc toStr(rc: CoreDbRc[void]): string =
if rc.isOk: "ok()" else:"err()"
proc toStr(rc: CoreDbRc[Blob]): string =
if rc.isOk: "ok(Blob[" & $rc.value.len & "])" else: "err(..)"
proc toStr(rc: Result[Hash256,void]): string =
if rc.isOk: "ok(" & rc.value.toStr & ")" else: "err()"
proc toStr(rc: Result[Account,void]): string =
if rc.isOk: "ok(Account)" else: "err()"
proc toStr[T](rc: CoreDbRc[T]; ifOk: static[string]): string =
if rc.isOk: "ok(" & ifOk & ")" else: "err(..)"
proc toStr(rc: CoreDbRc[CoreDbRef]): string = rc.toStr "dbRef"
proc toStr(rc: CoreDbRc[CoreDbVidRef]): string = rc.toStr "vidRef"
proc toStr(rc: CoreDbRc[CoreDbAccount]): string = rc.toStr "accRef"
proc toStr(rc: CoreDbRc[CoreDxTxID]): string = rc.toStr "txId"
proc toStr(rc: CoreDbRc[CoreDxTxRef]): string = rc.toStr "txRef"
proc toStr(rc: CoreDbRc[CoreDxCaptRef]): string = rc.toStr "captRef"
else:
template setTrackLegaApiOnly(w: CoreDbChldRefs|CoreDbRef) = discard
template ifTrackLegaApi(w: CoreDbChldRefs|CoreDbRef; code: untyped) = discard
template ifTrackNewApi(w: CoreDxChldRefs|CoreDbRef; code: untyped) = discard
# ---------
func toCoreDxPhkRef(mpt: CoreDxMptRef): CoreDxPhkRef =
## MPT => pre-hashed MPT (aka PHK)
result = CoreDxPhkRef(
@ -125,11 +220,11 @@ func toCoreDxPhkRef(mpt: CoreDxMptRef): CoreDxPhkRef =
result.methods.pairsIt =
iterator(): (Blob, Blob) {.apiRaise.} =
mpt.parent.itNotImplemented("pairs/phk")
mpt.parent.itNotImplemented("phk/pairs()")
result.methods.replicateIt =
iterator(): (Blob, Blob) {.apiRaise.} =
mpt.parent.itNotImplemented("replicate/phk")
mpt.parent.itNotImplemented("phk/replicate()")
when AutoValidateDescriptors:
result.validate
@ -146,6 +241,7 @@ proc bless*(db: CoreDbRef): CoreDbRef =
## Verify descriptor
when AutoValidateDescriptors:
db.validate
db.ifTrackNewApi: info newApiTxt "CoreDbRef.init()", dbType=db.dbType
db
proc bless*(db: CoreDbRef; child: CoreDbVidRef): CoreDbVidRef =
@ -187,21 +283,24 @@ proc bless*[T: CoreDxTrieRelated | CoreDbErrorRef | CoreDbBackends](
proc dbType*(db: CoreDbRef): CoreDbType =
## Getter
db.dbType
result = db.dbType
db.ifTrackNewApi: info newApiTxt "dbType()", result
proc compensateLegacySetup*(db: CoreDbRef) =
## On the persistent legacy hexary trie, this function is needed for
## bootstrapping and Genesis setup when the `purge` flag is activated.
## Otherwise the database backend may defect on an internal inconsistency.
db.methods.legacySetupFn()
db.ifTrackNewApi: info newApiTxt "compensateLegacySetup()"
func parent*(cld: CoreDxChldRefs): CoreDbRef =
proc parent*(cld: CoreDxChldRefs): CoreDbRef =
## Getter, common method for all sub-modules
cld.parent
result = cld.parent
proc backend*(dsc: CoreDxKvtRef | CoreDxTrieRelated | CoreDbRef): auto =
## Getter, retrieves the *raw* backend object for special/localised support.
dsc.methods.backendFn()
result = dsc.methods.backendFn()
dsc.ifTrackNewApi: info newApiTxt "backend()"
proc finish*(db: CoreDbRef; flush = false) =
## Database destructor. If the argument `flush` is set `false`, the database
@ -211,11 +310,13 @@ proc finish*(db: CoreDbRef; flush = false) =
## depends on the backend database. Currently, only the `AristoDbRocks` type
## backend removes the database on `true`.
db.methods.destroyFn flush
db.ifTrackNewApi: info newApiTxt "finish()"
proc `$$`*(e: CoreDbErrorRef): string =
## Pretty print error symbol, note that this directive may have side effects
## as it calls a backend function.
e.parent.methods.errorPrintFn(e)
result = e.parent.methods.errorPrintFn(e)
e.ifTrackNewApi: info newApiTxt "$$()", result
proc hash*(vid: CoreDbVidRef): Result[Hash256,void] =
## Getter (well, sort of), retrieves the hash for a `vid` argument. The
@ -226,9 +327,17 @@ proc hash*(vid: CoreDbVidRef): Result[Hash256,void] =
## The value `EMPTY_ROOT_HASH` is also returned on an empty `vid` argument
## `CoreDbVidRef(nil)`, say.
##
result = block:
if not vid.isNil and vid.ready:
return vid.parent.methods.vidHashFn vid
vid.parent.methods.vidHashFn vid
else:
ok EMPTY_ROOT_HASH
# Note: tracker will be silent if `vid` is NIL
vid.ifTrackNewApi: info newApiTxt "hash()", result=result.toStr
proc hashOrEmpty*(vid: CoreDbVidRef): Hash256 =
## Convenience wrapper, returns `EMPTY_ROOT_HASH` where `hash()` would fail.
vid.hash.valueOr: EMPTY_ROOT_HASH
proc recast*(account: CoreDbAccount): Result[Account,void] =
## Convert the argument `account` to the portable Ethereum representation
@ -237,11 +346,18 @@ proc recast*(account: CoreDbAccount): Result[Account,void] =
##
## Note that for the legacy backend, this function always succeeds.
##
let vid = account.storageVid
result = block:
let rc = vid.hash
if rc.isOk:
ok Account(
nonce: account.nonce,
balance: account.balance,
codeHash: account.codeHash,
storageRoot: ? account.storageVid.hash)
storageRoot: rc.value)
else:
err()
vid.ifTrackNewApi: info newApiTxt "recast()", result=result.toStr
proc getRoot*(
db: CoreDbRef;
@ -261,7 +377,9 @@ proc getRoot*(
## return
## db.newAccMpt root
##
db.methods.getRootFn(root, createOk)
result = db.methods.getRootFn(root, createOk)
db.ifTrackNewApi:
info newApiTxt "getRoot()", root=root.toStr, result=result.toStr
# ------------------------------------------------------------------------------
# Public key-value table methods
@ -269,28 +387,38 @@ proc getRoot*(
proc newKvt*(db: CoreDbRef): CoreDxKvtRef =
## Getter (pseudo constructor)
db.methods.newKvtFn()
result = db.methods.newKvtFn()
db.ifTrackNewApi: info newApiTxt "newKvt()"
proc get*(kvt: CoreDxKvtRef; key: openArray[byte]): CoreDbRc[Blob] =
kvt.methods.getFn key
result = kvt.methods.getFn key
kvt.ifTrackNewApi:
info newApiTxt "kvt/get()", key=key.toStr, result=result.toStr
proc del*(kvt: CoreDxKvtRef; key: openArray[byte]): CoreDbRc[void] =
kvt.methods.delFn key
result = kvt.methods.delFn key
kvt.ifTrackNewApi:
info newApiTxt "kvt/del()", key=key.toStr, result=result.toStr
proc put*(
kvt: CoreDxKvtRef;
key: openArray[byte];
value: openArray[byte];
val: openArray[byte];
): CoreDbRc[void] =
kvt.methods.putFn(key, value)
result = kvt.methods.putFn(key, val)
kvt.ifTrackNewApi: info newApiTxt "kvt/put()",
key=key.toStr, val=val.toSeq.toStr, result=result.toStr
proc contains*(kvt: CoreDxKvtRef; key: openArray[byte]): CoreDbRc[bool] =
kvt.methods.containsFn key
result = kvt.methods.containsFn key
kvt.ifTrackNewApi:
info newApiTxt "kvt/contains()", key=key.toStr, result=result.toStr
iterator pairs*(kvt: CoreDxKvtRef): (Blob, Blob) {.apiRaise.} =
## Iterator supported on memory DB (otherwise implementation dependent)
for k,v in kvt.methods.pairsIt():
yield (k,v)
kvt.ifTrackNewApi: info newApiTxt "kvt/pairs()"
# ------------------------------------------------------------------------------
# Public Merkle Patricia Tree, hexary trie constructors
@ -299,26 +427,30 @@ iterator pairs*(kvt: CoreDxKvtRef): (Blob, Blob) {.apiRaise.} =
proc newMpt*(db: CoreDbRef; root: CoreDbVidRef; prune = true): CoreDxMptRef =
## Constructor, will defect on failure (note that the legacy backend
## always succeeds)
db.methods.newMptFn(root, prune).valueOr: raiseAssert $$error
result = db.methods.newMptFn(root, prune).valueOr: raiseAssert $$error
db.ifTrackNewApi: info newApiTxt "newMpt", root=root.toStr, prune
proc newAccMpt*(db: CoreDbRef; root: CoreDbVidRef; prune = true): CoreDxAccRef =
## Similar to `newMpt()` for handling accounts. Although this sub-trie can
## be emulated by means of `newMpt(..).toPhk()`, it is recommended using
## this constructor which implies its own subset of methods to handle that
## trie.
db.methods.newAccFn(root, prune).valueOr: raiseAssert $$error
result = db.methods.newAccFn(root, prune).valueOr: raiseAssert $$error
db.ifTrackNewApi: info newApiTxt "newAccMpt", root=root.toStr, prune
proc toMpt*(phk: CoreDxPhkRef): CoreDxMptRef =
## Replaces the pre-hashed argument trie `phk` by the non pre-hashed *MPT*.
## Note that this does not apply to an accounts trie that was created by
## `newAccMpt()`.
phk.fromMpt
result = phk.fromMpt
phk.ifTrackNewApi: info newApiTxt "phk/toMpt()"
proc toPhk*(mpt: CoreDxMptRef): CoreDxPhkRef =
## Replaces argument `mpt` by a pre-hashed *MPT*.
## Note that this does not apply to an accounts trie that was created by
## `newAaccMpt()`.
mpt.toCoreDxPhkRef
result = mpt.toCoreDxPhkRef
mpt.ifTrackNewApi: info newApiTxt "mpt/toPhk()"
# ------------------------------------------------------------------------------
# Public common methods for all hexary trie databases (`mpt`, `phk`, or `acc`)
@ -326,11 +458,13 @@ proc toPhk*(mpt: CoreDxMptRef): CoreDxPhkRef =
proc isPruning*(dsc: CoreDxTrieRefs | CoreDxAccRef): bool =
## Getter
dsc.methods.isPruningFn()
result = dsc.methods.isPruningFn()
dsc.ifTrackNewApi: info newApiTxt "isPruning()", result
proc rootVid*(dsc: CoreDxTrieRefs | CoreDxAccRef): CoreDbVidRef =
## Getter, result is not `nil`
dsc.methods.rootVidFn()
result = dsc.methods.rootVidFn()
dsc.ifTrackNewApi: info newApiTxt "rootVid()", result=result.toStr
# ------------------------------------------------------------------------------
# Public generic hexary trie database methods (`mpt` or `phk`)
@ -338,30 +472,44 @@ proc rootVid*(dsc: CoreDxTrieRefs | CoreDxAccRef): CoreDbVidRef =
proc fetch*(trie: CoreDxTrieRefs; key: openArray[byte]): CoreDbRc[Blob] =
## Fetch data from the argument `trie`
trie.methods.fetchFn(key)
result = trie.methods.fetchFn(key)
trie.ifTrackNewApi:
info newApiTxt "trie/fetch()", key=key.toStr, result=result.toStr
proc delete*(trie: CoreDxTrieRefs; key: openArray[byte]): CoreDbRc[void] =
trie.methods.deleteFn key
result = trie.methods.deleteFn key
trie.ifTrackNewApi:
info newApiTxt "trie/delete()", key=key.toStr, result=result.toStr
proc merge*(
trie: CoreDxTrieRefs;
key: openArray[byte];
value: openArray[byte];
val: openArray[byte];
): CoreDbRc[void] =
trie.methods.mergeFn(key, value)
when trie is CoreDbMptRef:
const info = "mpt/merge()"
else:
const info = "phk/merge()"
result = trie.methods.mergeFn(key, val)
trie.ifTrackNewApi: info newApiTxt info,
key=key.toStr, val=val.toSeq.toStr, result=result.toStr
proc contains*(trie: CoreDxTrieRefs; key: openArray[byte]): CoreDbRc[bool] =
trie.methods.containsFn key
result = trie.methods.containsFn key
trie.ifTrackNewApi:
info newApiTxt "trie/contains()", key=key.toStr, result=result.toStr
iterator pairs*(mpt: CoreDxMptRef): (Blob, Blob) {.apiRaise.} =
## Trie traversal, only supported for `CoreDxMptRef`
for k,v in mpt.methods.pairsIt():
yield (k,v)
mpt.ifTrackNewApi: info newApiTxt "mpt/pairs()"
iterator replicate*(mpt: CoreDxMptRef): (Blob, Blob) {.apiRaise.} =
## Low level trie dump, only supported for `CoreDxMptRef`
for k,v in mpt.methods.replicateIt():
yield (k,v)
mpt.ifTrackNewApi: info newApiTxt "mpt/replicate()"
# ------------------------------------------------------------------------------
# Public trie database methods for accounts
@ -369,20 +517,28 @@ iterator replicate*(mpt: CoreDxMptRef): (Blob, Blob) {.apiRaise.} =
proc fetch*(acc: CoreDxAccRef; address: EthAddress): CoreDbRc[CoreDbAccount] =
## Fetch data from the argument `trie`
acc.methods.fetchFn address
result = acc.methods.fetchFn address
acc.ifTrackNewApi:
info newApiTxt "acc/fetch()", address=address.toStr, result=result.toStr
proc delete*(acc: CoreDxAccRef; address: EthAddress): CoreDbRc[void] =
acc.methods.deleteFn address
result = acc.methods.deleteFn address
acc.ifTrackNewApi:
info newApiTxt "acc/delete()", address=address.toStr, result=result.toStr
proc merge*(
acc: CoreDxAccRef;
address: EthAddress;
account: CoreDbAccount;
): CoreDbRc[void] =
acc.methods.mergeFn(address, account)
result = acc.methods.mergeFn(address, account)
acc.ifTrackNewApi:
info newApiTxt "acc/merge()", address=address.toStr, result=result.toStr
proc contains*(acc: CoreDxAccRef; address: EthAddress): CoreDbRc[bool] =
acc.methods.containsFn address
result = acc.methods.containsFn address
acc.ifTrackNewApi:
info newApiTxt "acc/contains()", address=address.toStr, result=result.toStr
# ------------------------------------------------------------------------------
# Public transaction related methods
@ -390,31 +546,37 @@ proc contains*(acc: CoreDxAccRef; address: EthAddress): CoreDbRc[bool] =
proc toTransactionID*(db: CoreDbRef): CoreDbRc[CoreDxTxID] =
## Getter, current transaction state
db.methods.getIdFn()
result = db.methods.getIdFn()
db.ifTrackNewApi: info newApiTxt "toTransactionID()", result=result.toStr
proc shortTimeReadOnly*(
id: CoreDxTxID;
action: proc() {.noRaise.};
): CoreDbRc[void] =
## Run `action()` in an earlier transaction environment.
id.methods.roWrapperFn action
result = id.methods.roWrapperFn action
id.ifTrackNewApi: info newApiTxt "shortTimeReadOnly()", result=result.toStr
proc newTransaction*(db: CoreDbRef): CoreDbRc[CoreDxTxRef] =
## Constructor
db.methods.beginFn()
result = db.methods.beginFn()
db.ifTrackNewApi: info newApiTxt "newTransaction()", result=result.toStr
proc commit*(tx: CoreDxTxRef, applyDeletes = true): CoreDbRc[void] =
tx.methods.commitFn applyDeletes
result = tx.methods.commitFn applyDeletes
tx.ifTrackNewApi: info newApiTxt "tx/commit()", result=result.toStr
proc rollback*(tx: CoreDxTxRef): CoreDbRc[void] =
tx.methods.rollbackFn()
result = tx.methods.rollbackFn()
tx.ifTrackNewApi: info newApiTxt "tx/rollback()", result=result.toStr
proc dispose*(tx: CoreDxTxRef): CoreDbRc[void] =
tx.methods.disposeFn()
result = tx.methods.disposeFn()
tx.ifTrackNewApi: info newApiTxt "tx/dispose()", result=result.toStr
proc safeDispose*(tx: CoreDxTxRef): CoreDbRc[void] =
tx.methods.safeDisposeFn()
result = tx.methods.safeDisposeFn()
tx.ifTrackNewApi: info newApiTxt "tx/safeDispose()", result=result.toStr
# ------------------------------------------------------------------------------
# Public tracer methods
@ -425,18 +587,22 @@ proc newCapture*(
flags: set[CoreDbCaptFlags] = {};
): CoreDbRc[CoreDxCaptRef] =
## Constructor
db.methods.captureFn flags
result = db.methods.captureFn flags
db.ifTrackNewApi: info newApiTxt "db/capture()", result=result.toStr
proc recorder*(db: CoreDxCaptRef): CoreDbRc[CoreDbRef] =
proc recorder*(cp: CoreDxCaptRef): CoreDbRc[CoreDbRef] =
## Getter
db.methods.recorderFn()
result = cp.methods.recorderFn()
cp.ifTrackNewApi: info newApiTxt "capt/recorder()", result=result.toStr
proc logDb*(db: CoreDxCaptRef): CoreDbRc[CoreDbRef] =
db.methods.logDbFn()
proc logDb*(cp: CoreDxCaptRef): CoreDbRc[CoreDbRef] =
result = cp.methods.logDbFn()
cp.ifTrackNewApi: info newApiTxt "capt/logDb()", result=result.toStr
proc flags*(db: CoreDxCaptRef): set[CoreDbCaptFlags] =
proc flags*(cp: CoreDxCaptRef): set[CoreDbCaptFlags] =
## Getter
db.methods.getFlagsFn()
result = cp.methods.getFlagsFn()
cp.ifTrackNewApi: info newApiTxt "capt/flags()", result=result.toStr
# ------------------------------------------------------------------------------
# Public methods, legacy API
@ -444,43 +610,68 @@ proc flags*(db: CoreDxCaptRef): set[CoreDbCaptFlags] =
when ProvideCoreDbLegacyAPI:
func parent*(cld: CoreDbChldRefs): CoreDbRef =
proc parent*(cld: CoreDbChldRefs): CoreDbRef =
## Getter, common method for all sub-modules
cld.distinctBase.parent()
result = cld.distinctBase.parent
proc backend*(dsc: CoreDbChldRefs): auto =
dsc.distinctBase.backend
dsc.setTrackLegaApiOnly
result = dsc.distinctBase.backend
dsc.ifTrackLegaApi: info legaApiTxt "parent()"
# ----------------
proc kvt*(db: CoreDbRef): CoreDbKvtRef =
## Legacy pseudo constructor, see `toKvt()` for production constructor
db.newKvt().CoreDbKvtRef
db.setTrackLegaApiOnly
result = db.newKvt().CoreDbKvtRef
db.ifTrackLegaApi: info legaApiTxt "kvt()"
proc get*(kvt: CoreDbKvtRef; key: openArray[byte]): Blob =
kvt.distinctBase.get(key).expect "kvt/get()"
kvt.setTrackLegaApiOnly
const info = "kvt/get()"
result = kvt.distinctBase.get(key).expect info
kvt.ifTrackLegaApi:
info legaApiTxt info, key=key.toStr, result=result.toStr
proc del*(kvt: CoreDbKvtRef; key: openArray[byte]): void =
kvt.distinctBase.del(key).expect "kvt/del()"
kvt.setTrackLegaApiOnly
const info = "kvt/del()"
kvt.distinctBase.del(key).expect info
kvt.ifTrackLegaApi: info legaApiTxt info, key=key.toStr
proc put*(db: CoreDbKvtRef; key: openArray[byte]; value: openArray[byte]) =
db.distinctBase.put(key, value).expect "kvt/put()"
proc put*(kvt: CoreDbKvtRef; key: openArray[byte]; val: openArray[byte]) =
kvt.setTrackLegaApiOnly
const info = "kvt/put()"
kvt.distinctBase.put(key, val).expect info
kvt.ifTrackLegaApi:
info legaApiTxt info, key=key.toStr, val=val.toSeq.toStr
proc contains*(kvt: CoreDbKvtRef; key: openArray[byte]): bool =
kvt.distinctBase.contains(key).expect "kvt/contains()"
kvt.setTrackLegaApiOnly
const info = "kvt/contains()"
result = kvt.distinctBase.contains(key).expect info
kvt.ifTrackLegaApi: info legaApiTxt info, key=key.toStr, result
iterator pairs*(kvt: CoreDbKvtRef): (Blob, Blob) {.apiRaise.} =
kvt.setTrackLegaApiOnly
for k,v in kvt.distinctBase.pairs():
yield (k,v)
kvt.ifTrackLegaApi: info legaApiTxt "kvt/pairs()"
# ----------------
proc toMpt*(phk: CoreDbPhkRef): CoreDbMptRef =
phk.distinctBase.toMpt.CoreDbMptRef
phk.setTrackLegaApiOnly
result = phk.distinctBase.toMpt.CoreDbMptRef
phk.ifTrackLegaApi: info legaApiTxt "phk/toMpt()"
proc mptPrune*(db: CoreDbRef; root: Hash256; prune = true): CoreDbMptRef =
let vid = db.getRoot(root, createOk=true).expect "mpt/getRoot()"
db.newMpt(vid, prune).CoreDbMptRef
db.setTrackLegaApiOnly
const info = "mptPrune()"
let vid = db.getRoot(root, createOk=true).expect info
result = db.newMpt(vid, prune).CoreDbMptRef
db.ifTrackLegaApi: info legaApiTxt info, root=root.toStr, prune
proc mptPrune*(db: CoreDbRef; prune = true): CoreDbMptRef =
db.newMpt(CoreDbVidRef(nil), prune).CoreDbMptRef
@ -488,11 +679,16 @@ when ProvideCoreDbLegacyAPI:
# ----------------
proc toPhk*(mpt: CoreDbMptRef): CoreDbPhkRef =
mpt.distinctBase.toPhk.CoreDbPhkRef
mpt.setTrackLegaApiOnly
result = mpt.distinctBase.toPhk.CoreDbPhkRef
mpt.ifTrackLegaApi: info legaApiTxt "mpt/toMpt()"
proc phkPrune*(db: CoreDbRef; root: Hash256; prune = true): CoreDbPhkRef =
let vid = db.getRoot(root, createOk=true).expect "phk/getRoot()"
db.newMpt(vid, prune).toCoreDxPhkRef.CoreDbPhkRef
db.setTrackLegaApiOnly
const info = "phkPrune()"
let vid = db.getRoot(root, createOk=true).expect info
result = db.newMpt(vid, prune).toCoreDxPhkRef.CoreDbPhkRef
db.ifTrackLegaApi: info legaApiTxt info, root=root.toStr, prune
proc phkPrune*(db: CoreDbRef; prune = true): CoreDbPhkRef =
db.newMpt(CoreDbVidRef(nil), prune).toCoreDxPhkRef.CoreDbPhkRef
@ -500,42 +696,73 @@ when ProvideCoreDbLegacyAPI:
# ----------------
proc isPruning*(trie: CoreDbTrieRefs): bool =
trie.distinctBase.isPruning()
trie.setTrackLegaApiOnly
result = trie.distinctBase.isPruning()
trie.ifTrackLegaApi: info legaApiTxt "trie/isPruning()", result
proc get*(trie: CoreDbTrieRefs; key: openArray[byte]): Blob =
trie.distinctBase.fetch(key).expect "trie/get()"
trie.setTrackLegaApiOnly
const info = "trie/get()"
result = trie.distinctBase.fetch(key).expect "trie/get()"
trie.ifTrackLegaApi:
info legaApiTxt info, key=key.toStr, result=result.toStr
proc del*(trie: CoreDbTrieRefs; key: openArray[byte]) =
trie.distinctBase.delete(key).expect "trie/del()"
trie.setTrackLegaApiOnly
const info = "trie/del()"
trie.distinctBase.delete(key).expect info
trie.ifTrackLegaApi: info legaApiTxt info, key=key.toStr
proc put*(trie: CoreDbTrieRefs; key: openArray[byte]; val: openArray[byte]) =
trie.distinctBase.merge(key, val).expect "trie/put()"
trie.setTrackLegaApiOnly
when trie is CoreDbMptRef:
const info = "mpt/put()"
else:
const info = "phk/put()"
trie.distinctBase.merge(key, val).expect info
trie.ifTrackLegaApi:
info legaApiTxt info, key=key.toStr, val=val.toSeq.toStr
proc contains*(trie: CoreDbTrieRefs; key: openArray[byte]): bool =
trie.distinctBase.contains(key).expect "trie/contains()"
trie.setTrackLegaApiOnly
const info = "trie/contains()"
result = trie.distinctBase.contains(key).expect info
trie.ifTrackLegaApi: info legaApiTxt info, key=key.toStr, result
proc rootHash*(trie: CoreDbTrieRefs): Hash256 =
trie.distinctBase.rootVid().hash().expect "trie/rootHash()"
trie.setTrackLegaApiOnly
const info = "trie/rootHash()"
result = trie.distinctBase.rootVid().hash().expect info
trie.ifTrackLegaApi: info legaApiTxt info, result=result.toStr
iterator pairs*(mpt: CoreDbMptRef): (Blob, Blob) {.apiRaise.} =
## Trie traversal, not supported for `CoreDbPhkRef`
mpt.setTrackLegaApiOnly
for k,v in mpt.distinctBase.pairs():
yield (k,v)
mpt.ifTrackLegaApi: info legaApiTxt "mpt/pairs()"
iterator replicate*(mpt: CoreDbMptRef): (Blob, Blob) {.apiRaise.} =
## Low level trie dump, not supported for `CoreDbPhkRef`
mpt.setTrackLegaApiOnly
for k,v in mpt.distinctBase.replicate():
yield (k,v)
mpt.ifTrackLegaApi: info legaApiTxt "mpt/replicate()"
# ----------------
proc getTransactionID*(db: CoreDbRef): CoreDbTxID =
(db.toTransactionID().expect "getTransactionID()").CoreDbTxID
db.setTrackLegaApiOnly
const info = "getTransactionID()"
result = (db.toTransactionID().expect info).CoreDbTxID
db.ifTrackLegaApi: info legaApiTxt info
proc shortTimeReadOnly*(
id: CoreDbTxID;
action: proc() {.catchRaise.};
) {.catchRaise.} =
id.setTrackLegaApiOnly
const info = "txId/shortTimeReadOnly()"
var oops = none(ref CatchableError)
proc safeFn() =
try:
@ -544,7 +771,7 @@ when ProvideCoreDbLegacyAPI:
oops = some(e)
# Action has finished now
id.distinctBase.shortTimeReadOnly(safeFn).expect "txId/shortTimeReadOnly()"
id.distinctBase.shortTimeReadOnly(safeFn).expect info
# Delayed exception
if oops.isSome:
@ -553,21 +780,37 @@ when ProvideCoreDbLegacyAPI:
msg = "delayed and reraised" &
", name=\"" & $e.name & "\", msg=\"" & e.msg & "\""
raise (ref TxWrapperApiError)(msg: msg)
id.ifTrackLegaApi: info legaApiTxt info
proc beginTransaction*(db: CoreDbRef): CoreDbTxRef =
(db.distinctBase.newTransaction().expect "newTransaction()").CoreDbTxRef
db.setTrackLegaApiOnly
const info = "newTransaction()"
result = (db.distinctBase.newTransaction().expect info).CoreDbTxRef
db.ifTrackLegaApi: info legaApiTxt info
proc commit*(tx: CoreDbTxRef, applyDeletes = true) =
tx.distinctBase.commit(applyDeletes).expect "tx/commit()"
tx.setTrackLegaApiOnly
const info = "tx/commit()"
tx.distinctBase.commit(applyDeletes).expect info
tx.ifTrackLegaApi: info legaApiTxt info
proc rollback*(tx: CoreDbTxRef) =
tx.distinctBase.rollback().expect "tx/rollback()"
tx.setTrackLegaApiOnly
const info = "tx/rollback()"
tx.distinctBase.rollback().expect info
tx.ifTrackLegaApi: info legaApiTxt info
proc dispose*(tx: CoreDbTxRef) =
tx.distinctBase.dispose().expect "tx/dispose()"
tx.setTrackLegaApiOnly
const info = "tx/dispose()"
tx.distinctBase.dispose().expect info
tx.ifTrackLegaApi: info legaApiTxt info
proc safeDispose*(tx: CoreDbTxRef) =
tx.distinctBase.safeDispose().expect "tx/safeDispose()"
tx.setTrackLegaApiOnly
const info = "tx/safeDispose()"
tx.distinctBase.safeDispose().expect info
tx.ifTrackLegaApi: info legaApiTxt info
# ----------------
@ -575,16 +818,27 @@ when ProvideCoreDbLegacyAPI:
db: CoreDbRef;
flags: set[CoreDbCaptFlags] = {};
): CoreDbCaptRef =
db.newCapture(flags).expect("db/capture()").CoreDbCaptRef
db.setTrackLegaApiOnly
const info = "db/capture()"
result = db.newCapture(flags).expect(info).CoreDbCaptRef
db.ifTrackLegaApi: info legaApiTxt info
proc recorder*(db: CoreDbCaptRef): CoreDbRef =
db.distinctBase.recorder().expect("db/recorder()")
proc recorder*(cp: CoreDbCaptRef): CoreDbRef =
cp.setTrackLegaApiOnly
const info = "capt/recorder()"
result = cp.distinctBase.recorder().expect info
cp.ifTrackLegaApi: info legaApiTxt info
proc logDb*(db: CoreDbCaptRef): CoreDbRef =
db.distinctBase.logDb().expect("db/logDb()")
proc logDb*(cp: CoreDbCaptRef): CoreDbRef =
cp.setTrackLegaApiOnly
const info = "capt/logDb()"
result = cp.distinctBase.logDb().expect info
cp.ifTrackLegaApi: info legaApiTxt info
proc flags*(db: CoreDbCaptRef): set[CoreDbCaptFlags] =
db.distinctBase.flags()
proc flags*(cp: CoreDbCaptRef): set[CoreDbCaptFlags] =
cp.setTrackLegaApiOnly
result = cp.distinctBase.flags()
cp.ifTrackLegaApi: info legaApiTxt "capt/flags()", result=result.toStr
# ------------------------------------------------------------------------------
# End

View File

@ -24,12 +24,9 @@ type
Ooops
LegacyDbMemory
LegacyDbPersistent
AristoDbMemory ## Memory backend emulator
AristoDbRocks ## RocksDB backend
AristoDbVoid ## No backend (to be prefered over `XxxDbMemory`)
const
CoreDbPersistentTypes* = {LegacyDbPersistent, AristoDbRocks}
CoreDbPersistentTypes* = {LegacyDbPersistent}
type
CoreDbRc*[T] = Result[T,CoreDbErrorRef]
@ -215,7 +212,11 @@ type
# --------------------------------------------------
CoreDbRef* = ref object of RootRef
## Database descriptor
dbType*: CoreDbType
dbType*: CoreDbType ## Type of database backend
trackLegaApi*: bool ## Debugging support
trackNewApi*: bool ## Debugging support
trackLedgerApi*: bool ## Debugging suggestion for subsequent ledger
localDbOnly*: bool ## Debugging, suggestion to ignore async fetch
methods*: CoreDbBaseFns
CoreDbErrorRef* = ref object of RootRef

View File

@ -42,13 +42,13 @@ export
CoreDxPhkRef,
CoreDxTxID,
CoreDxTxRef,
`$$`,
backend,
beginTransaction,
capture,
commit,
compensateLegacySetup,
contains,
dbType,
del,
delete,
dispose,
@ -58,6 +58,7 @@ export
getRoot,
getTransactionID,
hash,
hashOrEmpty,
isLegacy,
isPruning,
kvt,
@ -66,6 +67,7 @@ export
mptPrune,
newAccMpt,
newCapture,
newKvt,
newMpt,
newTransaction,
pairs,

View File

@ -32,9 +32,6 @@ proc newCoreDbRef*(
when dbType == LegacyDbPersistent:
newLegacyPersistentCoreDbRef path
elif dbType == AristoDbRocks:
newAristoRocksDbCoreDbRef path
else:
{.error: "Unsupported dbType for persistent newCoreDbRef()".}

View File

@ -11,8 +11,10 @@
{.push raises: [].}
import
std/[algorithm, sequtils, strutils, tables],
eth/[common, trie/hexary],
./core_db
chronicles,
"."/[core_db, storage_types]
type
DB = CoreDbRef
@ -20,11 +22,49 @@ type
StorageTrie* = distinct CoreDbPhkRef
DistinctTrie* = AccountsTrie | StorageTrie
# ------------------------------------------------------------------------------
# Private helper
# ------------------------------------------------------------------------------
func toBase(t: DistinctTrie): CoreDbPhkRef =
## Note that `CoreDbPhkRef` is a distinct variant of `CoreDxPhkRef` for
## the legacy API.
t.CoreDbPhkRef
# ------------------------------------------------------------------------------
# Public debugging helpers
# ------------------------------------------------------------------------------
proc toSvp*(sl: StorageTrie): seq[(UInt256,UInt256)] =
## Dump as slot id-value pair sequence
let
db = sl.toBase.parent
save = db.trackLegaApi
db.trackLegaApi = false
defer: db.trackLegaApi = save
let kvt = db.kvt
var kvp: Table[UInt256,UInt256]
try:
for (slotHash,val) in sl.toBase.toMpt.pairs:
if slotHash.len == 0:
kvp[high UInt256] = high UInt256
else:
let slotRlp = kvt.get(slotHashToSlotKey(slotHash).toOpenArray)
if slotRlp.len == 0:
kvp[high UInt256] = high UInt256
else:
kvp[rlp.decode(slotRlp,UInt256)] = rlp.decode(val,UInt256)
except CatchableError as e:
raiseAssert "Ooops(" & $e.name & "): " & e.msg
kvp.keys.toSeq.sorted.mapIt((it,kvp.getOrDefault(it,high UInt256)))
proc toStr*(w: seq[(UInt256,UInt256)]): string =
"[" & w.mapIt("(" & it[0].toHex & "," & it[1].toHex & ")").join(", ") & "]"
# ------------------------------------------------------------------------------
# Public helpers
# ------------------------------------------------------------------------------
# I don't understand why "borrow" doesn't work here. --Adam
proc rootHash* (t: DistinctTrie): KeccakHash = t.toBase.rootHash()
proc rootHashHex*(t: DistinctTrie): string = $t.toBase.rootHash()
@ -33,6 +73,9 @@ proc isPruning* (t: DistinctTrie): bool = t.toBase.isPruning()
proc mpt* (t: DistinctTrie): CoreDbMptRef = t.toBase.toMpt()
func phk* (t: DistinctTrie): CoreDbPhkRef = t.toBase
# ------------------------------------------------------------------------------
# Public functions: accounts trie
# ------------------------------------------------------------------------------
template initAccountsTrie*(db: DB, rootHash: KeccakHash, isPruning = true): AccountsTrie =
AccountsTrie(db.phkPrune(rootHash, isPruning))
@ -56,9 +99,11 @@ proc putAccountBytes*(trie: var AccountsTrie, address: EthAddress, value: openAr
proc delAccountBytes*(trie: var AccountsTrie, address: EthAddress) =
CoreDbPhkRef(trie).del(address)
# ------------------------------------------------------------------------------
# Public functions: storage trie
# ------------------------------------------------------------------------------
template initStorageTrie*(db: DB, rootHash: KeccakHash, isPruning = true): StorageTrie =
proc initStorageTrie*(db: DB, rootHash: KeccakHash, isPruning = true): StorageTrie =
StorageTrie(db.phkPrune(rootHash, isPruning))
template initStorageTrie*(db: DB, isPruning = true): StorageTrie =
@ -96,3 +141,7 @@ proc storageTrieForAccount*(trie: AccountsTrie, account: Account, isPruning = tr
# it will create virtual container for each account.
# see nim-eth#9
initStorageTrie(trie.db, account.storageRoot, isPruning)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

View File

@ -28,9 +28,30 @@ import
export
AccountsCache,
AccountsLedgerRef,
LedgerType,
base,
init
# ------------------------------------------------------------------------------
# Public constructor
# ------------------------------------------------------------------------------
proc init*(
ldgType: LedgerType;
db: CoreDbRef;
root: Hash256;
pruneTrie: bool;
): LedgerRef =
case ldgType:
of LegacyAccountsCache:
result = AccountsCache.init(db, root, pruneTrie)
of LedgerCache:
result = AccountsLedgerRef.init(db, root, pruneTrie)
else:
raiseAssert: "Missing ledger type label"
# ------------------------------------------------------------------------------
# Public iterators
# ------------------------------------------------------------------------------

View File

@ -10,6 +10,7 @@
import
std/[tables, hashes, sets],
chronicles,
eth/[common, rlp],
../../../stateless/multi_keys,
../../constants,

View File

@ -18,6 +18,7 @@
import
std/[tables, hashes, sets],
chronicles,
eth/[common, rlp],
results,
../../../stateless/multi_keys,
@ -54,7 +55,6 @@ type
codeTouched*: bool
AccountsLedgerRef* = ref object
kvt: CoreDbKvtRef # Legacy API is god enough here
ledger: AccountLedger
savePoint: LedgerSavePoint
witnessCache: Table[EthAddress, WitnessData]
@ -114,6 +114,9 @@ proc beginSavepoint*(ac: AccountsLedgerRef): LedgerSavePoint {.gcsafe.}
# take this out once those are gone.
proc rawTrie*(ac: AccountsLedgerRef): AccountLedger = ac.ledger
proc db(ac: AccountsLedgerRef): CoreDbRef = ac.ledger.db
proc kvt(ac: AccountsLedgerRef): CoreDbKvtRef = ac.db.kvt
func newCoreDbAccount: CoreDbAccount =
CoreDbAccount(
nonce: emptyAcc.nonce,
@ -131,7 +134,6 @@ template noRlpException(info: static[string]; code: untyped) =
proc init*(x: typedesc[AccountsLedgerRef], db: CoreDbRef,
root: KeccakHash, pruneTrie = true): AccountsLedgerRef =
new result
result.kvt = db.kvt
result.ledger = AccountLedger.init(db, root, pruneTrie)
result.witnessCache = initTable[EthAddress, WitnessData]()
discard result.beginSavepoint

View File

@ -183,6 +183,19 @@ proc ledgerExtras(lc: impl.AccountsCache): LedgerExtras =
rawRootHashFn: proc(): Hash256 =
lc.rawTrie.rootHash())
proc newLegacyAccountsCache(
db: CoreDbRef;
root: Hash256;
pruneTrie: bool): LedgerRef =
## Constructor
let lc = impl.AccountsCache.init(db, root, pruneTrie)
wrp.AccountsCache(
ldgType: LegacyAccountsCache,
ac: lc,
extras: lc.ledgerExtras(),
methods: lc.ledgerMethods()).bless db
# ------------------------------------------------------------------------------
# Public iterators
# ------------------------------------------------------------------------------
@ -224,13 +237,7 @@ proc init*(
db: CoreDbRef;
root: Hash256;
pruneTrie: bool): LedgerRef =
let lc = impl.AccountsCache.init(db, root, pruneTrie)
result = T(
ldgType: LegacyAccountsCache,
ac: lc,
extras: lc.ledgerExtras(),
methods: lc.ledgerMethods())
result.validate
db.newLegacyAccountsCache(root, pruneTrie)
# ------------------------------------------------------------------------------
# End

View File

@ -138,10 +138,10 @@ proc ledgerMethods(lc: impl.AccountsLedgerRef): LedgerFns =
safeDisposeFn: proc(sp: LedgerSpRef) =
lc.safeDispose(sp.savePoint),
selfDestruct6780Fn: proc(eAddr: EthAddress) =
selfDestructFn: proc(eAddr: EthAddress) =
lc.selfDestruct(eAddr),
selfDestructFn: proc(eAddr: EthAddress) =
selfDestruct6780Fn: proc(eAddr: EthAddress) =
lc.selfDestruct6780(eAddr),
selfDestructLenFn: proc(): int =
@ -170,6 +170,18 @@ proc ledgerExtras(lc: impl.AccountsLedgerRef): LedgerExtras =
rawRootHashFn: proc(): Hash256 =
lc.rawTrie.rootHash())
proc newAccountsLedgerRef(
db: CoreDbRef;
root: Hash256;
pruneTrie: bool): LedgerRef =
let lc = impl.AccountsLedgerRef.init(db, root, pruneTrie)
wrp.AccountsLedgerRef(
ldgType: LedgerCache,
ac: lc,
extras: lc.ledgerExtras(),
methods: lc.ledgerMethods()).bless db
# ------------------------------------------------------------------------------
# Public iterators
# ------------------------------------------------------------------------------
@ -210,13 +222,7 @@ proc init*(
db: CoreDbRef;
root: Hash256;
pruneTrie: bool): LedgerRef =
let lc = impl.AccountsLedgerRef.init(db, root, pruneTrie)
result = T(
ldgType: LedgerCache,
ac: lc,
extras: lc.ledgerExtras(),
methods: lc.ledgerMethods())
result.validate
db.newAccountsLedgerRef(root, pruneTrie)
# ------------------------------------------------------------------------------
# End

View File

@ -14,7 +14,9 @@
import
eth/common,
chronicles,
../../../stateless/multi_keys,
../core_db,
./base/[base_desc, validate]
type
@ -30,16 +32,57 @@ when defined(release):
else:
const AutoValidateDescriptors = true
const
EnableApiTracking = true and false
## When enabled, API functions are logged. Tracking is enabled by setting
## the `trackApi` flag to `true`.
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
when EnableApiTracking:
import std/strutils, stew/byteutils
template apiTxt(info: static[string]): static[string] =
"Ledger API " & info
template ifTrackApi(ldg: LedgerRef; code: untyped) =
when EnableApiTracking:
if ldg.trackApi:
code
proc oaToStr(w: openArray[byte]): string =
w.toHex.toLowerAscii
proc toStr(w: EthAddress): string =
w.oaToStr
proc toStr(w: Hash256): string =
w.data.oaToStr
proc toStr(w: Blob): string =
if 0 < w.len and w.len < 5: "<" & w.oaToStr & ">"
else: "Blob[" & $w.len & "]"
proc toStr(w: seq[Log]): string =
"Logs[" & $w.len & "]"
else:
template ifTrackApi(ldg: LedgerRef; code: untyped) = discard
# ------------------------------------------------------------------------------
# Public constructor helper
# ------------------------------------------------------------------------------
proc bless*(ldg: LedgerRef; db: CoreDbRef): LedgerRef =
when AutoValidateDescriptors:
proc validate*(ldg: LedgerRef) =
validate.validate(ldg)
else:
template validate*(ldg: LedgerRef) =
discard
ldg.validate()
when EnableApiTracking:
ldg.trackApi = db.trackLedgerApi
if ldg.trackApi:
info apiTxt "LedgerRef.init()", ldgType=ldg.ldgType
ldg
# ------------------------------------------------------------------------------
# Public methods
@ -47,145 +90,197 @@ else:
proc accessList*(ldg: LedgerRef, eAddr: EthAddress) =
ldg.methods.accessListFn(eAddr)
ldg.ifTrackApi: info apiTxt "accessList()", eAddr=eAddr.toStr
proc accessList*(ldg: LedgerRef, eAddr: EthAddress, slot: UInt256) =
ldg.methods.accessList2Fn(eAddr, slot)
ldg.ifTrackApi: info apiTxt "accessList()", eAddr=eAddr.toStr, slot
proc accountExists*(ldg: LedgerRef, eAddr: EthAddress): bool =
ldg.methods.accountExistsFn(eAddr)
result = ldg.methods.accountExistsFn(eAddr)
ldg.ifTrackApi: info apiTxt "accountExists()", eAddr=eAddr.toStr, result
proc addBalance*(ldg: LedgerRef, eAddr: EthAddress, delta: UInt256) =
ldg.methods.addBalanceFn(eAddr, delta)
ldg.ifTrackApi: info apiTxt "addBalance()", eAddr=eAddr.toStr, delta
proc addLogEntry*(ldg: LedgerRef, log: Log) =
ldg.methods.addLogEntryFn(log)
ldg.ifTrackApi: info apiTxt "addLogEntry()"
proc beginSavepoint*(ldg: LedgerRef): LedgerSpRef =
ldg.methods.beginSavepointFn()
result = ldg.methods.beginSavepointFn()
ldg.ifTrackApi: info apiTxt "beginSavepoint()"
proc clearStorage*(ldg: LedgerRef, eAddr: EthAddress) =
ldg.methods.clearStorageFn(eAddr)
ldg.ifTrackApi: info apiTxt "clearStorage()", eAddr=eAddr.toStr
proc clearTransientStorage*(ldg: LedgerRef) =
ldg.methods.clearTransientStorageFn()
ldg.ifTrackApi: info apiTxt "clearTransientStorage()"
proc collectWitnessData*(ldg: LedgerRef) =
ldg.methods.collectWitnessDataFn()
ldg.ifTrackApi: info apiTxt "collectWitnessData()"
proc commit*(ldg: LedgerRef, sp: LedgerSpRef) =
ldg.methods.commitFn(sp)
ldg.ifTrackApi: info apiTxt "commit()"
proc deleteAccount*(ldg: LedgerRef, eAddr: EthAddress) =
ldg.methods.deleteAccountFn(eAddr)
ldg.ifTrackApi: info apiTxt "deleteAccount()", eAddr=eAddr.toStr
proc dispose*(ldg: LedgerRef, sp: LedgerSpRef) =
ldg.methods.disposeFn(sp)
ldg.ifTrackApi: info apiTxt "dispose()"
proc getAndClearLogEntries*(ldg: LedgerRef): seq[Log] =
ldg.methods.getAndClearLogEntriesFn()
result = ldg.methods.getAndClearLogEntriesFn()
ldg.ifTrackApi: info apiTxt "getAndClearLogEntries()"
proc getBalance*(ldg: LedgerRef, eAddr: EthAddress): UInt256 =
ldg.methods.getBalanceFn(eAddr)
result = ldg.methods.getBalanceFn(eAddr)
ldg.ifTrackApi: info apiTxt "getBalance()", eAddr=eAddr.toStr, result
proc getCode*(ldg: LedgerRef, eAddr: EthAddress): Blob =
ldg.methods.getCodeFn(eAddr)
result = ldg.methods.getCodeFn(eAddr)
ldg.ifTrackApi:
info apiTxt "getCode()", eAddr=eAddr.toStr, result=result.toStr
proc getCodeHash*(ldg: LedgerRef, eAddr: EthAddress): Hash256 =
ldg.methods.getCodeHashFn(eAddr)
result = ldg.methods.getCodeHashFn(eAddr)
ldg.ifTrackApi:
info apiTxt "getCodeHash()", eAddr=eAddr.toStr, result=result.toStr
proc getCodeSize*(ldg: LedgerRef, eAddr: EthAddress): int =
ldg.methods.getCodeSizeFn(eAddr)
result = ldg.methods.getCodeSizeFn(eAddr)
ldg.ifTrackApi: info apiTxt "getCodeSize()", eAddr=eAddr.toStr, result
proc getCommittedStorage*(ldg: LedgerRef, eAddr: EthAddress, slot: UInt256): UInt256 =
ldg.methods.getCommittedStorageFn(eAddr, slot)
result = ldg.methods.getCommittedStorageFn(eAddr, slot)
ldg.ifTrackApi:
info apiTxt "getCommittedStorage()", eAddr=eAddr.toStr, slot, result
proc getNonce*(ldg: LedgerRef, eAddr: EthAddress): AccountNonce =
ldg.methods.getNonceFn(eAddr)
result = ldg.methods.getNonceFn(eAddr)
ldg.ifTrackApi: info apiTxt "getNonce()", eAddr=eAddr.toStr, result
proc getStorage*(ldg: LedgerRef, eAddr: EthAddress, slot: UInt256): UInt256 =
ldg.methods.getStorageFn(eAddr, slot)
result = ldg.methods.getStorageFn(eAddr, slot)
ldg.ifTrackApi: info apiTxt "getStorage()", eAddr=eAddr.toStr, slot, result
proc getStorageRoot*(ldg: LedgerRef, eAddr: EthAddress): Hash256 =
ldg.methods.getStorageRootFn(eAddr)
result = ldg.methods.getStorageRootFn(eAddr)
ldg.ifTrackApi:
info apiTxt "getStorageRoot()", eAddr=eAddr.toStr, result=result.toStr
proc getTransientStorage*(ldg: LedgerRef, eAddr: EthAddress, slot: UInt256): UInt256 =
ldg.methods.getTransientStorageFn(eAddr, slot)
result = ldg.methods.getTransientStorageFn(eAddr, slot)
ldg.ifTrackApi:
info apiTxt "getTransientStorage()", eAddr=eAddr.toStr, slot, result
proc hasCodeOrNonce*(ldg: LedgerRef, eAddr: EthAddress): bool =
ldg.methods.hasCodeOrNonceFn(eAddr)
result = ldg.methods.hasCodeOrNonceFn(eAddr)
ldg.ifTrackApi: info apiTxt "hasCodeOrNonce()", eAddr=eAddr.toStr, result
proc inAccessList*(ldg: LedgerRef, eAddr: EthAddress): bool =
ldg.methods.inAccessListFn(eAddr)
result = ldg.methods.inAccessListFn(eAddr)
ldg.ifTrackApi: info apiTxt "inAccessList()", eAddr=eAddr.toStr, result
proc inAccessList*(ldg: LedgerRef, eAddr: EthAddress, slot: UInt256): bool =
ldg.methods.inAccessList2Fn(eAddr, slot)
result = ldg.methods.inAccessList2Fn(eAddr, slot)
ldg.ifTrackApi: info apiTxt "inAccessList()", eAddr=eAddr.toStr, slot, result
proc incNonce*(ldg: LedgerRef, eAddr: EthAddress) =
ldg.methods.incNonceFn(eAddr)
ldg.ifTrackApi: info apiTxt "incNonce()", eAddr=eAddr.toStr
proc isDeadAccount*(ldg: LedgerRef, eAddr: EthAddress): bool =
ldg.methods.isDeadAccountFn(eAddr)
result = ldg.methods.isDeadAccountFn(eAddr)
ldg.ifTrackApi: info apiTxt "isDeadAccount()", eAddr=eAddr.toStr, result
proc isEmptyAccount*(ldg: LedgerRef, eAddr: EthAddress): bool =
ldg.methods.isEmptyAccountFn(eAddr)
result = ldg.methods.isEmptyAccountFn(eAddr)
ldg.ifTrackApi: info apiTxt "isEmptyAccount()", eAddr=eAddr.toStr, result
proc isTopLevelClean*(ldg: LedgerRef): bool =
ldg.methods.isTopLevelCleanFn()
result = ldg.methods.isTopLevelCleanFn()
ldg.ifTrackApi: info apiTxt "isTopLevelClean()", result
proc logEntries*(ldg: LedgerRef): seq[Log] =
ldg.methods.logEntriesFn()
result = ldg.methods.logEntriesFn()
ldg.ifTrackApi: info apiTxt "logEntries()", result=result.toStr
proc makeMultiKeys*(ldg: LedgerRef): MultikeysRef =
ldg.methods.makeMultiKeysFn()
result = ldg.methods.makeMultiKeysFn()
ldg.ifTrackApi: info apiTxt "makeMultiKeys()"
proc persist*(ldg: LedgerRef, clearEmptyAccount = false, clearCache = true) =
ldg.methods.persistFn(clearEmptyAccount, clearCache)
ldg.ifTrackApi: info apiTxt "persist()", clearEmptyAccount, clearCache
proc ripemdSpecial*(ldg: LedgerRef) =
ldg.methods.ripemdSpecialFn()
ldg.ifTrackApi: info apiTxt "ripemdSpecial()"
proc rollback*(ldg: LedgerRef, sp: LedgerSpRef) =
ldg.methods.rollbackFn(sp)
ldg.ifTrackApi: info apiTxt "rollback()"
proc rootHash*(ldg: LedgerRef): Hash256 =
ldg.methods.rootHashFn()
result = ldg.methods.rootHashFn()
ldg.ifTrackApi: info apiTxt "rootHash()", result=result.toStr
proc safeDispose*(ldg: LedgerRef, sp: LedgerSpRef) =
ldg.methods.safeDisposeFn(sp)
ldg.ifTrackApi: info apiTxt "safeDispose()"
proc selfDestruct*(ldg: LedgerRef, eAddr: EthAddress) =
ldg.methods.selfDestructFn(eAddr)
ldg.ifTrackApi: info apiTxt "selfDestruct()"
proc selfDestruct6780*(ldg: LedgerRef, eAddr: EthAddress) =
ldg.methods.selfDestruct6780Fn(eAddr)
ldg.ifTrackApi: info apiTxt "selfDestruct6780()"
proc selfDestructLen*(ldg: LedgerRef): int =
ldg.methods.selfDestructLenFn()
result = ldg.methods.selfDestructLenFn()
ldg.ifTrackApi: info apiTxt "selfDestructLen()", result
proc setBalance*(ldg: LedgerRef, eAddr: EthAddress, balance: UInt256) =
ldg.methods.setBalanceFn(eAddr, balance)
ldg.ifTrackApi: info apiTxt "setBalance()", eAddr=eAddr.toStr, balance
proc setCode*(ldg: LedgerRef, eAddr: EthAddress, code: Blob) =
ldg.methods.setCodeFn(eAddr, code)
ldg.ifTrackApi: info apiTxt "setCode()", eAddr=eAddr.toStr, code=code.toStr
proc setNonce*(ldg: LedgerRef, eAddr: EthAddress, nonce: AccountNonce) =
ldg.methods.setNonceFn(eAddr, nonce)
ldg.ifTrackApi: info apiTxt "setNonce()", eAddr=eAddr.toStr, nonce
proc setStorage*(ldg: LedgerRef, eAddr: EthAddress, slot, val: UInt256) =
ldg.methods.setStorageFn(eAddr, slot, val)
ldg.ifTrackApi: info apiTxt "setStorage()", eAddr=eAddr.toStr, slot, val
proc setTransientStorage*(ldg: LedgerRef, eAddr: EthAddress, slot, val: UInt256) =
ldg.methods.setTransientStorageFn(eAddr, slot, val)
ldg.ifTrackApi:
info apiTxt "setTransientStorage()", eAddr=eAddr.toStr, slot, val
proc subBalance*(ldg: LedgerRef, eAddr: EthAddress, delta: UInt256) =
ldg.methods.subBalanceFn(eAddr, delta)
ldg.ifTrackApi: info apiTxt "setTransientStorage()", eAddr=eAddr.toStr, delta
# ------------------------------------------------------------------------------
# Public methods, extensions to go away
# ------------------------------------------------------------------------------
proc rawRootHash*(ldg: LedgerRef): Hash256 =
ldg.extras.rawRootHashFn()
result = ldg.extras.rawRootHashFn()
ldg.ifTrackApi: info apiTxt "rawRootHash()", result=result.toStr
# ------------------------------------------------------------------------------
# Public virtual read-only methods

View File

@ -29,6 +29,7 @@ type
LedgerRef* = ref object of RootRef
## Root object with closures
ldgType*: LedgerType ## For debugging
trackApi*: bool ## For debugging
extras*: LedgerExtras ## Support might go away
methods*: LedgerFns

View File

@ -10,21 +10,55 @@
{.push raises: [].}
## Re-write of `distinct_tries.nim` to be imported into `accounts_cache.nim`
## Re-write of `distinct_tries.nim` to be imported into `accounts_ledger.nim`
## for using new database API.
##
import
std/typetraits,
std/[algorithm, sequtils, strutils, tables, typetraits],
chronicles,
eth/common,
results,
../core_db
".."/[core_db, storage_types]
type
AccountLedger* = distinct CoreDxAccRef
StorageLedger* = distinct CoreDxPhkRef
SomeLedger* = AccountLedger | StorageLedger
# ------------------------------------------------------------------------------
# Public debugging helpers
# ------------------------------------------------------------------------------
proc toSvp*(sl: StorageLedger): seq[(UInt256,UInt256)] =
## Dump as slot id-value pair sequence
let
db = sl.distinctBase.parent
save = db.trackNewApi
db.trackNewApi = false
defer: db.trackNewApi = save
let kvt = db.newKvt
var kvp: Table[UInt256,UInt256]
try:
for (slotHash,val) in sl.distinctBase.toMpt.pairs:
let rc = kvt.get(slotHashToSlotKey(slotHash).toOpenArray)
if rc.isErr:
warn "StorageLedger.dump()", slotHash, error=($$rc.error)
else:
kvp[rlp.decode(rc.value,UInt256)] = rlp.decode(val,UInt256)
except CatchableError as e:
raiseAssert "Ooops(" & $e.name & "): " & e.msg
kvp.keys.toSeq.sorted.mapIt((it,kvp.getOrDefault(it,high UInt256)))
proc toStr*(w: seq[(UInt256,UInt256)]): string =
"[" & w.mapIt("(" & it[0].toHex & "," & it[1].toHex & ")").join(", ") & "]"
# ------------------------------------------------------------------------------
# Public helpers
# ------------------------------------------------------------------------------
proc db*(t: SomeLedger): CoreDbRef =
t.distinctBase.parent
proc rootHash*(t: SomeLedger): Hash256 =
t.distinctBase.rootVid().hash().expect "SomeLedger/rootHash()"
@ -32,6 +66,9 @@ proc rootHash*(t: SomeLedger): Hash256 =
proc rootVid*(t: SomeLedger): CoreDbVidRef =
t.distinctBase.rootVid
# ------------------------------------------------------------------------------
# Public functions: accounts ledger
# ------------------------------------------------------------------------------
proc init*(
T: type AccountLedger;
@ -60,17 +97,26 @@ proc merge*(al: AccountLedger; eAddr: EthAddress; account: CoreDbAccount) =
proc delete*(al: AccountLedger, eAddr: EthAddress) =
al.distinctBase.delete(eAddr).expect "AccountLedger/delete()"
# ------------------------------------------------------------------------------
# Public functions: storage ledger
# ------------------------------------------------------------------------------
proc init*(
T: type StorageLedger;
al: AccountLedger;
account: CoreDbAccount;
isPruning = true;
isPruning = false;
): T =
## Storage trie constructor.
##
## Note that the argument `isPruning` should be left `false` on the legacy
## `CoreDb` backend. Otherwise, pruning might kill some unwanted entries from
## storage tries ending up with an unstable database leading to crashes (see
## https://github.com/status-im/nimbus-eth1/issues/932.)
al.distinctBase.parent.newMpt(account.storageVid, isPruning).toPhk.T
proc init*(T: type StorageLedger; db: CoreDbRef, isPruning = true): T =
db.newMpt(CoreDbVidRef(nil), isPruning).toPhk.T
#proc init*(T: type StorageLedger; db: CoreDbRef, isPruning = false): T =
# db.newMpt(CoreDbVidRef(nil), isPruning).toPhk.T
proc fetch*(sl: StorageLedger, slot: UInt256): Result[Blob,void] =
sl.distinctBase.fetch(slot.toBytesBE).mapErr proc(ign: CoreDbErrorRef)=discard
@ -90,4 +136,6 @@ iterator storage*(
for (key,val) in al.distinctBase.parent.newMpt(account.storageVid).pairs:
yield (key,val)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

View File

@ -60,7 +60,7 @@ type
proc pruneTrie*(db: AccountStateDB): bool =
db.trie.isPruning
func db*(db: AccountStateDB): CoreDbRef =
proc db*(db: AccountStateDB): CoreDbRef =
db.trie.db
proc kvt*(db: AccountStateDB): CoreDbKvtRef =

View File

@ -10,18 +10,23 @@ import
proc ifNecessaryGetAccount*(vmState: BaseVMState, address: EthAddress): Future[void] {.async.} =
if vmState.com.db.localDbOnly: return
await vmState.asyncFactory.ifNecessaryGetAccount(vmState.com.db, vmState.parent.blockNumber, vmState.parent.stateRoot, address, vmState.stateDB.rawTrie.rootHash)
proc ifNecessaryGetCode*(vmState: BaseVMState, address: EthAddress): Future[void] {.async.} =
if vmState.com.db.localDbOnly: return
await vmState.asyncFactory.ifNecessaryGetCode(vmState.com.db, vmState.parent.blockNumber, vmState.parent.stateRoot, address, vmState.stateDB.rawTrie.rootHash)
proc ifNecessaryGetSlots*(vmState: BaseVMState, address: EthAddress, slots: seq[UInt256]): Future[void] {.async.} =
if vmState.com.db.localDbOnly: return
await vmState.asyncFactory.ifNecessaryGetSlots(vmState.com.db, vmState.parent.blockNumber, vmState.parent.stateRoot, address, slots, vmState.stateDB.rawTrie.rootHash)
proc ifNecessaryGetSlot*(vmState: BaseVMState, address: EthAddress, slot: UInt256): Future[void] {.async.} =
if vmState.com.db.localDbOnly: return
await ifNecessaryGetSlots(vmState, address, @[slot])
proc ifNecessaryGetBlockHeaderByNumber*(vmState: BaseVMState, blockNumber: BlockNumber): Future[void] {.async.} =
if vmState.com.db.localDbOnly: return
await vmState.asyncFactory.ifNecessaryGetBlockHeaderByNumber(vmState.com.db, blockNumber)
#[
@ -38,9 +43,11 @@ proc fetchAndPopulateNodes*(vmState: BaseVMState, paths: seq[seq[seq[byte]]], no
# Sometimes it's convenient to be able to do multiple at once.
proc ifNecessaryGetAccounts*(vmState: BaseVMState, addresses: seq[EthAddress]): Future[void] {.async.} =
if vmState.com.db.localDbOnly: return
for address in addresses:
await ifNecessaryGetAccount(vmState, address)
proc ifNecessaryGetCodeForAccounts*(vmState: BaseVMState, addresses: seq[EthAddress]): Future[void] {.async.} =
if vmState.com.db.localDbOnly: return
for address in addresses:
await ifNecessaryGetCode(vmState, address)

View File

@ -16,6 +16,7 @@
import
std/[sequtils, strformat, strutils, tables, times],
eth/common,
stew/byteutils,
../../nimbus/constants
export
@ -106,7 +107,7 @@ proc pp*(q: openArray[int]; itemsPerLine: int; lineSep: string): string =
proc pp*(a: MDigest[256]; collapse = true): string =
if not collapse:
a.data.mapIt(it.toHex(2)).join.toLowerAscii
a.data.toHex.toLowerAscii
elif a == ZERO_HASH256:
"ZERO_HASH256"
elif a == EMPTY_ROOT_HASH:
@ -118,7 +119,7 @@ proc pp*(a: MDigest[256]; collapse = true): string =
elif a == ZERO_HASH256:
"ZERO_HASH256"
else:
a.data.mapIt(it.toHex(2)).join[56 .. 63].toLowerAscii
a.data.toHex.join[56 .. 63].toLowerAscii
proc pp*(a: openArray[MDigest[256]]; collapse = true): string =
"@[" & a.toSeq.mapIt(it.pp).join(" ") & "]"
@ -132,7 +133,7 @@ proc pp*(q: openArray[byte]; noHash = false): string =
for n in 0..31: a[n] = q[n]
MDigest[256](data: a).pp
else:
q.toSeq.mapIt(it.toHex(2)).join.toLowerAscii.pp(hex = true)
q.toHex.toLowerAscii.pp(hex = true)
# ------------------------------------------------------------------------------
# Elapsed time pretty printer

View File

@ -17,9 +17,10 @@ import
eth/common,
results,
unittest2,
../../nimbus/[db/core_db/persistent, core/chain],
../../nimbus/db/[core_db/persistent, ledger],
../../nimbus/core/chain,
./replay/pp,
./test_coredb/[coredb_test_xx, test_legacy]
./test_coredb/[coredb_test_xx, test_chainsync]
const
baseDir = [".", "..", ".."/"..", $DirSep]
@ -122,14 +123,14 @@ proc legacyRunner(
suite "Legacy DB: test Core API interfaces"&
&", capture={fileInfo}, {sayPersistent}":
test &"Legaci API, {numBlocksInfo} blocks":
test &"Ledger API, {numBlocksInfo} blocks":
let
com = openLegacyDB(persistent, dbDir, capture.network)
defer:
com.db.finish(flush = true)
if persistent: dbDir.flushDbDir
check noisy.testChainSyncLegacyApi(filePath, com, numBlocks)
check noisy.testChainSync(filePath, com, numBlocks)
# ------------------------------------------------------------------------------
# Main function(s)
@ -149,8 +150,8 @@ when isMainModule:
# dumps `bulkTest2`, `bulkTest3`, .. from the `nimbus-eth1-blobs` package.
# For specs see `tests/test_coredb/bulk_test_xx.nim`.
var testList = @[bulkTest0] # This test is superseded by `bulkTest1` and `2`
# testList = @[failSample0]
when true: # and false:
testList = @[failSample0]
when true and false:
testList = @[bulkTest2, bulkTest3]
for n,capture in testList:

View File

@ -46,9 +46,9 @@ const
failSample0* = CaptureSpecs(
name: "fail-goerli",
network: bulkTest0.network,
file: bulkTest0.file,
numBlocks: 18004)
network: bulkTest2.network,
file: bulkTest2.file,
numBlocks: 147_621)
failSample1* = CaptureSpecs(
name: "fail-main",

View File

@ -37,7 +37,7 @@ proc setErrorLevel {.used.} =
# Public test function
# ------------------------------------------------------------------------------
proc test_chainSyncLegacyApi*(
proc test_chainSync*(
noisy: bool;
filePath: string;
com: CommonRef;
@ -88,7 +88,12 @@ proc test_chainSyncLegacyApi*(
xCheck runPersistBlocks1Rc == ValidationResult.OK
dotsOrSpace = " "
if noisy: setTraceLevel()
if noisy:
setTraceLevel()
com.db.trackLegaApi = true
com.db.trackNewApi = true
com.db.trackLedgerApi = true
com.db.localDbOnly = true
if lastOneExtra:
let
headers0 = headers9[0..0]