status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

avoid closure environment for mpt methods (#2408) 

An instance of `CoreDbMptRef` is created for and stored in every account
- when we are processing blocks and have many accounts in memory, this
closure environment takes up hundreds of mb of memory (around block 5M,
it is the 4:th largest memory consumer!) - incidentally, this also
removes a circular reference in the setup that causes the
`AristoCodeDbMptRef` to linger in memory much longer than it
has to which is the core reason why it takes so much.

The real solution here is to remove the methods indirection entirely,
but this PR provides relief until that has been done.

Similar treatment is given to some of the other core api functions to
avoid circulars there too.
This commit is contained in:
Jacek Sieka 2024-06-24 07:56:41 +02:00 committed by GitHub
parent 99ff8dc876
commit 9521582005
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 131 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

147
nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim
View File

@ -183,20 +183,19 @@ func toVoidRc[T](
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Private `MPT` call back functions # Private `MPT` call back functions
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
proc mptMethods(): CoreDbMptFns =
# These templates are a hack to remove a closure environment that was using
# hundreds of mb of memory to have this syntactic convenience
# TODO remove methods / abstraction entirely - it is no longer needed
template base: untyped = cMpt.base
template db: untyped = base.parent # Ditto
template api: untyped = base.api # Ditto
template mpt: untyped = base.ctx.mpt # Ditto
proc mptMethods(cMpt: AristoCoreDbMptRef): CoreDbMptFns = proc mptBackend(cMpt: AristoCoreDbMptRef): CoreDbMptBackendRef =
## Generic columns database handlers
let
cMpt = cMpt # So it can savely be captured
base = cMpt.base # Will not change and can be captured
db = base.parent # Ditto
api = base.api # Ditto
mpt = base.ctx.mpt # Ditto
proc mptBackend(): CoreDbMptBackendRef =
db.bless AristoCoreDbMptBE(adb: mpt) db.bless AristoCoreDbMptBE(adb: mpt)
proc mptColFn(): CoreDbColRef = proc mptColFn(cMpt: AristoCoreDbMptRef): CoreDbColRef =
if cMpt.mptRoot.distinctBase < LEAST_FREE_VID: if cMpt.mptRoot.distinctBase < LEAST_FREE_VID:
return db.bless(AristoColRef( return db.bless(AristoColRef(
base: base, base: base,
@ -219,7 +218,7 @@ proc mptMethods(cMpt: AristoCoreDbMptRef): CoreDbMptFns =
stoRoot: cMpt.mptRoot, stoRoot: cMpt.mptRoot,
stoAddr: cMpt.address) stoAddr: cMpt.address)
proc mptFetch(key: openArray[byte]): CoreDbRc[Blob] = proc mptFetch(cMpt: AristoCoreDbMptRef, key: openArray[byte]): CoreDbRc[Blob] =
const info = "fetchFn()" const info = "fetchFn()"
let rc = block: let rc = block:
@ -241,7 +240,7 @@ proc mptMethods(cMpt: AristoCoreDbMptRef): CoreDbMptFns =
else: else:
err(rc.error.toError(base, info, MptNotFound)) err(rc.error.toError(base, info, MptNotFound))
proc mptMerge(k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] = proc mptMerge(cMpt: AristoCoreDbMptRef, k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] =
const info = "mergeFn()" const info = "mergeFn()"
if cMpt.accPath.isValid: if cMpt.accPath.isValid:
@ -257,7 +256,7 @@ proc mptMethods(cMpt: AristoCoreDbMptRef): CoreDbMptFns =
ok() ok()
proc mptDelete(key: openArray[byte]): CoreDbRc[void] = proc mptDelete(cMpt: AristoCoreDbMptRef, key: openArray[byte]): CoreDbRc[void] =
const info = "deleteFn()" const info = "deleteFn()"
let rc = block: let rc = block:
@ -281,7 +280,7 @@ proc mptMethods(cMpt: AristoCoreDbMptRef): CoreDbMptFns =
ok() ok()
proc mptHasPath(key: openArray[byte]): CoreDbRc[bool] = proc mptHasPath(cMpt: AristoCoreDbMptRef, key: openArray[byte]): CoreDbRc[bool] =
const info = "hasPathFn()" const info = "hasPathFn()"
let rc = block: let rc = block:
@ -295,52 +294,50 @@ proc mptMethods(cMpt: AristoCoreDbMptRef): CoreDbMptFns =
return err(rc.error.toError(base, info)) return err(rc.error.toError(base, info))
ok(rc.value) ok(rc.value)
## Generic columns database handlers
CoreDbMptFns( CoreDbMptFns(
backendFn: proc(): CoreDbMptBackendRef = backendFn: proc(cMpt: CoreDbMptRef): CoreDbMptBackendRef =
mptBackend(), mptBackend(AristoCoreDbMptRef(cMpt)),
fetchFn: proc(k: openArray[byte]): CoreDbRc[Blob] = fetchFn: proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[Blob] =
mptFetch(k), mptFetch(AristoCoreDbMptRef(cMpt), k),
deleteFn: proc(k: openArray[byte]): CoreDbRc[void] = deleteFn: proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[void] =
mptDelete(k), mptDelete(AristoCoreDbMptRef(cMpt), k),
mergeFn: proc(k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] = mergeFn: proc(cMpt: CoreDbMptRef, k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] =
mptMerge(k, v), mptMerge(AristoCoreDbMptRef(cMpt), k, v),
hasPathFn: proc(k: openArray[byte]): CoreDbRc[bool] = hasPathFn: proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[bool] =
mptHasPath(k), mptHasPath(AristoCoreDbMptRef(cMpt), k),
getColFn: proc(): CoreDbColRef = getColFn: proc(cMpt: CoreDbMptRef): CoreDbColRef =
mptColFn()) mptColFn(AristoCoreDbMptRef(cMpt)))
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Private account call back functions # Private account call back functions
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
proc accMethods(cAcc: AristoCoreDbAccRef): CoreDbAccFns = proc accMethods(): CoreDbAccFns =
## Account columns database handlers ## Account columns database handlers
let template base: untyped = cAcc.base
cAcc = cAcc # So it can savely be captured template db: untyped = base.parent
base = cAcc.base # Will not change and can be captured template api: untyped = base.api
db = base.parent # Ditto template mpt: untyped = base.ctx.mpt
api = base.api # Ditto
mpt = base.ctx.mpt # Ditto
proc getColFn(): CoreDbColRef = proc getColFn(cAcc: AristoCoreDbAccRef): CoreDbColRef =
db.bless AristoColRef( db.bless AristoColRef(
base: base, base: base,
colType: CtAccounts) colType: CtAccounts)
proc accCloneMpt(): CoreDbRc[CoreDbMptRef] = proc accCloneMpt(cAcc: AristoCoreDbAccRef): CoreDbRc[CoreDbMptRef] =
var xpt = AristoCoreDbMptRef( var xpt = AristoCoreDbMptRef(
base: base, base: base,
mptRoot: AccountsVID) mptRoot: AccountsVID)
xpt.methods = xpt.mptMethods xpt.methods = mptMethods()
ok(db.bless xpt) ok(db.bless xpt)
proc accFetch(address: EthAddress): CoreDbRc[CoreDbAccount] = proc accFetch(cAcc: AristoCoreDbAccRef, address: EthAddress): CoreDbRc[CoreDbAccount] =
const info = "acc/fetchFn()" const info = "acc/fetchFn()"
let let
@ -352,7 +349,7 @@ proc accMethods(cAcc: AristoCoreDbAccRef): CoreDbAccFns =
ok cAcc.toCoreDbAccount(acc, address) ok cAcc.toCoreDbAccount(acc, address)
proc accMerge(account: CoreDbAccount): CoreDbRc[void] = proc accMerge(cAcc: AristoCoreDbAccRef, account: CoreDbAccount): CoreDbRc[void] =
const info = "acc/mergeFn()" const info = "acc/mergeFn()"
let let
@ -363,7 +360,7 @@ proc accMethods(cAcc: AristoCoreDbAccRef): CoreDbAccFns =
return err(rc.error.toError(base, info)) return err(rc.error.toError(base, info))
ok() ok()
proc accDelete(address: EthAddress): CoreDbRc[void] = proc accDelete(cAcc: AristoCoreDbAccRef, address: EthAddress): CoreDbRc[void] =
const info = "acc/deleteFn()" const info = "acc/deleteFn()"
let key = address.keccakHash.data let key = address.keccakHash.data
@ -374,7 +371,7 @@ proc accMethods(cAcc: AristoCoreDbAccRef): CoreDbAccFns =
ok() ok()
proc accStoDelete(address: EthAddress): CoreDbRc[void] = proc accStoDelete(cAcc: AristoCoreDbAccRef, address: EthAddress): CoreDbRc[void] =
const info = "stoDeleteFn()" const info = "stoDeleteFn()"
let rc = api.deleteStorageTree(mpt, address.to(PathID)) let rc = api.deleteStorageTree(mpt, address.to(PathID))
@ -383,7 +380,7 @@ proc accMethods(cAcc: AristoCoreDbAccRef): CoreDbAccFns =
ok() ok()
proc accHasPath(address: EthAddress): CoreDbRc[bool] = proc accHasPath(cAcc: AristoCoreDbAccRef, address: EthAddress): CoreDbRc[bool] =
const info = "hasPathFn()" const info = "hasPathFn()"
let let
@ -394,40 +391,39 @@ proc accMethods(cAcc: AristoCoreDbAccRef): CoreDbAccFns =
CoreDbAccFns( CoreDbAccFns(
getMptFn: proc(): CoreDbRc[CoreDbMptRef] = getMptFn: proc(cAcc: CoreDbAccRef): CoreDbRc[CoreDbMptRef] =
accCloneMpt(), accCloneMpt(AristoCoreDbAccRef(cAcc)),
fetchFn: proc(address: EthAddress): CoreDbRc[CoreDbAccount] = fetchFn: proc(cAcc: CoreDbAccRef, address: EthAddress): CoreDbRc[CoreDbAccount] =
accFetch(address), accFetch(AristoCoreDbAccRef(cAcc), address),
deleteFn: proc(address: EthAddress): CoreDbRc[void] = deleteFn: proc(cAcc: CoreDbAccRef, address: EthAddress): CoreDbRc[void] =
accDelete(address), accDelete(AristoCoreDbAccRef(cAcc), address),
stoDeleteFn: proc(address: EthAddress): CoreDbRc[void] = stoDeleteFn: proc(cAcc: CoreDbAccRef, address: EthAddress): CoreDbRc[void] =
accStoDelete(address), accStoDelete(AristoCoreDbAccRef(cAcc), address),
mergeFn: proc(acc: CoreDbAccount): CoreDbRc[void] = mergeFn: proc(cAcc: CoreDbAccRef, acc: CoreDbAccount): CoreDbRc[void] =
accMerge(acc), accMerge(AristoCoreDbAccRef(cAcc), acc),
hasPathFn: proc(address: EthAddress): CoreDbRc[bool] = hasPathFn: proc(cAcc: CoreDbAccRef, address: EthAddress): CoreDbRc[bool] =
accHasPath(address), accHasPath(AristoCoreDbAccRef(cAcc), address),
getColFn: proc(): CoreDbColRef = getColFn: proc(cAcc: CoreDbAccRef): CoreDbColRef =
getColFn()) getColFn(AristoCoreDbAccRef(cAcc)))
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Private context call back functions # Private context call back functions
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
proc ctxMethods(cCtx: AristoCoreDbCtxRef): CoreDbCtxFns = proc ctxMethods(cCtx: AristoCoreDbCtxRef): CoreDbCtxFns =
let template base: untyped = cCtx.base
cCtx = cCtx # So it can savely be captured template db: untyped = base.parent
base = cCtx.base # Will not change and can be captured template api: untyped = base.api
db = base.parent # Ditto template mpt: untyped = cCtx.mpt
api = base.api # Ditto
mpt = cCtx.mpt # Ditto
proc ctxNewCol( proc ctxNewCol(
cCtx: AristoCoreDbCtxRef,
colType: CoreDbColType; colType: CoreDbColType;
colState: Hash256; colState: Hash256;
address: Opt[EthAddress]; address: Opt[EthAddress];
@ -463,7 +459,7 @@ proc ctxMethods(cCtx: AristoCoreDbCtxRef): CoreDbCtxFns =
err(aristo.GenericError.toError(base, info, RootNotFound)) err(aristo.GenericError.toError(base, info, RootNotFound))
proc ctxGetMpt(col: CoreDbColRef): CoreDbRc[CoreDbMptRef] = proc ctxGetMpt(cCtx: AristoCoreDbCtxRef, col: CoreDbColRef): CoreDbRc[CoreDbMptRef] =
const const
info = "ctx/getMptFn()" info = "ctx/getMptFn()"
let let
@ -505,10 +501,10 @@ proc ctxMethods(cCtx: AristoCoreDbCtxRef): CoreDbCtxFns =
col.reset = false col.reset = false
newMpt.base = base newMpt.base = base
newMpt.methods = newMpt.mptMethods() newMpt.methods = mptMethods()
ok(db.bless newMpt) ok(db.bless newMpt)
proc ctxGetAcc(col: CoreDbColRef): CoreDbRc[CoreDbAccRef] = proc ctxGetAcc(cCtx: AristoCoreDbCtxRef, col: CoreDbColRef): CoreDbRc[CoreDbAccRef] =
const info = "getAccFn()" const info = "getAccFn()"
let col = AristoColRef(col) let col = AristoColRef(col)
@ -517,31 +513,32 @@ proc ctxMethods(cCtx: AristoCoreDbCtxRef): CoreDbCtxFns =
return err(error.toError(base, info, RootUnacceptable)) return err(error.toError(base, info, RootUnacceptable))
let acc = AristoCoreDbAccRef(base: base) let acc = AristoCoreDbAccRef(base: base)
acc.methods = acc.accMethods() acc.methods = accMethods()
ok(db.bless acc) ok(db.bless acc)
proc ctxForget() = proc ctxForget(cCtx: AristoCoreDbCtxRef) =
api.forget(mpt).isOkOr: api.forget(mpt).isOkOr:
raiseAssert "forgetFn(): " & $error raiseAssert "forgetFn(): " & $error
CoreDbCtxFns( CoreDbCtxFns(
newColFn: proc( newColFn: proc(
cCtx: CoreDbCtxRef;
col: CoreDbColType; col: CoreDbColType;
colState: Hash256; colState: Hash256;
address: Opt[EthAddress]; address: Opt[EthAddress];
): CoreDbRc[CoreDbColRef] = ): CoreDbRc[CoreDbColRef] =
ctxNewCol(col, colState, address), ctxNewCol(AristoCoreDbCtxRef(cCtx), col, colState, address),
getMptFn: proc(col: CoreDbColRef): CoreDbRc[CoreDbMptRef] = getMptFn: proc(cCtx: CoreDbCtxRef, col: CoreDbColRef): CoreDbRc[CoreDbMptRef] =
ctxGetMpt(col), ctxGetMpt(AristoCoreDbCtxRef(cCtx), col),
getAccFn: proc(col: CoreDbColRef): CoreDbRc[CoreDbAccRef] = getAccFn: proc(cCtx: CoreDbCtxRef, col: CoreDbColRef): CoreDbRc[CoreDbAccRef] =
ctxGetAcc(col), ctxGetAcc(AristoCoreDbCtxRef(cCtx), col),
forgetFn: proc() = forgetFn: proc(cCtx: CoreDbCtxRef) =
ctxForget()) ctxForget(AristoCoreDbCtxRef(cCtx)))
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
# Public handlers and helpers # Public handlers and helpers

60
nimbus/db/core_db/base.nim
View File

@ -11,7 +11,6 @@
{.push raises: [].} {.push raises: [].}
import import
chronicles,
eth/common, eth/common,
"../.."/[constants, errors], "../.."/[constants, errors],
./base/[api_tracking, base_desc] ./base/[api_tracking, base_desc]
@ -205,13 +204,20 @@ proc parent*[T: CoreDbKvtRef |
## ##
result = child.parent result = child.parent
proc backend*(dsc: CoreDbKvtRef | CoreDbMptRef): auto = proc backend*(dsc: CoreDbKvtRef): auto =
## Getter, retrieves the *raw* backend object for special/localised support. ## Getter, retrieves the *raw* backend object for special/localised support.
## ##
dsc.setTrackNewApi AnyBackendFn dsc.setTrackNewApi AnyBackendFn
result = dsc.methods.backendFn() result = dsc.methods.backendFn()
dsc.ifTrackNewApi: debug newApiTxt, api, elapsed dsc.ifTrackNewApi: debug newApiTxt, api, elapsed
proc backend*(mpt: CoreDbMptRef): auto =
## Getter, retrieves the *raw* backend object for special/localised support.
##
mpt.setTrackNewApi AnyBackendFn
result = mpt.methods.backendFn(mpt)
mpt.ifTrackNewApi: debug newApiTxt, api, elapsed
proc finish*(db: CoreDbRef; eradicate = false) = proc finish*(db: CoreDbRef; eradicate = false) =
## Database destructor. If the argument `eradicate` is set `false`, the ## Database destructor. If the argument `eradicate` is set `false`, the
## database is left as-is and only the in-memory handlers are cleaned up. ## database is left as-is and only the in-memory handlers are cleaned up.
@ -337,7 +343,7 @@ proc forget*(ctx: CoreDbCtxRef) =
## context. This function fails if `ctx` is the default context. ## context. This function fails if `ctx` is the default context.
## ##
ctx.setTrackNewApi CtxForgetFn ctx.setTrackNewApi CtxForgetFn
ctx.methods.forgetFn() ctx.methods.forgetFn(ctx)
ctx.ifTrackNewApi: debug newApiTxt, api, elapsed ctx.ifTrackNewApi: debug newApiTxt, api, elapsed
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -381,7 +387,7 @@ proc newColumn*(
## db.getAcc col ## db.getAcc col
## ##
ctx.setTrackNewApi CtxNewColFn ctx.setTrackNewApi CtxNewColFn
result = ctx.methods.newColFn(colType, colState, address) result = ctx.methods.newColFn(ctx, colType, colState, address)
ctx.ifTrackNewApi: ctx.ifTrackNewApi:
debug newApiTxt, api, elapsed, colType, colState, address, result debug newApiTxt, api, elapsed, colType, colState, address, result
@ -393,7 +399,7 @@ proc newColumn*(
## Shortcut for `ctx.newColumn(CtStorage,colState,some(address))`. ## Shortcut for `ctx.newColumn(CtStorage,colState,some(address))`.
## ##
ctx.setTrackNewApi CtxNewColFn ctx.setTrackNewApi CtxNewColFn
result = ctx.methods.newColFn(CtStorage, colState, Opt.some(address)) result = ctx.methods.newColFn(ctx, CtStorage, colState, Opt.some(address))
ctx.ifTrackNewApi: debug newApiTxt, api, elapsed, colState, address, result ctx.ifTrackNewApi: debug newApiTxt, api, elapsed, colState, address, result
proc newColumn*( proc newColumn*(
@ -406,7 +412,7 @@ proc newColumn*(
## ##
ctx.setTrackNewApi CtxNewColFn ctx.setTrackNewApi CtxNewColFn
result = ctx.methods.newColFn( result = ctx.methods.newColFn(
CtStorage, EMPTY_ROOT_HASH, Opt.some(address)).valueOr: ctx, CtStorage, EMPTY_ROOT_HASH, Opt.some(address)).valueOr:
raiseAssert error.prettyText() raiseAssert error.prettyText()
ctx.ifTrackNewApi: debug newApiTxt, api, elapsed, address, result ctx.ifTrackNewApi: debug newApiTxt, api, elapsed, address, result
@ -474,7 +480,7 @@ proc getMpt*(
## function `getColumn()`. ## function `getColumn()`.
## ##
ctx.setTrackNewApi CtxGetMptFn ctx.setTrackNewApi CtxGetMptFn
result = ctx.methods.getMptFn col result = ctx.methods.getMptFn(ctx, col)
ctx.ifTrackNewApi: debug newApiTxt, api, elapsed, col, result ctx.ifTrackNewApi: debug newApiTxt, api, elapsed, col, result
proc getMpt*( proc getMpt*(
@ -487,8 +493,8 @@ proc getMpt*(
## return a non-nil descriptor or throw an exception. ## return a non-nil descriptor or throw an exception.
## ##
ctx.setTrackNewApi CtxGetMptFn ctx.setTrackNewApi CtxGetMptFn
let col = ctx.methods.newColFn(colType, EMPTY_ROOT_HASH, address).value let col = ctx.methods.newColFn(ctx, colType, EMPTY_ROOT_HASH, address).value
result = ctx.methods.getMptFn(col).valueOr: result = ctx.methods.getMptFn(ctx, col).valueOr:
raiseAssert error.prettyText() raiseAssert error.prettyText()
ctx.ifTrackNewApi: debug newApiTxt, api, colType, elapsed ctx.ifTrackNewApi: debug newApiTxt, api, colType, elapsed
@ -500,7 +506,7 @@ proc getMpt*(acc: CoreDbAccRef): CoreDbMptRef =
## argument. ## argument.
## ##
acc.setTrackNewApi AccToMptFn acc.setTrackNewApi AccToMptFn
result = acc.methods.getMptFn().valueOr: result = acc.methods.getMptFn(acc).valueOr:
raiseAssert error.prettyText() raiseAssert error.prettyText()
acc.ifTrackNewApi: acc.ifTrackNewApi:
let colState = result.methods.getColFn() let colState = result.methods.getColFn()
@ -526,7 +532,7 @@ proc getAcc*(
## This function works similar to `getMpt()` for handling accounts. ## This function works similar to `getMpt()` for handling accounts.
## ##
ctx.setTrackNewApi CtxGetAccFn ctx.setTrackNewApi CtxGetAccFn
result = ctx.methods.getAccFn col result = ctx.methods.getAccFn(ctx, col)
ctx.ifTrackNewApi: debug newApiTxt, api, elapsed, col, result ctx.ifTrackNewApi: debug newApiTxt, api, elapsed, col, result
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -537,14 +543,14 @@ proc getColumn*(acc: CoreDbAccRef): CoreDbColRef =
## Getter, result is not `nil` ## Getter, result is not `nil`
## ##
acc.setTrackNewApi AccGetColFn acc.setTrackNewApi AccGetColFn
result = acc.methods.getColFn() result = acc.methods.getColFn(acc)
acc.ifTrackNewApi: debug newApiTxt, api, elapsed, result acc.ifTrackNewApi: debug newApiTxt, api, elapsed, result
proc getColumn*(mpt: CoreDbMptRef): CoreDbColRef = proc getColumn*(mpt: CoreDbMptRef): CoreDbColRef =
## Variant of `getColumn()` ## Variant of `getColumn()`
## ##
mpt.setTrackNewApi MptGetColFn mpt.setTrackNewApi MptGetColFn
result = mpt.methods.getColFn() result = mpt.methods.getColFn(mpt)
mpt.ifTrackNewApi: debug newApiTxt, api, elapsed, result mpt.ifTrackNewApi: debug newApiTxt, api, elapsed, result
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -556,9 +562,9 @@ proc fetch*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[Blob] =
## non-empty `Blob` or an error code. ## non-empty `Blob` or an error code.
## ##
mpt.setTrackNewApi MptFetchFn mpt.setTrackNewApi MptFetchFn
result = mpt.methods.fetchFn key result = mpt.methods.fetchFn(mpt, key)
mpt.ifTrackNewApi: mpt.ifTrackNewApi:
let col = mpt.methods.getColFn() let col = mpt.methods.getColFn(mpt)
debug newApiTxt, api, elapsed, col, key=key.toStr, result debug newApiTxt, api, elapsed, col, key=key.toStr, result
proc fetchOrEmpty*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[Blob] = proc fetchOrEmpty*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[Blob] =
@ -566,16 +572,16 @@ proc fetchOrEmpty*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[Blob] =
## on the database. ## on the database.
## ##
mpt.setTrackNewApi MptFetchOrEmptyFn mpt.setTrackNewApi MptFetchOrEmptyFn
result = mpt.methods.fetchFn key result = mpt.methods.fetchFn(mpt, key)
if result.isErr and result.error.error == MptNotFound: if result.isErr and result.error.error == MptNotFound:
result = CoreDbRc[Blob].ok(EmptyBlob) result = CoreDbRc[Blob].ok(EmptyBlob)
mpt.ifTrackNewApi: mpt.ifTrackNewApi:
let col = mpt.methods.getColFn() let col = mpt.methods.getColFn(mpt)
debug newApiTxt, api, elapsed, col, key=key.toStr, result debug newApiTxt, api, elapsed, col, key=key.toStr, result
proc delete*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[void] = proc delete*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[void] =
mpt.setTrackNewApi MptDeleteFn mpt.setTrackNewApi MptDeleteFn
result = mpt.methods.deleteFn key result = mpt.methods.deleteFn(mpt, key)
mpt.ifTrackNewApi: mpt.ifTrackNewApi:
let col = mpt.methods.getColFn() let col = mpt.methods.getColFn()
debug newApiTxt, api, elapsed, col, key=key.toStr, result debug newApiTxt, api, elapsed, col, key=key.toStr, result
@ -586,9 +592,9 @@ proc merge*(
val: openArray[byte]; val: openArray[byte];
): CoreDbRc[void] = ): CoreDbRc[void] =
mpt.setTrackNewApi MptMergeFn mpt.setTrackNewApi MptMergeFn
result = mpt.methods.mergeFn(key, val) result = mpt.methods.mergeFn(mpt, key, val)
mpt.ifTrackNewApi: mpt.ifTrackNewApi:
let col = mpt.methods.getColFn() let col = mpt.methods.getColFn(mpt)
debug newApiTxt, api, elapsed, col, key=key.toStr, val=val.toLenStr, result debug newApiTxt, api, elapsed, col, key=key.toStr, val=val.toLenStr, result
proc hasPath*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[bool] = proc hasPath*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[bool] =
@ -596,9 +602,9 @@ proc hasPath*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[bool] =
## than a `Result[]`. ## than a `Result[]`.
## ##
mpt.setTrackNewApi MptHasPathFn mpt.setTrackNewApi MptHasPathFn
result = mpt.methods.hasPathFn key result = mpt.methods.hasPathFn(mpt, key)
mpt.ifTrackNewApi: mpt.ifTrackNewApi:
let col = mpt.methods.getColFn() let col = mpt.methods.getColFn(mpt)
debug newApiTxt, api, elapsed, col, key=key.toStr, result debug newApiTxt, api, elapsed, col, key=key.toStr, result
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
@ -609,7 +615,7 @@ proc fetch*(acc: CoreDbAccRef; address: EthAddress): CoreDbRc[CoreDbAccount] =
## Fetch data from the argument `acc`. ## Fetch data from the argument `acc`.
## ##
acc.setTrackNewApi AccFetchFn acc.setTrackNewApi AccFetchFn
result = acc.methods.fetchFn address result = acc.methods.fetchFn(acc, address)
acc.ifTrackNewApi: acc.ifTrackNewApi:
let storage = if result.isErr: "n/a" else: result.value.storage.prettyText() let storage = if result.isErr: "n/a" else: result.value.storage.prettyText()
debug newApiTxt, api, elapsed, address, storage, result debug newApiTxt, api, elapsed, address, storage, result
@ -617,7 +623,7 @@ proc fetch*(acc: CoreDbAccRef; address: EthAddress): CoreDbRc[CoreDbAccount] =
proc delete*(acc: CoreDbAccRef; address: EthAddress): CoreDbRc[void] = proc delete*(acc: CoreDbAccRef; address: EthAddress): CoreDbRc[void] =
acc.setTrackNewApi AccDeleteFn acc.setTrackNewApi AccDeleteFn
result = acc.methods.deleteFn address result = acc.methods.deleteFn(acc, address)
acc.ifTrackNewApi: debug newApiTxt, api, elapsed, address, result acc.ifTrackNewApi: debug newApiTxt, api, elapsed, address, result
proc stoDelete*(acc: CoreDbAccRef; address: EthAddress): CoreDbRc[void] = proc stoDelete*(acc: CoreDbAccRef; address: EthAddress): CoreDbRc[void] =
@ -632,7 +638,7 @@ proc stoDelete*(acc: CoreDbAccRef; address: EthAddress): CoreDbRc[void] =
## backend. ## backend.
## ##
acc.setTrackNewApi AccStoDeleteFn acc.setTrackNewApi AccStoDeleteFn
result = acc.methods.stoDeleteFn address result = acc.methods.stoDeleteFn(acc, address)
acc.ifTrackNewApi: debug newApiTxt, api, elapsed, address, result acc.ifTrackNewApi: debug newApiTxt, api, elapsed, address, result
@ -641,7 +647,7 @@ proc merge*(
account: CoreDbAccount; account: CoreDbAccount;
): CoreDbRc[void] = ): CoreDbRc[void] =
acc.setTrackNewApi AccMergeFn acc.setTrackNewApi AccMergeFn
result = acc.methods.mergeFn account result = acc.methods.mergeFn(acc, account)
acc.ifTrackNewApi: acc.ifTrackNewApi:
let address = account.address let address = account.address
debug newApiTxt, api, elapsed, address, result debug newApiTxt, api, elapsed, address, result
@ -651,7 +657,7 @@ proc hasPath*(acc: CoreDbAccRef; address: EthAddress): CoreDbRc[bool] =
## Would be named `contains` if it returned `bool` rather than `Result[]`. ## Would be named `contains` if it returned `bool` rather than `Result[]`.
## ##
acc.setTrackNewApi AccHasPathFn acc.setTrackNewApi AccHasPathFn
result = acc.methods.hasPathFn address result = acc.methods.hasPathFn(acc, address)
acc.ifTrackNewApi: debug newApiTxt, api, elapsed, address, result acc.ifTrackNewApi: debug newApiTxt, api, elapsed, address, result

43
nimbus/db/core_db/base/base_desc.nim
View File

@ -160,16 +160,14 @@ type
# -------------------------------------------------- # --------------------------------------------------
# Sub-descriptor: MPT context methods # Sub-descriptor: MPT context methods
# -------------------------------------------------- # --------------------------------------------------
CoreDbCtxFromTxFn* =
proc(root: Hash256; kind: CoreDbColType): CoreDbRc[CoreDbCtxRef] {.noRaise.}
CoreDbCtxNewColFn* = proc( CoreDbCtxNewColFn* = proc(
colType: CoreDbColType; colState: Hash256; address: Opt[EthAddress]; cCtx: CoreDbCtxRef; colType: CoreDbColType; colState: Hash256; address: Opt[EthAddress];
): CoreDbRc[CoreDbColRef] {.noRaise.} ): CoreDbRc[CoreDbColRef] {.noRaise.}
CoreDbCtxGetMptFn* = proc( CoreDbCtxGetMptFn* = proc(
root: CoreDbColRef): CoreDbRc[CoreDbMptRef] {.noRaise.} cCtx: CoreDbCtxRef; root: CoreDbColRef): CoreDbRc[CoreDbMptRef] {.noRaise.}
CoreDbCtxGetAccFn* = proc( CoreDbCtxGetAccFn* = proc(
root: CoreDbColRef): CoreDbRc[CoreDbAccRef] {.noRaise.} cCtx: CoreDbCtxRef; root: CoreDbColRef): CoreDbRc[CoreDbAccRef] {.noRaise.}
CoreDbCtxForgetFn* = proc() {.noRaise.} CoreDbCtxForgetFn* = proc(cCtx: CoreDbCtxRef) {.noRaise.}
CoreDbCtxFns* = object CoreDbCtxFns* = object
## Methods for context maniulation ## Methods for context maniulation
@ -181,20 +179,20 @@ type
# -------------------------------------------------- # --------------------------------------------------
# Sub-descriptor: generic Mpt/hexary trie methods # Sub-descriptor: generic Mpt/hexary trie methods
# -------------------------------------------------- # --------------------------------------------------
CoreDbMptBackendFn* = proc(): CoreDbMptBackendRef {.noRaise.} CoreDbMptBackendFn* = proc(cMpt: CoreDbMptRef): CoreDbMptBackendRef {.noRaise.}
CoreDbMptFetchFn* = CoreDbMptFetchFn* =
proc(k: openArray[byte]): CoreDbRc[Blob] {.noRaise.} proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[Blob] {.noRaise.}
CoreDbMptFetchAccountFn* = CoreDbMptFetchAccountFn* =
proc(k: openArray[byte]): CoreDbRc[CoreDbAccount] {.noRaise.} proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[CoreDbAccount] {.noRaise.}
CoreDbMptDeleteFn* = CoreDbMptDeleteFn* =
proc(k: openArray[byte]): CoreDbRc[void] {.noRaise.} proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[void] {.noRaise.}
CoreDbMptMergeFn* = CoreDbMptMergeFn* =
proc(k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] {.noRaise.} proc(cMpt: CoreDbMptRef, k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] {.noRaise.}
CoreDbMptMergeAccountFn* = CoreDbMptMergeAccountFn* =
proc(k: openArray[byte]; v: CoreDbAccount): CoreDbRc[void] {.noRaise.} proc(cMpt: CoreDbMptRef, k: openArray[byte]; v: CoreDbAccount): CoreDbRc[void] {.noRaise.}
CoreDbMptHasPathFn* = proc(k: openArray[byte]): CoreDbRc[bool] {.noRaise.} CoreDbMptHasPathFn* = proc(cMpt: CoreDbMptRef, k: openArray[byte]): CoreDbRc[bool] {.noRaise.}
CoreDbMptGetColFn* = proc(): CoreDbColRef {.noRaise.} CoreDbMptGetColFn* = proc(cMpt: CoreDbMptRef): CoreDbColRef {.noRaise.}
CoreDbMptForgetFn* = proc(): CoreDbRc[void] {.noRaise.} CoreDbMptForgetFn* = proc(cMpt: CoreDbMptRef): CoreDbRc[void] {.noRaise.}
CoreDbMptFns* = object CoreDbMptFns* = object
## Methods for trie objects ## Methods for trie objects
@ -209,14 +207,13 @@ type
# ---------------------------------------------------- # ----------------------------------------------------
# Sub-descriptor: Mpt/hexary trie methods for accounts # Sub-descriptor: Mpt/hexary trie methods for accounts
# ------------------------------------------------------ # ------------------------------------------------------
CoreDbAccGetMptFn* = proc(): CoreDbRc[CoreDbMptRef] {.noRaise.} CoreDbAccGetMptFn* = proc(cAcc: CoreDbAccRef): CoreDbRc[CoreDbMptRef] {.noRaise.}
CoreDbAccFetchFn* = proc(k: EthAddress): CoreDbRc[CoreDbAccount] {.noRaise.} CoreDbAccFetchFn* = proc(cAcc: CoreDbAccRef, k: EthAddress): CoreDbRc[CoreDbAccount] {.noRaise.}
CoreDbAccDeleteFn* = proc(k: EthAddress): CoreDbRc[void] {.noRaise.} CoreDbAccDeleteFn* = proc(cAcc: CoreDbAccRef, k: EthAddress): CoreDbRc[void] {.noRaise.}
CoreDbAccStoDeleteFn* = proc(k: EthAddress): CoreDbRc[void] {.noRaise.} CoreDbAccStoDeleteFn* = proc(cAcc: CoreDbAccRef,k: EthAddress): CoreDbRc[void] {.noRaise.}
CoreDbAccMergeFn* = proc(v: CoreDbAccount): CoreDbRc[void] {.noRaise.} CoreDbAccMergeFn* = proc(cAcc: CoreDbAccRef, v: CoreDbAccount): CoreDbRc[void] {.noRaise.}
CoreDbAccHasPathFn* = proc(k: EthAddress): CoreDbRc[bool] {.noRaise.} CoreDbAccHasPathFn* = proc(cAcc: CoreDbAccRef, k: EthAddress): CoreDbRc[bool] {.noRaise.}
CoreDbAccGetColFn* = proc(): CoreDbColRef {.noRaise.} CoreDbAccGetColFn* = proc(cAcc: CoreDbAccRef): CoreDbColRef {.noRaise.}
CoreDbAccForgetFn* = proc(): CoreDbRc[void] {.noRaise.}
CoreDbAccFns* = object CoreDbAccFns* = object
## Methods for trie objects ## Methods for trie objects

6
nimbus/nimbus_import.nim
View File

@ -21,6 +21,9 @@ import
./db/era1_db, ./db/era1_db,
beacon_chain/era_db beacon_chain/era_db
declareGauge nec_import_block_number,
"Latest imported block number"
declareCounter nec_imported_blocks, declareCounter nec_imported_blocks,
"Blocks processed during import" "Blocks processed during import"
@ -105,6 +108,8 @@ proc importBlocks*(conf: NimbusConf, com: CommonRef) =
if csv != nil: if csv != nil:
close(csv) close(csv)
nec_import_block_number.set(start.int64)
template blockNumber(): uint64 = template blockNumber(): uint64 =
start + imported start + imported
@ -155,6 +160,7 @@ proc importBlocks*(conf: NimbusConf, com: CommonRef) =
avgMGps = f(gas.float / 1000000 / diff0), avgMGps = f(gas.float / 1000000 / diff0),
elapsed = shortLog(time2 - time0, 3) elapsed = shortLog(time2 - time0, 3)
metrics.set(nec_import_block_number, int64(blockNumber))
nec_imported_blocks.inc(blocks.len) nec_imported_blocks.inc(blocks.len)
nec_imported_transactions.inc(statsRes[].txs) nec_imported_transactions.inc(statsRes[].txs)
nec_imported_gas.inc(statsRes[].gas) nec_imported_gas.inc(statsRes[].gas)