Separate constructor helpers for core db and ledger (#2480)

* Extract `CoreDb` constructor helpers from `base.nim` into separate module

why:
  This makes it easier to avoid circular imports.

* Extract `Ledger` constructor helpers from `base.nim` into separate module

why:
  Move `accounts_ledger.nim` file to sub-folder `backend`. That way the
  layout resembles that of the `core_db`.

nimbus/db/core_db/backend/aristo_db.nim

@@ -15,8 +15,7 @@ import
   ../../aristo/[aristo_init/memory_only, aristo_walk],
   ../../kvt as use_kvt,
   ../../kvt/[kvt_init/memory_only, kvt_walk],
-  ../base,
+  ../base/[base_config, base_desc, base_helpers]
-  ../base/[base_config, base_desc]
 
 # ------------------------------------------------------------------------------
 # Public constructor and helper

nimbus/db/core_db/base.nim

@@ -15,7 +15,7 @@ import
   eth/common,
   "../.."/[constants, errors],
   ".."/[kvt, aristo],
-  ./base/[api_tracking, base_config, base_desc]
+  ./base/[api_tracking, base_config, base_desc, base_helpers]
 
 export
   CoreDbAccRef,
@@ -33,18 +33,6 @@ export
   CoreDbTxRef,
   CoreDbType
 
-when CoreDbAutoValidateDescriptors:
-  import
-    ./base/base_validate
-
-when CoreDbEnableApiJumpTable:
-  discard
-else:
-  import
-    ../aristo/[
-      aristo_delete, aristo_desc, aristo_fetch, aristo_merge, aristo_tx],
-    ../kvt/[kvt_desc, kvt_utils, kvt_tx]
-
 when CoreDbEnableApiTracking:
   {.warning: "*** Provided API logging for CoreDB (disabled by default)".}
   import
@@ -54,102 +42,21 @@ when CoreDbEnableApiTracking:
   const
     logTxt = "API"
 
+
 when CoreDbEnableProfiling:
-  {.warning: "*** Enabled API profiling for CoreDB".}
+  {.warning: "*** Enabled profiling for CoreDB (also tracer API available)".}
   export
     CoreDbFnInx,
     CoreDbProfListRef
 
-# ------------------------------------------------------------------------------
-# Private KVT helpers
-# ------------------------------------------------------------------------------
 
-template kvt(dsc: CoreDbKvtRef): KvtDbRef =
+when CoreDbEnableApiJumpTable:
-  dsc.distinctBase.kvt
+  discard
-
+else:
-template ctx(kvt: CoreDbKvtRef): CoreDbCtxRef =
+  import
-  kvt.distinctBase
+    ../aristo/[
-
+      aristo_delete, aristo_desc, aristo_fetch, aristo_merge, aristo_tx],
-# ---------------
+    ../kvt/[kvt_desc, kvt_utils, kvt_tx]
-
-template call(api: KvtApiRef; fn: untyped; args: varArgs[untyped]): untyped =
-  when CoreDbEnableApiJumpTable:
-    api.fn(args)
-  else:
-    fn(args)
-
-template call(kvt: CoreDbKvtRef; fn: untyped; args: varArgs[untyped]): untyped =
-  kvt.distinctBase.parent.kvtApi.call(fn, args)
-
-# ---------------
-
-func toError(e: KvtError; s: string; error = Unspecified): CoreDbErrorRef =
-  CoreDbErrorRef(
-    error:    error,
-    ctx:      s,
-    isAristo: false,
-    kErr:     e)
-
-# ------------------------------------------------------------------------------
-# Private Aristo helpers
-# ------------------------------------------------------------------------------
-
-template mpt(dsc: CoreDbAccRef | CoreDbMptRef): AristoDbRef =
-  dsc.distinctBase.mpt
-
-template mpt(tx: CoreDbTxRef): AristoDbRef =
-  tx.ctx.mpt
-
-template ctx(acc: CoreDbAccRef): CoreDbCtxRef =
-  acc.distinctBase
-
-# ---------------
-
-template call(api: AristoApiRef; fn: untyped; args: varArgs[untyped]): untyped =
-  when CoreDbEnableApiJumpTable:
-    api.fn(args)
-  else:
-    fn(args)
-
-template call(
-    acc: CoreDbAccRef | CoreDbMptRef;
-    fn: untyped;
-    args: varArgs[untyped];
-      ): untyped =
-  acc.distinctBase.parent.ariApi.call(fn, args)
-
-# ---------------
-
-func toError(e: AristoError; s: string; error = Unspecified): CoreDbErrorRef =
-  CoreDbErrorRef(
-    error:    error,
-    ctx:      s,
-    isAristo: true,
-    aErr:     e)
-
-# ------------------------------------------------------------------------------
-# Public constructor helper
-# ------------------------------------------------------------------------------
-
-proc bless*(db: CoreDbRef): CoreDbRef =
-  ## Verify descriptor
-  when CoreDbAutoValidateDescriptors:
-    db.validate
-  when CoreDbEnableProfiling:
-    db.profTab = CoreDbProfListRef.init()
-  db
-
-proc bless*(db: CoreDbRef; ctx: CoreDbCtxRef): CoreDbCtxRef =
-  ctx.parent = db
-  when CoreDbAutoValidateDescriptors:
-    ctx.validate
-  ctx
-
-proc bless*(ctx: CoreDbCtxRef; dsc: CoreDbMptRef | CoreDbTxRef): auto =
-  dsc.ctx = ctx
-  when CoreDbAutoValidateDescriptors:
-    dsc.validate
-  dsc
 
 # ------------------------------------------------------------------------------
 # Public context constructors and administration

nimbus/db/core_db/base/api_tracking.nim

@@ -23,7 +23,6 @@ type
     ## Needed for local `$` as it would be ambiguous for `Duration`
 
   CoreDbApiTrackRef* =
-    # CoreDbCaptRef |
     CoreDbRef | CoreDbKvtRef | CoreDbCtxRef | CoreDbMptRef | CoreDbAccRef |
     CoreDbTxRef
 
@@ -90,7 +89,7 @@ type
     TxRollbackFn        = "rollback"
     TxSaveDisposeFn     = "safeDispose"
 
-proc toStr*(e: CoreDbErrorRef): string {.gcsafe.}
+func toStr*(e: CoreDbErrorRef): string {.gcsafe.}
 
 # ------------------------------------------------------------------------------
 # Private helpers
@@ -113,44 +112,43 @@ func toLenStr(w: Blob): string =
 func toStr(ela: Duration): string =
   aristo_profile.toStr(ela)
 
-proc toStr*(rc: CoreDbRc[int]|CoreDbRc[UInt256]): string =
+func toStr*(rc: CoreDbRc[int]|CoreDbRc[UInt256]): string =
   if rc.isOk: "ok(" & $rc.value & ")" else: "err(" & rc.error.toStr & ")"
 
-proc toStr(rc: CoreDbRc[bool]): string =
+func toStr(rc: CoreDbRc[bool]): string =
   if rc.isOk: "ok(" & $rc.value & ")" else: "err(" & rc.error.toStr & ")"
 
-proc toStr(rc: CoreDbRc[void]): string =
+func toStr(rc: CoreDbRc[void]): string =
   if rc.isOk: "ok()" else: "err(" & rc.error.toStr & ")"
 
-proc toStr(rc: CoreDbRc[Blob]): string =
+func toStr(rc: CoreDbRc[Blob]): string =
   if rc.isOk: "ok(Blob[" & $rc.value.len & "])"
   else: "err(" & rc.error.toStr & ")"
 
-proc toStr(rc: CoreDbRc[Hash256]): string =
+func toStr(rc: CoreDbRc[Hash256]): string =
   if rc.isOk: "ok(" & rc.value.toStr & ")" else: "err(" & rc.error.toStr & ")"
 
-proc toStr(rc: CoreDbRc[Account]): string =
+func toStr(rc: CoreDbRc[Account]): string =
   if rc.isOk: "ok(Account)" else: "err(" & rc.error.toStr & ")"
 
-proc toStr(rc: CoreDbRc[CoreDbAccount]): string =
+func toStr(rc: CoreDbRc[CoreDbAccount]): string =
   if rc.isOk: "ok(AristoAccount)" else: "err(" & rc.error.toStr & ")"
 
-proc toStr[T](rc: CoreDbRc[T]; ifOk: static[string]): string =
+func toStr[T](rc: CoreDbRc[T]; ifOk: static[string]): string =
   if rc.isOk: "ok(" & ifOk & ")" else: "err(" & rc.error.toStr & ")"
 
-proc toStr(rc: CoreDbRc[CoreDbRef]): string = rc.toStr "db"
+func toStr(rc: CoreDbRc[CoreDbRef]): string = rc.toStr "db"
-proc toStr(rc: CoreDbRc[CoreDbKvtRef]): string = rc.toStr "kvt"
+func toStr(rc: CoreDbRc[CoreDbKvtRef]): string = rc.toStr "kvt"
-proc toStr(rc: CoreDbRc[CoreDbTxRef]): string = rc.toStr "tx"
+func toStr(rc: CoreDbRc[CoreDbTxRef]): string = rc.toStr "tx"
-#proc toStr(rc: CoreDbRc[CoreDbCaptRef]): string = rc.toStr "capt"
+func toStr(rc: CoreDbRc[CoreDbCtxRef]): string = rc.toStr "ctx"
-proc toStr(rc: CoreDbRc[CoreDbCtxRef]): string = rc.toStr "ctx"
+func toStr(rc: CoreDbRc[CoreDbMptRef]): string = rc.toStr "mpt"
-proc toStr(rc: CoreDbRc[CoreDbMptRef]): string = rc.toStr "mpt"
+func toStr(rc: CoreDbRc[CoreDbAccRef]): string = rc.toStr "acc"
-proc toStr(rc: CoreDbRc[CoreDbAccRef]): string = rc.toStr "acc"
 
 # ------------------------------------------------------------------------------
 # Public API logging helpers
 # ------------------------------------------------------------------------------
 
-proc toStr*(e: CoreDbErrorRef): string =
+func toStr*(e: CoreDbErrorRef): string =
   result = $e.error & "("
   result &= (if e.isAristo: "Aristo" else: "Kvt")
   result &= ", ctx=" & $e.ctx & ", error="
@@ -164,7 +162,7 @@ func toLenStr*(w: openArray[byte]): string =
   if 0 < w.len and w.len < 5: "<" & w.oaToStr & ">"
   else: "openArray[" & $w.len & "]"
 
-proc `$`*[T](rc: CoreDbRc[T]): string = rc.toStr
+func `$`*[T](rc: CoreDbRc[T]): string = rc.toStr
 func `$`*(t: Elapsed): string = t.Duration.toStr
 func `$`*(e: EthAddress): string = e.toStr
 func `$$`*(h: Hash256): string = h.toStr # otherwise collision w/existing `$`

nimbus/db/core_db/base/base_desc.nim

@@ -15,10 +15,6 @@ import
   "../.."/[aristo, aristo/aristo_profile, kvt],
   ./base_config
 
-# Annotation helpers
-{.pragma:  noRaise, gcsafe, raises: [].}
-{.pragma: apiRaise, gcsafe, raises: [CoreDbApiError].}
-
 type
   CoreDbType* = enum
     Ooops
@@ -62,10 +58,6 @@ type
     StoNotFound
     TxPending
 
-  CoreDbCaptFlags* {.pure.} = enum
-    PersistPut
-    PersistDel
-
   # --------------------------------------------------
   # Production descriptors
   # --------------------------------------------------
@@ -118,27 +110,6 @@ type
     else:
       kErr*: KvtError
 
-when false: # TODO
-  type
-    # --------------------------------------------------
-    # Sub-descriptor: capture recorder methods
-    # --------------------------------------------------
-    CoreDbCaptRecorderFn* = proc(): CoreDbRef {.noRaise.}
-    CoreDbCaptLogDbFn* = proc(): TableRef[Blob,Blob] {.noRaise.}
-    CoreDbCaptFlagsFn* = proc(): set[CoreDbCaptFlags] {.noRaise.}
-    CoreDbCaptForgetFn* = proc() {.noRaise.}
-
-    CoreDbCaptFns* = object
-      recorderFn*: CoreDbCaptRecorderFn
-      logDbFn*: CoreDbCaptLogDbFn
-      getFlagsFn*: CoreDbCaptFlagsFn
-      forgetFn*: CoreDbCaptForgetFn
-
-    CoreDbCaptRef* = ref object
-      ## Db transaction tracer derived from `CoreDbRef`
-      parent*: CoreDbRef
-      methods*: CoreDbCaptFns
-
 # ------------------------------------------------------------------------------
 # End
 # ------------------------------------------------------------------------------

nimbus/db/core_db/base/base_helpers.nim

@@ -0,0 +1,118 @@
+# Nimbus
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+{.push raises: [].}
+
+import
+  "../.."/[aristo, kvt],
+  "."/[base_config, base_desc]
+
+when CoreDbAutoValidateDescriptors:
+  import
+    ./base_validate
+
+when CoreDbEnableProfiling:
+  import
+    ./api_tracking
+
+# ------------------------------------------------------------------------------
+# Public constructor helper
+# ------------------------------------------------------------------------------
+
+proc bless*(db: CoreDbRef): CoreDbRef =
+  ## Verify descriptor
+  when CoreDbAutoValidateDescriptors:
+    db.validate
+  when CoreDbEnableProfiling:
+    db.profTab = CoreDbProfListRef.init()
+  db
+
+proc bless*(db: CoreDbRef; ctx: CoreDbCtxRef): CoreDbCtxRef =
+  ctx.parent = db
+  when CoreDbAutoValidateDescriptors:
+    ctx.validate
+  ctx
+
+proc bless*(ctx: CoreDbCtxRef; dsc: CoreDbMptRef | CoreDbTxRef): auto =
+  dsc.ctx = ctx
+  when CoreDbAutoValidateDescriptors:
+    dsc.validate
+  dsc
+
+# ------------------------------------------------------------------------------
+# Public KVT helpers
+# ------------------------------------------------------------------------------
+
+template kvt*(dsc: CoreDbKvtRef): KvtDbRef =
+  dsc.distinctBase.kvt
+
+template ctx*(kvt: CoreDbKvtRef): CoreDbCtxRef =
+  kvt.distinctBase
+
+# ---------------
+
+template call*(api: KvtApiRef; fn: untyped; args: varArgs[untyped]): untyped =
+  when CoreDbEnableApiJumpTable:
+    api.fn(args)
+  else:
+    fn(args)
+
+template call*(kvt: CoreDbKvtRef; fn: untyped; args: varArgs[untyped]): untyped =
+  kvt.distinctBase.parent.kvtApi.call(fn, args)
+
+# ---------------
+
+func toError*(e: KvtError; s: string; error = Unspecified): CoreDbErrorRef =
+  CoreDbErrorRef(
+    error:    error,
+    ctx:      s,
+    isAristo: false,
+    kErr:     e)
+
+# ------------------------------------------------------------------------------
+# Public Aristo helpers
+# ------------------------------------------------------------------------------
+
+template mpt*(dsc: CoreDbAccRef | CoreDbMptRef): AristoDbRef =
+  dsc.distinctBase.mpt
+
+template mpt*(tx: CoreDbTxRef): AristoDbRef =
+  tx.ctx.mpt
+
+template ctx*(acc: CoreDbAccRef): CoreDbCtxRef =
+  acc.distinctBase
+
+# ---------------
+
+template call*(api: AristoApiRef; fn: untyped; args: varArgs[untyped]): untyped =
+  when CoreDbEnableApiJumpTable:
+    api.fn(args)
+  else:
+    fn(args)
+
+template call*(
+    acc: CoreDbAccRef | CoreDbMptRef;
+    fn: untyped;
+    args: varArgs[untyped];
+      ): untyped =
+  acc.distinctBase.parent.ariApi.call(fn, args)
+
+# ---------------
+
+func toError*(e: AristoError; s: string; error = Unspecified): CoreDbErrorRef =
+  CoreDbErrorRef(
+    error:    error,
+    ctx:      s,
+    isAristo: true,
+    aErr:     e)
+
+# ------------------------------------------------------------------------------
+# End
+# ------------------------------------------------------------------------------

nimbus/db/ledger.nim

@@ -19,7 +19,7 @@ import
   eth/common,
   ./core_db,
   ./ledger/backend/accounts_ledger,
-  ./ledger/base/[base_config, base_desc],
+  ./ledger/base/[base_config, base_desc, base_helpers],
   ./ledger/[base, base_iterators]
 
 export

nimbus/db/ledger/base.nim

@@ -28,8 +28,6 @@ export
   LedgerRef,
   LedgerSpRef
 
-proc ldgProfData*(db: CoreDbRef): LedgerProfListRef {.gcsafe.}
-
 # ------------------------------------------------------------------------------
 # Logging/tracking helpers (some public)
 # ------------------------------------------------------------------------------
@@ -50,36 +48,10 @@ when LedgerEnableApiProfiling:
     LedgerFnInx,
     LedgerProfListRef
 
-# ------------------------------------------------------------------------------
-# Public constructor helper
-# ------------------------------------------------------------------------------
-
-proc bless*(ldg: LedgerRef; db: CoreDbRef): LedgerRef =
-  ldg.beginTrackApi LdgBlessFn
-  when LedgerEnableApiTracking:
-    ldg.trackApi = db.trackLedgerApi
-  when LedgerEnableApiProfiling:
-    ldg.profTab = db.ldgProfData()
-  ldg.ifTrackApi: debug apiTxt, api, elapsed
-  ldg
-
 # ------------------------------------------------------------------------------
 # Public methods
 # ------------------------------------------------------------------------------
 
-proc ldgProfData*(db: CoreDbRef): LedgerProfListRef =
-  ## Return profiling data table (only available in profiling mode). If
-  ## available (i.e. non-nil), result data can be organised by the functions
-  ## available with `aristo_profile`.
-  ##
-  ## Note that profiling these data have accumulated over several ledger
-  ## sessions running on the same `CoreDb` instance.
-  ##
-  when LedgerEnableApiProfiling:
-    if db.ledgerHook.isNil:
-      db.ledgerHook = LedgerProfListRef.init()
-    cast[LedgerProfListRef](db.ledgerHook)
-
 proc accessList*(ldg: LedgerRef, eAddr: EthAddress) =
   ldg.beginTrackApi LdgAccessListFn
   ldg.ac.accessList(eAddr)

nimbus/db/ledger/base/api_tracking.nim

@@ -27,8 +27,6 @@ type
     ## Profiling table index
     SummaryItem                = "total"
 
-    LdgBlessFn                 = "LedgerRef.init"
-
     LdgAccessListFn            = "accessList"
     LdgAccountExistsFn         = "accountExists"
     LdgAddBalanceFn            = "addBalance"

nimbus/db/ledger/base/base_helpers.nim

@@ -0,0 +1,41 @@
+# Nimbus
+# Copyright (c) 2024 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import
+  ../../core_db,
+  "."/[api_tracking, base_config, base_desc]
+
+# ------------------------------------------------------------------------------
+# Public constructor helper
+# ------------------------------------------------------------------------------
+
+when LedgerEnableApiProfiling:
+  proc ldgProfData*(db: CoreDbRef): LedgerProfListRef =
+    ## Return profiling data table (only available in profiling mode). If
+    ## available (i.e. non-nil), result data can be organised by the functions
+    ## available with `aristo_profile`.
+    ##
+    ## Note that profiling these data have accumulated over several ledger
+    ## sessions running on the same `CoreDb` instance.
+    ##
+    if db.ledgerHook.isNil:
+      db.ledgerHook = LedgerProfListRef.init()
+    cast[LedgerProfListRef](db.ledgerHook)
+
+proc bless*(ldg: LedgerRef; db: CoreDbRef): LedgerRef =
+  when LedgerEnableApiTracking:
+    ldg.trackApi = db.trackLedgerApi
+  when LedgerEnableApiProfiling:
+    ldg.profTab = db.ldgProfData()
+  ldg
+
+# ------------------------------------------------------------------------------
+# End
+# ------------------------------------------------------------------------------

tests/test_coredb/test_chainsync.nim

@@ -19,9 +19,13 @@ import
   ../replay/[pp, undump_blocks, undump_blocks_era1, xcheck],
   ./test_helpers
 
+when CoreDbEnableProfiling or
+     LedgerEnableApiProfiling:
+  import
+    std/sequtils
+
 when CoreDbEnableProfiling:
   import
-    std/sequtils,
     ../../nimbus/db/aristo/[aristo_api, aristo_profile],
     ../../nimbus/db/kvt/kvt_api
   var
@@ -30,9 +34,8 @@ when CoreDbEnableProfiling:
     cdbProfData: CoreDbProfListRef
 
 when LedgerEnableApiProfiling:
-  when not CoreDbEnableProfiling:
+  import
-    import
+    ../../nimbus/db/ledger/base/base_helpers
-      std/sequtils
   var
     ldgProfData: LedgerProfListRef