avoid copying data when merging save points (#2584)

Saving both memory and processing, we can move entries from one
savepoint to another, specially when the target is empty as it often is
during transaction processing

nimbus.nimble

@@ -53,8 +53,8 @@ proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
   exec "nim " & lang & " --out:build/" & name & " " & extra_params & " " & srcDir & name & ".nim"
 
 proc test(path: string, name: string, params = "", lang = "c") =
-  # Verify stack usage is kept low by setting 750k stack limit in tests.
+  # Verify stack usage is kept low by setting 1mb stack limit in tests.
-  const stackLimitKiB = 750
+  const stackLimitKiB = 1024
   when not defined(windows):
     const (buildOption, runPrefix) = ("", "ulimit -s " & $stackLimitKiB & " && ")
   else:

nimbus/db/access_list.nim

@@ -11,7 +11,8 @@
 import
   std/[tables, sets],
   stint,
-  eth/common
+  eth/common,
+  ../utils/mergeutils
 
 type
   SlotSet = HashSet[UInt256]
@@ -49,12 +50,9 @@ func contains*(ac: var AccessList, address: EthAddress, slot: UInt256): bool =
   ac.slots.withValue(address, val):
     result = slot in val[]
 
-proc merge*(ac: var AccessList, other: AccessList) {.inline.} =
+proc mergeAndReset*(ac, other: var AccessList) =
-  for k, v in other.slots:
+  # move values in `other` to `ac`
-    ac.slots.withValue(k, val):
+  ac.slots.mergeAndReset(other.slots)
-      val[].incl v
-    do:
-      ac.slots[k] = v
 
 proc add*(ac: var AccessList, address: EthAddress) =
   if address notin ac.slots:

nimbus/db/ledger/backend/accounts_ledger.nim

@@ -16,6 +16,7 @@ import
   eth/common,
   results,
   stew/keyed_queue,
+  ../../../utils/mergeutils,
   ../../../evm/code_bytes,
   ../../../stateless/multi_keys,
   "../../.."/[constants, utils/utils],
@@ -209,16 +210,12 @@ proc commit*(ac: AccountsLedgerRef, sp: LedgerSavePoint) =
   doAssert not sp.parentSavepoint.isNil
 
   ac.savePoint = sp.parentSavepoint
-  for k, v in sp.cache:
+  ac.savePoint.cache.mergeAndReset(sp.cache)
-    sp.parentSavepoint.cache[k] = v
+  ac.savePoint.dirty.mergeAndReset(sp.dirty)
-
+  ac.savePoint.transientStorage.mergeAndReset(sp.transientStorage)
-  for k, v in sp.dirty:
+  ac.savePoint.accessList.mergeAndReset(sp.accessList)
-    sp.parentSavepoint.dirty[k] = v
+  ac.savePoint.selfDestruct.mergeAndReset(sp.selfDestruct)
-
+  ac.savePoint.logEntries.mergeAndReset(sp.logEntries)
-  ac.savePoint.transientStorage.merge(sp.transientStorage)
-  ac.savePoint.accessList.merge(sp.accessList)
-  ac.savePoint.selfDestruct.incl sp.selfDestruct
-  ac.savePoint.logEntries.add sp.logEntries
   sp.state = Committed
 
   when debugAccountsLedgerRef:
@@ -629,9 +626,6 @@ proc selfDestructLen*(ac: AccountsLedgerRef): int =
 proc addLogEntry*(ac: AccountsLedgerRef, log: Log) =
   ac.savePoint.logEntries.add log
 
-proc logEntries*(ac: AccountsLedgerRef): lent seq[Log] =
-  ac.savePoint.logEntries
-
 proc getAndClearLogEntries*(ac: AccountsLedgerRef): seq[Log] =
   swap(result, ac.savePoint.logEntries)
 

nimbus/db/ledger/base.nim

@@ -203,11 +203,6 @@ proc isTopLevelClean*(ldg: LedgerRef): bool =
   result = ldg.ac.isTopLevelClean()
   ldg.ifTrackApi: debug apiTxt, api, elapsed, result
 
-proc logEntries*(ldg: LedgerRef): seq[Log] =
-  ldg.beginTrackApi LdgLogEntriesFn
-  result = ldg.ac.logEntries()
-  ldg.ifTrackApi: debug apiTxt, api, elapsed, result
-
 proc makeMultiKeys*(ldg: LedgerRef): MultiKeysRef =
   ldg.beginTrackApi LdgMakeMultiKeysFn
   result = ldg.ac.makeMultiKeys()

nimbus/db/ledger/base/api_tracking.nim

@@ -57,7 +57,6 @@ type
     LdgIsDeadAccountFn         = "isDeadAccount"
     LdgIsEmptyAccountFn        = "isEmptyAccount"
     LdgIsTopLevelCleanFn       = "isTopLevelClean"
-    LdgLogEntriesFn            = "logEntries"
     LdgMakeMultiKeysFn         = "makeMultiKeys"
     LdgPersistFn               = "persist"
     LdgRipemdSpecialFn         = "ripemdSpecial"

nimbus/db/transient_storage.nim

@@ -11,7 +11,8 @@
 import
   tables,
   stint,
-  eth/common
+  eth/common,
+  ../utils/mergeutils
 
 type
   StorageTable = ref object
@@ -24,9 +25,8 @@ type
 # Private helpers
 #######################################################################
 
-proc merge(a, b: StorageTable) =
+proc mergeAndReset*(a, b: StorageTable) =
-  for k, v in b.map:
+  a.map.mergeAndReset(b.map)
-    a.map[k] = v
 
 #######################################################################
 # Public functions
@@ -58,12 +58,8 @@ proc setStorage*(ac: var TransientStorage,
 
   table.map[slot] = value
 
-proc merge*(ac: var TransientStorage, other: TransientStorage) =
+proc mergeAndReset*(ac, other: var TransientStorage) =
-  for k, v in other.map:
+  ac.map.mergeAndReset(other.map)
-    ac.map.withValue(k, val):
-      val[].merge(v)
-    do:
-      ac.map[k] = v
 
 proc clear*(ac: var TransientStorage) {.inline.} =
   ac.map.clear()

nimbus/utils/mergeutils.nim

@@ -0,0 +1,51 @@
+# Nimbus
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+{.push raises: [], gcsafe.}
+
+import
+  std/[tables, sets]
+
+# Utilities for merging source data into a target taking care to move data and
+# leave the source empty
+func mergeAndReset*(tgt, src: var auto) =
+  tgt = move(src)
+
+func mergeAndReset*(tgt, src: var seq) =
+  mixin mergeAndReset
+  if tgt.len == 0:
+    swap(tgt, src)
+  else:
+    let tlen = tgt.len
+    tgt.setLen(tgt.len + src.len)
+    for i, sv in src.mpairs():
+      mergeAndReset(tgt[tlen + i], sv)
+    src.reset()
+
+func mergeAndReset*(tgt, src: var HashSet) =
+  mixin mergeAndReset
+  if tgt.len == 0:
+    swap(tgt, src)
+  else:
+    for sv in src.items():
+      tgt.incl(sv)
+    src.reset()
+
+func mergeAndReset*(tgt, src: var Table) =
+  mixin mergeAndReset
+  if tgt.len == 0:
+    swap(tgt, src)
+  else:
+    for k, sv in src.mpairs():
+      tgt.withValue(k, tv):
+        mergeAndReset(tv[], sv)
+      do:
+        tgt[k] = move(sv)
+    src.reset()