From 6b68ff92d33d08ddfd2def83d84130b5661a6d4f Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Sat, 22 Jun 2024 22:33:37 +0200 Subject: [PATCH] Allocation-free nibbles buffer (#2406) This buffer eleminates a large part of allocations during MPT traversal, reducing overall memory usage and GC pressure. Ideally, we would use it throughout in the API instead of `openArray[byte]` since the built-in length limit appropriately exposes the natural 64-nibble depth constraint that `openArray` fails to capture. --- hive_integration/nodocker/rpc/rpc_tests.nim | 2 +- nimbus/core/tx_pool/tx_tabs/tx_rank.nim | 2 +- nimbus/core/tx_pool/tx_tabs/tx_sender.nim | 2 +- nimbus/core/tx_pool/tx_tabs/tx_status.nim | 2 +- nimbus/db/aristo/aristo_api.nim | 6 +- nimbus/db/aristo/aristo_blobify.nim | 12 +- nimbus/db/aristo/aristo_check/check_be.nim | 2 +- nimbus/db/aristo/aristo_check/check_top.nim | 2 +- nimbus/db/aristo/aristo_constants.nim | 5 +- nimbus/db/aristo/aristo_debug.nim | 6 +- nimbus/db/aristo/aristo_delete.nim | 14 +- .../aristo/aristo_desc/desc_identifiers.nim | 12 +- nimbus/db/aristo/aristo_desc/desc_nibbles.nim | 141 ++++++++++++++++++ .../db/aristo/aristo_desc/desc_structural.nim | 6 +- nimbus/db/aristo/aristo_fetch.nim | 6 +- nimbus/db/aristo/aristo_hike.nim | 22 +-- .../aristo_merge/merge_payload_helper.nim | 12 +- nimbus/db/aristo/aristo_nearby.nim | 36 ++--- nimbus/db/aristo/aristo_path.nim | 26 +--- nimbus/db/aristo/aristo_serialise.nim | 8 +- nimbus/db/aristo/aristo_utils.nim | 4 +- .../backend/aristo_db/handlers_aristo.nim | 4 +- nimbus/db/trie_get_branch.nim | 106 ------------- 23 files changed, 233 insertions(+), 205 deletions(-) create mode 100644 nimbus/db/aristo/aristo_desc/desc_nibbles.nim delete mode 100644 nimbus/db/trie_get_branch.nim diff --git a/hive_integration/nodocker/rpc/rpc_tests.nim b/hive_integration/nodocker/rpc/rpc_tests.nim index 5938fd8d4..5276fbad4 100644 --- a/hive_integration/nodocker/rpc/rpc_tests.nim +++ b/hive_integration/nodocker/rpc/rpc_tests.nim @@ -9,7 +9,7 @@ import std/strutils, - eth/[common], + eth/common, stew/byteutils, stint, chronos, diff --git a/nimbus/core/tx_pool/tx_tabs/tx_rank.nim b/nimbus/core/tx_pool/tx_tabs/tx_rank.nim index deb7afe88..a2d6ef3b8 100644 --- a/nimbus/core/tx_pool/tx_tabs/tx_rank.nim +++ b/nimbus/core/tx_pool/tx_tabs/tx_rank.nim @@ -17,7 +17,7 @@ import std/[tables], ../tx_info, - eth/[common], + eth/common, stew/[sorted_set], results diff --git a/nimbus/core/tx_pool/tx_tabs/tx_sender.nim b/nimbus/core/tx_pool/tx_tabs/tx_sender.nim index f3cdc17fa..637d8c66d 100644 --- a/nimbus/core/tx_pool/tx_tabs/tx_sender.nim +++ b/nimbus/core/tx_pool/tx_tabs/tx_sender.nim @@ -17,7 +17,7 @@ import ../tx_info, ../tx_item, - eth/[common], + eth/common, stew/[keyed_queue, keyed_queue/kq_debug, sorted_set], results, ../../eip4844 diff --git a/nimbus/core/tx_pool/tx_tabs/tx_status.nim b/nimbus/core/tx_pool/tx_tabs/tx_status.nim index c5f742f3d..68de2c8c8 100644 --- a/nimbus/core/tx_pool/tx_tabs/tx_status.nim +++ b/nimbus/core/tx_pool/tx_tabs/tx_status.nim @@ -15,7 +15,7 @@ import ../tx_info, ../tx_item, - eth/[common], + eth/common, stew/[keyed_queue, keyed_queue/kq_debug, sorted_set], results diff --git a/nimbus/db/aristo/aristo_api.nim b/nimbus/db/aristo/aristo_api.nim index 98eed8169..e5100e78a 100644 --- a/nimbus/db/aristo/aristo_api.nim +++ b/nimbus/db/aristo/aristo_api.nim @@ -14,7 +14,7 @@ import std/times, - eth/[common, trie/nibbles], + eth/common, results, ./aristo_desc/desc_backend, ./aristo_init/memory_db, @@ -247,7 +247,7 @@ type ## data record indexed by `path` exists on the database. AristoApiHikeUpFn* = - proc(path: NibblesSeq; + proc(path: NibblesBuf; root: VertexID; db: AristoDbRef; ): Result[Hike,(VertexID,AristoError,Hike)] @@ -762,7 +762,7 @@ func init*( result = api.hasPathStorage(a, b, c) profApi.hikeUp = - proc(a: NibblesSeq; b: VertexID; c: AristoDbRef): auto = + proc(a: NibblesBuf; b: VertexID; c: AristoDbRef): auto = AristoApiProfHikeUpFn.profileRunner: result = api.hikeUp(a, b, c) diff --git a/nimbus/db/aristo/aristo_blobify.nim b/nimbus/db/aristo/aristo_blobify.nim index 02b046cef..70a34d36b 100644 --- a/nimbus/db/aristo/aristo_blobify.nim +++ b/nimbus/db/aristo/aristo_blobify.nim @@ -12,7 +12,7 @@ import std/bitops, - eth/[common, trie/nibbles], + eth/common, results, stew/endians2, ./aristo_desc @@ -112,7 +112,7 @@ proc blobifyTo*(vtx: VertexRef; data: var Blob): Result[void,AristoError] = data &= [0x08u8] of Extension: let - pSegm = vtx.ePfx.hexPrefixEncode(isleaf = false) + pSegm = vtx.ePfx.toHexPrefix(isleaf = false) psLen = pSegm.len.byte if psLen == 0 or 33 < psLen: return err(BlobifyExtPathOverflow) @@ -123,7 +123,7 @@ proc blobifyTo*(vtx: VertexRef; data: var Blob): Result[void,AristoError] = data &= [0x80u8 or psLen] of Leaf: let - pSegm = vtx.lPfx.hexPrefixEncode(isleaf = true) + pSegm = vtx.lPfx.toHexPrefix(isleaf = true) psLen = pSegm.len.byte if psLen == 0 or 33 < psLen: return err(BlobifyLeafPathOverflow) @@ -280,7 +280,8 @@ proc deblobifyTo*( return err(DeblobExtTooShort) if 8 + sLen != rLen: # => slen is at least 1 return err(DeblobExtSizeGarbled) - let (isLeaf, pathSegment) = hexPrefixDecode record.toOpenArray(8, rLen - 1) + let (isLeaf, pathSegment) = + NibblesBuf.fromHexPrefix record.toOpenArray(8, rLen - 1) if isLeaf: return err(DeblobExtGotLeafPrefix) vtx = VertexRef( @@ -295,7 +296,8 @@ proc deblobifyTo*( pLen = rLen - sLen # payload length if rLen < sLen: return err(DeblobLeafSizeGarbled) - let (isLeaf, pathSegment) = hexPrefixDecode record.toOpenArray(pLen, rLen-1) + let (isLeaf, pathSegment) = + NibblesBuf.fromHexPrefix record.toOpenArray(pLen, rLen-1) if not isLeaf: return err(DeblobLeafGotExtPrefix) var pyl: PayloadRef diff --git a/nimbus/db/aristo/aristo_check/check_be.nim b/nimbus/db/aristo/aristo_check/check_be.nim index acf87babe..4892e2ae9 100644 --- a/nimbus/db/aristo/aristo_check/check_be.nim +++ b/nimbus/db/aristo/aristo_check/check_be.nim @@ -12,7 +12,7 @@ import std/[sets, tables], - eth/[common, trie/nibbles], + eth/common, results, stew/interval_set, ../../aristo, diff --git a/nimbus/db/aristo/aristo_check/check_top.nim b/nimbus/db/aristo/aristo_check/check_top.nim index 11cb6ac60..3a313fdfe 100644 --- a/nimbus/db/aristo/aristo_check/check_top.nim +++ b/nimbus/db/aristo/aristo_check/check_top.nim @@ -12,7 +12,7 @@ import std/[sequtils, sets, typetraits], - eth/[common, trie/nibbles], + eth/common, results, ".."/[aristo_desc, aristo_get, aristo_layers, aristo_serialise, aristo_utils] diff --git a/nimbus/db/aristo/aristo_constants.nim b/nimbus/db/aristo/aristo_constants.nim index 543de1cbc..d9c3998ca 100644 --- a/nimbus/db/aristo/aristo_constants.nim +++ b/nimbus/db/aristo/aristo_constants.nim @@ -12,16 +12,13 @@ import std/sets, - eth/[common, trie/nibbles], + eth/common, ./aristo_desc/desc_identifiers const EmptyBlob* = seq[byte].default ## Useful shortcut (borrowed from `sync/snap/constants.nim`) - EmptyNibbleSeq* = EmptyBlob.initNibbleRange - ## Useful shortcut (borrowed from `sync/snap/constants.nim`) - EmptyVidSeq* = seq[VertexID].default ## Useful shortcut diff --git a/nimbus/db/aristo/aristo_debug.nim b/nimbus/db/aristo/aristo_debug.nim index 3533c38c3..8ca7a1ace 100644 --- a/nimbus/db/aristo/aristo_debug.nim +++ b/nimbus/db/aristo/aristo_debug.nim @@ -12,7 +12,7 @@ import std/[algorithm, sequtils, sets, strutils, tables], - eth/[common, trie/nibbles], + eth/common, results, stew/[byteutils, interval_set], ./aristo_desc/desc_backend, @@ -192,11 +192,11 @@ proc ppKey(key: HashKey; db: AristoDbRef; pfx = true): string = result &= @(key.data).toHex.squeeze(hex=true,ignLen=true) & tag proc ppLeafTie(lty: LeafTie, db: AristoDbRef): string = - let pfx = lty.path.to(NibblesSeq) + let pfx = lty.path.to(NibblesBuf) "@" & lty.root.ppVid(pfx=false) & ":" & ($pfx).squeeze(hex=true,ignLen=(pfx.len==64)) -proc ppPathPfx(pfx: NibblesSeq): string = +proc ppPathPfx(pfx: NibblesBuf): string = let s = $pfx if s.len < 20: s else: s[0 .. 5] & ".." & s[s.len-8 .. ^1] & ":" & $s.len diff --git a/nimbus/db/aristo/aristo_delete.nim b/nimbus/db/aristo/aristo_delete.nim index a65efe10c..ac1e28b5d 100644 --- a/nimbus/db/aristo/aristo_delete.nim +++ b/nimbus/db/aristo/aristo_delete.nim @@ -17,7 +17,7 @@ import std/[sets, typetraits], - eth/[common, trie/nibbles], + eth/common, results, "."/[aristo_desc, aristo_get, aristo_hike, aristo_layers, aristo_utils, aristo_vid] @@ -96,7 +96,7 @@ proc collapseBranch( vid: br.vid, vtx: VertexRef( vType: Extension, - ePfx: @[nibble].initNibbleRange.slice(1), + ePfx: NibblesBuf.nibble(nibble), eVid: br.vtx.bVid[nibble])) if 2 < hike.legs.len: # (1) or (2) @@ -145,7 +145,7 @@ proc collapseExt( vid: br.vid, vtx: VertexRef( vType: Extension, - ePfx: @[nibble].initNibbleRange.slice(1) & vtx.ePfx, + ePfx: NibblesBuf.nibble(nibble) & vtx.ePfx, eVid: vtx.eVid)) db.disposeOfVtx(hike.root, br.vtx.bVid[nibble]) # `vtx` is obsolete now @@ -198,7 +198,7 @@ proc collapseLeaf( vid: br.vtx.bVid[nibble], vtx: VertexRef( vType: Leaf, - lPfx: @[nibble].initNibbleRange.slice(1) & vtx.lPfx, + lPfx: NibblesBuf.nibble(nibble) & vtx.lPfx, lData: vtx.lData)) db.layersResKey(hike.root, lf.vid) # `vtx` was modified @@ -354,7 +354,7 @@ proc deleteAccountPayload*( ## leaf entry referres to a storage tree, this one will be deleted as well. ## let - hike = path.initNibbleRange.hikeUp(VertexID(1), db).valueOr: + hike = NibblesBuf.fromBytes(path).hikeUp(VertexID(1), db).valueOr: if error[1] in HikeAcceptableStopsNotFound: return err(DelPathNotFound) return err(error[1]) @@ -391,7 +391,7 @@ proc deleteGenericData*( elif LEAST_FREE_VID <= root.distinctBase: return err(DelStoRootNotAccepted) - let hike = path.initNibbleRange.hikeUp(root, db).valueOr: + let hike = NibblesBuf.fromBytes(path).hikeUp(root, db).valueOr: if error[1] in HikeAcceptableStopsNotFound: return err(DelPathNotFound) return err(error[1]) @@ -438,7 +438,7 @@ proc deleteStorageData*( if not stoID.isValid: return err(DelStoRootMissing) - let stoHike = path.initNibbleRange.hikeUp(stoID, db).valueOr: + let stoHike = NibblesBuf.fromBytes(path).hikeUp(stoID, db).valueOr: if error[1] in HikeAcceptableStopsNotFound: return err(DelPathNotFound) return err(error[1]) diff --git a/nimbus/db/aristo/aristo_desc/desc_identifiers.nim b/nimbus/db/aristo/aristo_desc/desc_identifiers.nim index 7ab6573ae..c79d8829c 100644 --- a/nimbus/db/aristo/aristo_desc/desc_identifiers.nim +++ b/nimbus/db/aristo/aristo_desc/desc_identifiers.nim @@ -16,11 +16,15 @@ import std/[algorithm, sequtils, sets, strutils, hashes], - eth/[common, trie/nibbles], + eth/common, stew/byteutils, chronicles, results, - stint + stint, + ./desc_nibbles + +export + desc_nibbles type VertexID* = distinct uint64 @@ -267,9 +271,9 @@ func cmp*(a, b: LeafTie): int = # Public helpers: Reversible conversions between `PathID`, `HashKey`, etc. # ------------------------------------------------------------------------------ -func to*(pid: PathID; T: type NibblesSeq): T = +func to*(pid: PathID; T: type NibblesBuf): T = ## Representation of a `PathID` as `NibbleSeq` (preserving full information) - let nibbles = pid.pfx.toBytesBE.toSeq.initNibbleRange() + let nibbles = NibblesBuf.fromBytes(pid.pfx.toBytesBE) if pid.length < 64: nibbles.slice(0, pid.length.int) else: diff --git a/nimbus/db/aristo/aristo_desc/desc_nibbles.nim b/nimbus/db/aristo/aristo_desc/desc_nibbles.nim new file mode 100644 index 000000000..5427a2ee0 --- /dev/null +++ b/nimbus/db/aristo/aristo_desc/desc_nibbles.nim @@ -0,0 +1,141 @@ +# nimbus-eth1 +# Copyright (c) 2023-2024 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import stew/arrayops + +type NibblesBuf* = object + ## Allocation-free type for storing up to 64 4-bit nibbles, as seen in the + ## Ethereum MPT + bytes: array[32, byte] + ibegin, iend: int8 + # Where valid nibbles can be found - we use indices here to avoid copies + # wen slicing - iend not inclusive + +func fromBytes*(T: type NibblesBuf, bytes: openArray[byte]): T = + result.iend = 2 * (int8 result.bytes.copyFrom(bytes)) + +func nibble*(T: type NibblesBuf, nibble: byte): T = + result.bytes[0] = nibble shl 4 + result.iend = 1 + +template `[]`*(r: NibblesBuf, i: int): byte = + let pos = r.ibegin + i + if (pos and 1) != 0: + (r.bytes[pos shr 1] and 0xf) + else: + (r.bytes[pos shr 1] shr 4) + +template `[]=`*(r: NibblesBuf, i: int, v: byte) = + let pos = r.ibegin + i + r.bytes[pos shr 1] = + if (pos and 1) != 0: + (v and 0x0f) or (r.bytes[pos shr 1] and 0xf0) + else: + (v shl 4) or (r.bytes[pos shr 1] and 0x0f) + +func len*(r: NibblesBuf): int = + r.iend - r.ibegin + +func `==`*(lhs, rhs: NibblesBuf): bool = + if lhs.len == rhs.len: + for i in 0 ..< lhs.len: + if lhs[i] != rhs[i]: + return false + return true + else: + return false + +func `$`*(r: NibblesBuf): string = + result = newStringOfCap(64) + for i in 0 ..< r.len: + const chars = "0123456789abcdef" + result.add chars[r[i]] + +func slice*(r: NibblesBuf, ibegin: int, iend = -1): NibblesBuf = + result.bytes = r.bytes + result.ibegin = r.ibegin + ibegin.int8 + let e = + if iend < 0: + min(64, r.iend + iend + 1) + else: + min(64, r.ibegin + iend) + doAssert ibegin >= 0 and e <= result.bytes.len * 2 + result.iend = e.int8 + +template writeFirstByte(nibbleCountExpr) {.dirty.} = + let nibbleCount = nibbleCountExpr + var oddnessFlag = (nibbleCount and 1) != 0 + newSeq(result, (nibbleCount div 2) + 1) + result[0] = byte((int(isLeaf) * 2 + int(oddnessFlag)) shl 4) + var writeHead = 0 + +template writeNibbles(r) {.dirty.} = + for i in 0 ..< r.len: + let nextNibble = r[i] + if oddnessFlag: + result[writeHead] = result[writeHead] or nextNibble + else: + inc writeHead + result[writeHead] = nextNibble shl 4 + oddnessFlag = not oddnessFlag + +func toHexPrefix*(r: NibblesBuf, isLeaf = false): seq[byte] = + writeFirstByte(r.len) + writeNibbles(r) + +func toHexPrefix*(r1, r2: NibblesBuf, isLeaf = false): seq[byte] = + writeFirstByte(r1.len + r2.len) + writeNibbles(r1) + writeNibbles(r2) + +func sharedPrefixLen*(lhs, rhs: NibblesBuf): int = + result = 0 + while result < lhs.len and result < rhs.len: + if lhs[result] != rhs[result]: + break + inc result + +func startsWith*(lhs, rhs: NibblesBuf): bool = + sharedPrefixLen(lhs, rhs) == rhs.len + +func fromHexPrefix*( + T: type NibblesBuf, r: openArray[byte] +): tuple[isLeaf: bool, nibbles: NibblesBuf] = + if r.len > 0: + result.isLeaf = (r[0] and 0x20) != 0 + let hasOddLen = (r[0] and 0x10) != 0 + + var i = 0'i8 + if hasOddLen: + result.nibbles[0] = r[0] and 0x0f + i += 1 + + for j in 1 ..< r.len: + if i >= 64: + break + result.nibbles[i] = r[j] shr 4 + result.nibbles[i + 1] = r[j] and 0x0f + i += 2 + + result.nibbles.iend = i + else: + result.isLeaf = false + +func `&`*(a, b: NibblesBuf): NibblesBuf = + for i in 0 ..< a.len: + result[i] = a[i] + + for i in 0 ..< b.len: + result[i + a.len] = b[i] + + result.iend = int8(min(64, a.len + b.len)) + +template getBytes*(a: NibblesBuf): array[32, byte] = + a.bytes diff --git a/nimbus/db/aristo/aristo_desc/desc_structural.nim b/nimbus/db/aristo/aristo_desc/desc_structural.nim index f19abc5a6..3e7cec264 100644 --- a/nimbus/db/aristo/aristo_desc/desc_structural.nim +++ b/nimbus/db/aristo/aristo_desc/desc_structural.nim @@ -16,7 +16,7 @@ import std/[hashes, sets, tables], - eth/[common, trie/nibbles], + eth/common, "."/[desc_error, desc_identifiers] type @@ -57,10 +57,10 @@ type ## Vertex for building a hexary Patricia or Merkle Patricia Trie case vType*: VertexType of Leaf: - lPfx*: NibblesSeq ## Portion of path segment + lPfx*: NibblesBuf ## Portion of path segment lData*: PayloadRef ## Reference to data payload of Extension: - ePfx*: NibblesSeq ## Portion of path segment + ePfx*: NibblesBuf ## Portion of path segment eVid*: VertexID ## Edge to vertex with ID `eVid` of Branch: bVid*: array[16,VertexID] ## Edge list with vertex IDs diff --git a/nimbus/db/aristo/aristo_fetch.nim b/nimbus/db/aristo/aristo_fetch.nim index 81e6772bd..24f1ddf25 100644 --- a/nimbus/db/aristo/aristo_fetch.nim +++ b/nimbus/db/aristo/aristo_fetch.nim @@ -15,7 +15,7 @@ import std/typetraits, - eth/[common, trie/nibbles], + eth/common, results, "."/[aristo_desc, aristo_get, aristo_hike, aristo_utils] @@ -44,7 +44,7 @@ proc retrievePayload( if path.len == 0: return err(FetchPathInvalid) - let hike = path.initNibbleRange.hikeUp(root, db).valueOr: + let hike = NibblesBuf.fromBytes(path).hikeUp(root, db).valueOr: if error[1] in HikeAcceptableStopsNotFound: return err(FetchPathNotFound) return err(error[1]) @@ -74,7 +74,7 @@ proc hasPayload( if path.len == 0: return err(FetchPathInvalid) - let hike = path.initNibbleRange.hikeUp(VertexID(1), db).valueOr: + let hike = NibblesBuf.fromBytes(path).hikeUp(VertexID(1), db).valueOr: if error[1] in HikeAcceptableStopsNotFound: return ok(false) return err(error[1]) diff --git a/nimbus/db/aristo/aristo_hike.nim b/nimbus/db/aristo/aristo_hike.nim index ef4b08603..bc9e10641 100644 --- a/nimbus/db/aristo/aristo_hike.nim +++ b/nimbus/db/aristo/aristo_hike.nim @@ -11,7 +11,7 @@ {.push raises: [].} import - eth/[common, trie/nibbles], + eth/common, results, "."/[aristo_desc, aristo_get] @@ -25,7 +25,7 @@ type ## Trie traversal path root*: VertexID ## Handy for some fringe cases legs*: seq[Leg] ## Chain of vertices and IDs - tail*: NibblesSeq ## Portion of non completed path + tail*: NibblesBuf ## Portion of non completed path const HikeAcceptableStopsNotFound* = { @@ -43,13 +43,13 @@ const # Private functions # ------------------------------------------------------------------------------ -func getNibblesImpl(hike: Hike; start = 0; maxLen = high(int)): NibblesSeq = +func getNibblesImpl(hike: Hike; start = 0; maxLen = high(int)): NibblesBuf = ## May be needed for partial rebuild, as well for n in start ..< min(hike.legs.len, maxLen): let leg = hike.legs[n] case leg.wp.vtx.vType: of Branch: - result = result & @[leg.nibble.byte].initNibbleRange.slice(1) + result = result & NibblesBuf.nibble(leg.nibble.byte) of Extension: result = result & leg.wp.vtx.ePfx of Leaf: @@ -63,22 +63,22 @@ func to*(rc: Result[Hike,(VertexID,AristoError,Hike)]; T: type Hike): T = ## Extract `Hike` from either ok ot error part of argument `rc`. if rc.isOk: rc.value else: rc.error[2] -func to*(hike: Hike; T: type NibblesSeq): T = +func to*(hike: Hike; T: type NibblesBuf): T = ## Convert back hike.getNibblesImpl() & hike.tail -func legsTo*(hike: Hike; T: type NibblesSeq): T = +func legsTo*(hike: Hike; T: type NibblesBuf): T = ## Convert back hike.getNibblesImpl() -func legsTo*(hike: Hike; numLegs: int; T: type NibblesSeq): T = +func legsTo*(hike: Hike; numLegs: int; T: type NibblesBuf): T = ## variant of `legsTo()` hike.getNibblesImpl(0, numLegs) # -------- proc hikeUp*( - path: NibblesSeq; # Partial path + path: NibblesBuf; # Partial path root: VertexID; # Start vertex db: AristoDbRef; # Database ): Result[Hike,(VertexID,AristoError,Hike)] = @@ -114,7 +114,7 @@ proc hikeUp*( if hike.tail.len == hike.tail.sharedPrefixLen(leg.wp.vtx.lPfx): # Bingo, got full path hike.legs.add leg - hike.tail = EmptyNibbleSeq + hike.tail = NibblesBuf() # This is the only loop exit break @@ -142,7 +142,7 @@ proc hikeUp*( # There must be some more data (aka `tail`) after an `Extension` vertex. if hike.tail.len == 0: hike.legs.add leg - hike.tail = EmptyNibbleSeq + hike.tail = NibblesBuf() return err((vid,HikeExtTailEmpty,hike)) # Well, somehow odd if leg.wp.vtx.ePfx.len != hike.tail.sharedPrefixLen(leg.wp.vtx.ePfx): @@ -163,7 +163,7 @@ proc hikeUp*( db: AristoDbRef; ): Result[Hike,(VertexID,AristoError,Hike)] = ## Variant of `hike()` - lty.path.to(NibblesSeq).hikeUp(lty.root, db) + lty.path.to(NibblesBuf).hikeUp(lty.root, db) # ------------------------------------------------------------------------------ # End diff --git a/nimbus/db/aristo/aristo_merge/merge_payload_helper.nim b/nimbus/db/aristo/aristo_merge/merge_payload_helper.nim index 7232c7b5a..73c688815 100644 --- a/nimbus/db/aristo/aristo_merge/merge_payload_helper.nim +++ b/nimbus/db/aristo/aristo_merge/merge_payload_helper.nim @@ -12,7 +12,7 @@ import std/[sequtils, sets, typetraits], - eth/[common, trie/nibbles], + eth/common, results, ".."/[aristo_desc, aristo_get, aristo_hike, aristo_layers, aristo_vid] @@ -20,7 +20,7 @@ import # Private getters & setters # ------------------------------------------------------------------------------ -proc xPfx(vtx: VertexRef): NibblesSeq = +proc xPfx(vtx: VertexRef): NibblesBuf = case vtx.vType: of Leaf: return vtx.lPfx @@ -106,7 +106,7 @@ proc insertBranch( if linkVtx.vType == Leaf: # Double check path prefix - if 64 < hike.legsTo(NibblesSeq).len + linkVtx.lPfx.len: + if 64 < hike.legsTo(NibblesBuf).len + linkVtx.lPfx.len: return err(MergeBranchLinkLeafGarbled) let @@ -266,7 +266,7 @@ proc mergePayloadTopIsBranchAddLeaf( if db.pPrf.len == 0: # Not much else that can be done here raiseAssert "Dangling edge:" & - " pfx=" & $hike.legsTo(hike.legs.len-1,NibblesSeq) & + " pfx=" & $hike.legsTo(hike.legs.len-1,NibblesBuf) & " branch=" & $parent & " nibble=" & $nibble & " edge=" & $linkID & @@ -479,7 +479,7 @@ proc mergePayloadImpl*( ## leaf record. ## let - nibblesPath = path.initNibbleRange + nibblesPath = NibblesBuf.fromBytes(path) hike = nibblesPath.hikeUp(root, db).to(Hike) var okHike: Hike @@ -512,7 +512,7 @@ proc mergePayloadImpl*( okHike = Hike(root: wp.vid, legs: @[Leg(wp: wp, nibble: -1)]) # Double check the result (may be removed in future) - if okHike.to(NibblesSeq) != nibblesPath: + if okHike.to(NibblesBuf) != nibblesPath: return err(MergeAssemblyFailed) # Ooops ok() diff --git a/nimbus/db/aristo/aristo_nearby.nim b/nimbus/db/aristo/aristo_nearby.nim index e67b36c84..763f0a94a 100644 --- a/nimbus/db/aristo/aristo_nearby.nim +++ b/nimbus/db/aristo/aristo_nearby.nim @@ -22,7 +22,7 @@ import std/tables, - eth/[common, trie/nibbles], + eth/common, results, "."/[aristo_desc, aristo_get, aristo_hike, aristo_path] @@ -30,7 +30,7 @@ import # Private helpers # ------------------------------------------------------------------------------ -proc `<=`(a, b: NibblesSeq): bool = +proc `<=`(a, b: NibblesBuf): bool = ## Compare nibbles, different lengths are padded to the right with zeros let abMin = min(a.len, b.len) for n in 0 ..< abMin: @@ -47,7 +47,7 @@ proc `<=`(a, b: NibblesSeq): bool = return false true -proc `<`(a, b: NibblesSeq): bool = +proc `<`(a, b: NibblesBuf): bool = not (b <= a) # ------------------ @@ -75,7 +75,7 @@ proc branchNibbleMax*(vtx: VertexRef; maxInx: int8): int8 = proc toLeafTiePayload(hike: Hike): (LeafTie,PayloadRef) = ## Shortcut for iterators. This function will gloriously crash unless the ## `hike` argument is complete. - (LeafTie(root: hike.root, path: hike.to(NibblesSeq).pathToTag.value), + (LeafTie(root: hike.root, path: hike.to(NibblesBuf).pathToTag.value), hike.legs[^1].wp.vtx.lData) # ------------------------------------------------------------------------------ @@ -139,7 +139,7 @@ proc zeroAdjust( ## Adjust empty argument path to the first vertex entry to the right. Ths ## applies is the argument `hike` is before the first entry in the database. ## The result is a hike which is aligned with the first entry. - proc accept(p: Hike; pfx: NibblesSeq): bool = + proc accept(p: Hike; pfx: NibblesBuf): bool = when doLeast: p.tail <= pfx else: @@ -151,7 +151,7 @@ proc zeroAdjust( else: w.branchNibbleMax n - proc toHike(pfx: NibblesSeq, root: VertexID, db: AristoDbRef): Hike = + proc toHike(pfx: NibblesBuf, root: VertexID, db: AristoDbRef): Hike = when doLeast: pfx.pathPfxPad(0).hikeUp(root, db).to(Hike) else: @@ -163,7 +163,7 @@ proc zeroAdjust( let root = db.getVtx hike.root if root.isValid: block fail: - var pfx: NibblesSeq + var pfx: NibblesBuf case root.vType: of Branch: # Find first non-dangling link and assign it @@ -179,7 +179,7 @@ proc zeroAdjust( if n < 0: # Before or after the database range return err((hike.root,NearbyBeyondRange)) - pfx = @[n.byte].initNibbleRange.slice(1) + pfx = NibblesBuf.nibble(n.byte) of Extension: let ePfx = root.ePfx @@ -210,7 +210,7 @@ proc finalise( moveRight: static[bool]; # Direction of next vertex ): Result[Hike,(VertexID,AristoError)] = ## Handle some pathological cases after main processing failed - proc beyond(p: Hike; pfx: NibblesSeq): bool = + proc beyond(p: Hike; pfx: NibblesBuf): bool = when moveRight: pfx < p.tail else: @@ -239,14 +239,14 @@ proc finalise( if not vtx.isValid: return err((vid,NearbyDanglingLink)) - var pfx: NibblesSeq + var pfx: NibblesBuf case vtx.vType: of Leaf: pfx = vtx.lPfx of Extension: pfx = vtx.ePfx of Branch: - pfx = @[vtx.branchBorderNibble.byte].initNibbleRange.slice(1) + pfx = NibblesBuf.nibble(vtx.branchBorderNibble.byte) if hike.beyond pfx: return err((vid,NearbyBeyondRange)) @@ -274,7 +274,7 @@ proc nearbyNext( else: 0 < nibble - proc accept(p: Hike; pfx: NibblesSeq): bool = + proc accept(p: Hike; pfx: NibblesBuf): bool = when moveRight: p.tail <= pfx else: @@ -356,7 +356,7 @@ proc nearbyNext( uHike.tail = uHikeTail else: # Pop current `Branch` vertex on top and append nibble to `tail` - uHike.tail = @[top.nibble.byte].initNibbleRange.slice(1) & uHike.tail + uHike.tail = NibblesBuf.nibble(top.nibble.byte) & uHike.tail uHike.legs.setLen(uHike.legs.len - 1) # End while @@ -375,7 +375,7 @@ proc nearbyNextLeafTie( if 0 < hike.legs.len: if hike.legs[^1].wp.vtx.vType != Leaf: return err((hike.legs[^1].wp.vid,NearbyLeafExpected)) - let rc = hike.legsTo(NibblesSeq).pathToTag + let rc = hike.legsTo(NibblesBuf).pathToTag if rc.isOk: return ok rc.value return err((VertexID(0),rc.error)) @@ -434,14 +434,14 @@ iterator rightPairs*( if 0 < tail.len: let topNibble = tail[tail.len - 1] if topNibble < 15: - let newNibble = @[topNibble+1].initNibbleRange.slice(1) + let newNibble = NibblesBuf.nibble(topNibble+1) hike.tail = tail.slice(0, tail.len - 1) & newNibble hike.legs.setLen(hike.legs.len - 1) break reuseHike if 1 < tail.len: let nxtNibble = tail[tail.len - 2] if nxtNibble < 15: - let dblNibble = @[((nxtNibble+1) shl 4) + 0].initNibbleRange + let dblNibble = NibblesBuf.fromBytes([((nxtNibble+1) shl 4) + 0]) hike.tail = tail.slice(0, tail.len - 2) & dblNibble hike.legs.setLen(hike.legs.len - 1) break reuseHike @@ -497,14 +497,14 @@ iterator leftPairs*( if 0 < tail.len: let topNibble = tail[tail.len - 1] if 0 < topNibble: - let newNibble = @[topNibble - 1].initNibbleRange.slice(1) + let newNibble = NibblesBuf.nibble(topNibble - 1) hike.tail = tail.slice(0, tail.len - 1) & newNibble hike.legs.setLen(hike.legs.len - 1) break reuseHike if 1 < tail.len: let nxtNibble = tail[tail.len - 2] if 0 < nxtNibble: - let dblNibble = @[((nxtNibble-1) shl 4) + 15].initNibbleRange + let dblNibble = NibblesBuf.fromBytes([((nxtNibble-1) shl 4) + 15]) hike.tail = tail.slice(0, tail.len - 2) & dblNibble hike.legs.setLen(hike.legs.len - 1) break reuseHike diff --git a/nimbus/db/aristo/aristo_path.nim b/nimbus/db/aristo/aristo_path.nim index d70eae908..be5de1d44 100644 --- a/nimbus/db/aristo/aristo_path.nim +++ b/nimbus/db/aristo/aristo_path.nim @@ -12,7 +12,7 @@ import std/sequtils, - eth/[common, trie/nibbles], + eth/common, results, ./aristo_desc @@ -30,7 +30,7 @@ import # # where the `ignored` part is typically expected a zero nibble. -func pathPfxPad*(pfx: NibblesSeq; dblNibble: static[byte]): NibblesSeq +func pathPfxPad*(pfx: NibblesBuf; dblNibble: static[byte]): NibblesBuf # ------------------------------------------------------------------------------ # Public functions @@ -51,17 +51,7 @@ func pathAsBlob*(tag: PathID): Blob = else: return key[0 .. (tag.length - 1) div 2] -func pathAsHEP*(tag: PathID; isLeaf = false): Blob = - ## Convert the `tag` argument to a hex encoded partial path as used in `eth` - ## or `snap` protocol where full paths of nibble length 64 are encoded as 32 - ## byte `Blob` and non-leaf partial paths are *compact encoded* (i.e. per - ## the Ethereum wire protocol.) - if 64 <= tag.length: - @(tag.pfx.toBytesBE) - else: - tag.to(NibblesSeq).hexPrefixEncode(isLeaf=true) - -func pathToTag*(partPath: NibblesSeq): Result[PathID,AristoError] = +func pathToTag*(partPath: NibblesBuf): Result[PathID,AristoError] = ## Convert the argument `partPath` to a `PathID` type value. if partPath.len == 0: return ok VOID_PATH_ID @@ -83,9 +73,9 @@ func pathToTag*(partPath: openArray[byte]): Result[PathID,AristoError] = # -------------------- -func pathPfxPad*(pfx: NibblesSeq; dblNibble: static[byte]): NibblesSeq = +func pathPfxPad*(pfx: NibblesBuf; dblNibble: static[byte]): NibblesBuf = ## Extend (or cut) the argument nibbles sequence `pfx` for generating a - ## `NibblesSeq` with exactly 64 nibbles, the equivalent of a path key. + ## `NibblesBuf` with exactly 64 nibbles, the equivalent of a path key. ## ## This function must be handled with some care regarding a meaningful value ## for the `dblNibble` argument. Currently, only static values `0` and `255` @@ -95,11 +85,11 @@ func pathPfxPad*(pfx: NibblesSeq; dblNibble: static[byte]): NibblesSeq = let padLen = 64 - pfx.len if 0 <= padLen: - result = pfx & dblNibble.repeat(padLen div 2).mapIt(it.byte).initNibbleRange + result = pfx & NibblesBuf.fromBytes(dblNibble.repeat(padLen div 2).mapIt(it.byte)) if (padLen and 1) == 1: - result = result & @[dblNibble.byte].initNibbleRange.slice(1) + result = result & NibblesBuf.nibble(dblNibble.byte) else: - let nope = seq[byte].default.initNibbleRange + let nope = NibblesBuf() result = pfx.slice(0,64) & nope # nope forces re-alignment # ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_serialise.nim b/nimbus/db/aristo/aristo_serialise.nim index 9803d2541..dd6ba4ce0 100644 --- a/nimbus/db/aristo/aristo_serialise.nim +++ b/nimbus/db/aristo/aristo_serialise.nim @@ -12,7 +12,7 @@ import std/sequtils, - eth/[common, rlp, trie/nibbles], + eth/[common, rlp], results, "."/[aristo_constants, aristo_desc, aristo_get] @@ -98,7 +98,7 @@ proc read*(rlp: var Rlp; T: type NodeRef): T {.gcsafe, raises: [RlpError].} = of 2: if blobs[0].len == 0: return aristoError(RlpNonEmptyBlobExpected) - let (isLeaf, pathSegment) = hexPrefixDecode blobs[0] + let (isLeaf, pathSegment) = NibblesBuf.fromHexPrefix blobs[0] if isLeaf: return NodeRef( vType: Leaf, @@ -147,7 +147,7 @@ proc append*(writer: var RlpWriter; node: NodeRef) = of Extension: writer.startList(2) - writer.append node.ePfx.hexPrefixEncode(isleaf = false) + writer.append node.ePfx.toHexPrefix(isleaf = false) writer.addHashKey node.key[0] of Leaf: @@ -155,7 +155,7 @@ proc append*(writer: var RlpWriter; node: NodeRef) = ok(node.key[0]) # always succeeds writer.startList(2) - writer.append node.lPfx.hexPrefixEncode(isleaf = true) + writer.append node.lPfx.toHexPrefix(isleaf = true) writer.append node.lData.serialise(getKey0).value # --------------------- diff --git a/nimbus/db/aristo/aristo_utils.nim b/nimbus/db/aristo/aristo_utils.nim index 057e1a506..db1838dbd 100644 --- a/nimbus/db/aristo/aristo_utils.nim +++ b/nimbus/db/aristo/aristo_utils.nim @@ -15,7 +15,7 @@ import std/[sequtils, sets, typetraits], - eth/[common, trie/nibbles], + eth/common, results, "."/[aristo_constants, aristo_desc, aristo_get, aristo_hike, aristo_layers] @@ -184,7 +184,7 @@ proc retrieveStoAccHike*( ## vertex and the vertex ID. ## # Expand vertex path to account leaf - let hike = (@accPath).initNibbleRange.hikeUp(VertexID(1), db).valueOr: + let hike = accPath.to(NibblesBuf).hikeUp(VertexID(1), db).valueOr: return err(UtilsAccInaccessible) # Extract the account payload fro the leaf diff --git a/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim b/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim index 5274d37a1..977499815 100644 --- a/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim +++ b/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim @@ -13,7 +13,7 @@ import std/[strutils, typetraits], chronicles, - eth/[common, trie/nibbles], + eth/common, stew/byteutils, ../../../aristo, ../../../aristo/aristo_desc, @@ -487,7 +487,7 @@ proc ctxMethods(cCtx: AristoCoreDbCtxRef): CoreDbCtxFns = let error = (col.stoRoot,MptRootUnacceptable) return err(error.toError(base, info, RootUnacceptable)) # Verify path if there is a particular storge root VID - let rc = api.hikeUp(newMpt.accPath.to(NibblesSeq), AccountsVID, mpt) + let rc = api.hikeUp(newMpt.accPath.to(NibblesBuf), AccountsVID, mpt) if rc.isErr: return err(rc.error[1].toError(base, info, AccNotFound)) else: diff --git a/nimbus/db/trie_get_branch.nim b/nimbus/db/trie_get_branch.nim deleted file mode 100644 index 962b772b2..000000000 --- a/nimbus/db/trie_get_branch.nim +++ /dev/null @@ -1,106 +0,0 @@ -# Nimbus -# Copyright (c) 2023-2024 Status Research & Development GmbH -# Licensed under either of -# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or -# http://www.apache.org/licenses/LICENSE-2.0) -# * MIT license ([LICENSE-MIT](LICENSE-MIT) or -# http://opensource.org/licenses/MIT) -# at your option. This file may not be copied, modified, or distributed except -# according to those terms. - -# This implementation of getBranch on the CoreDbMptRef type is a temporary solution -# which can be removed once we get an equivient proc defined on the CoreDbMptRef type -# in the db layer. - -{.push raises: [].} - -import - eth/[rlp, trie/nibbles], - results, - "."/[core_db] - -type - TrieNodeKey = object - hash: KeccakHash - usedBytes: uint8 - -template len(key: TrieNodeKey): int = - key.usedBytes.int - -template asDbKey(k: TrieNodeKey): untyped = - doAssert k.usedBytes == 32 - k.hash.data - -template extensionNodeKey(r: Rlp): auto = - hexPrefixDecode r.listElem(0).toBytes - -proc getLocalBytes(x: TrieNodeKey): seq[byte] = - ## This proc should be used on nodes using the optimization - ## of short values within the key. - doAssert x.usedBytes < 32 - x.hash.data[0..