diff --git a/nimbus/db/aristo/README.md b/nimbus/db/aristo/README.md index 62186bb65..8c19b23dc 100644 --- a/nimbus/db/aristo/README.md +++ b/nimbus/db/aristo/README.md @@ -211,14 +211,14 @@ and implemented as 64 bit values, stored *Big Endian* in the serialisation. | | -- first vertexID 8 +--+--+--+--+--+--+--+--+--+ ... -- more vertexIDs - +--+--+ - | | -- access(16) bitmap + +--+--+ + | | -- access(16) bitmap +--+--+ || | -- marker(2) + unused(6) +--+ - where - marker(2) is the double bit array 00 + where + marker(2) is the double bit array 00 For a given index *n* between *0..15*, if the bit at position *n* of the it vector *access(16)* is reset to zero, then there is no *n*-th structural @@ -240,8 +240,8 @@ stored in the right byte of the serialised bitmap. || | -- marker(2) + pathSegmentLen(6) +--+ - where - marker(2) is the double bit array 10 + where + marker(2) is the double bit array 10 The path segment of the *Extension* record is compact encoded. So it has at least one byte. The first byte *P0* has bit 5 reset, i.e. *P0 and 0x20* is @@ -260,8 +260,8 @@ of the extension record (as *recordLen - 9*.) || | -- marker(2) + pathSegmentLen(6) +--+ - where - marker(2) is the double bit array 11 + where + marker(2) is the double bit array 11 A *Leaf* record path segment is compact encoded. So it has at least one byte. The first byte *P0* has bit 5 set, i.e. *P0 and 0x20* is non-zero (bit 4 is @@ -277,8 +277,8 @@ also set if the right nibble is the first part of the path.) || | -- marker(2) + unused(6) +--+ - where - marker(2) is the double bit array 01 + where + marker(2) is the double bit array 01 Currently, the descriptor record only contains data for producing unique vectorID values that can be used as structural keys. If this descriptor is diff --git a/nimbus/db/aristo/aristo_cache.nim b/nimbus/db/aristo/aristo_cache.nim index 9e8ac38cf..a8b5c180d 100644 --- a/nimbus/db/aristo/aristo_cache.nim +++ b/nimbus/db/aristo/aristo_cache.nim @@ -15,7 +15,7 @@ import eth/common, stew/results, ../../sync/snap/range_desc, - "."/[aristo_desc, aristo_error, aristo_transcode] + "."/[aristo_desc, aristo_error, aristo_transcode, aristo_vid] # ------------------------------------------------------------------------------ # Private helpers @@ -39,22 +39,22 @@ proc convertPartially( nd = NodeRef( vType: Extension, ePfx: vtx.ePfx, - eVtx: vtx.eVtx) - db.kMap.withValue(vtx.eVtx, keyPtr): + eVid: vtx.eVid) + db.kMap.withValue(vtx.eVid, keyPtr): nd.key[0] = keyPtr[] return - result.add vtx.eVtx + result.add vtx.eVid of Branch: nd = NodeRef( vType: Branch, - bVtx: vtx.bVtx) + bVid: vtx.bVid) for n in 0..15: - if vtx.bVtx[n].isZero: + if vtx.bVid[n].isZero: continue - db.kMap.withValue(vtx.bVtx[n], kPtr): + db.kMap.withValue(vtx.bVid[n], kPtr): nd.key[n] = kPtr[] continue - result.add vtx.bVtx[n] + result.add vtx.bVid[n] proc convertPartiallyOk( db: AristoDbRef; @@ -73,18 +73,18 @@ proc convertPartiallyOk( nd = NodeRef( vType: Extension, ePfx: vtx.ePfx, - eVtx: vtx.eVtx) - db.kMap.withValue(vtx.eVtx, keyPtr): + eVid: vtx.eVid) + db.kMap.withValue(vtx.eVid, keyPtr): nd.key[0] = keyPtr[] result = true of Branch: nd = NodeRef( vType: Branch, - bVtx: vtx.bVtx) + bVid: vtx.bVid) result = true for n in 0..15: - if not vtx.bVtx[n].isZero: - db.kMap.withValue(vtx.bVtx[n], kPtr): + if not vtx.bVid[n].isZero: + db.kMap.withValue(vtx.bVid[n], kPtr): nd.key[n] = kPtr[] continue return false @@ -93,7 +93,7 @@ proc cachedVID(db: AristoDbRef; nodeKey: NodeKey): VertexID = ## Get vertex ID from reverse cache db.pAmk.withValue(nodeKey, vidPtr): return vidPtr[] - result = VertexID.new(db) + result = db.vidFetch() db.pAmk[nodeKey] = result db.kMap[result] = nodeKey @@ -145,7 +145,7 @@ proc updated*(nd: NodeRef; db: AristoDbRef): NodeRef = vType: Extension, ePfx: nd.ePfx) if not nd.key[0].isZero: - result.eVtx = db.cachedVID nd.key[0] + result.eVid = db.cachedVID nd.key[0] result.key[0] = nd.key[0] of Branch: result = NodeRef( @@ -153,7 +153,7 @@ proc updated*(nd: NodeRef; db: AristoDbRef): NodeRef = key: nd.key) for n in 0..15: if not nd.key[n].isZero: - result.bVtx[n] = db.cachedVID nd.key[n] + result.bVid[n] = db.cachedVID nd.key[n] proc asNode*(vtx: VertexRef; db: AristoDbRef): NodeRef = ## Return a `NodeRef` object by augmenting missing `Merkel` hashes (aka diff --git a/nimbus/db/aristo/aristo_constants.nim b/nimbus/db/aristo/aristo_constants.nim new file mode 100644 index 000000000..4263a7007 --- /dev/null +++ b/nimbus/db/aristo/aristo_constants.nim @@ -0,0 +1,28 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +{.push raises: [].} + +import + ../../sync/snap/range_desc, + eth/[common, trie/nibbles] + +const + EmptyBlob* = seq[byte].default + ## Useful shortcut (borrowed from `sync/snap/constants.nim`) + + EmptyNibbleSeq* = EmptyBlob.initNibbleRange + ## Useful shortcut (borrowed from `sync/snap/constants.nim`) + + EMPTY_ROOT_KEY* = EMPTY_ROOT_HASH.to(NodeKey) + + EMPTY_CODE_KEY* = EMPTY_CODE_HASH.to(NodeKey) + +# End diff --git a/nimbus/db/aristo/aristo_debug.nim b/nimbus/db/aristo/aristo_debug.nim index c75b302c8..c53c23ceb 100644 --- a/nimbus/db/aristo/aristo_debug.nim +++ b/nimbus/db/aristo/aristo_debug.nim @@ -15,11 +15,7 @@ import eth/[common, trie/nibbles], stew/byteutils, ../../sync/snap/range_desc, - "."/aristo_desc - -const - EMPTY_ROOT_KEY = EMPTY_ROOT_HASH.to(NodeKey) - EMPTY_CODE_KEY = EMPTY_CODE_HASH.to(NodeKey) + "."/[aristo_constants, aristo_desc, aristo_vid] # ------------------------------------------------------------------------------ # Ptivate functions @@ -101,7 +97,7 @@ proc keyToVtxID*(db: AristoDbRef, key: NodeKey): VertexID = db.xMap.withValue(key, vidPtr): return vidPtr[] - result = VertexID.new db + result = db.vidFetch() db.xMap[key] = result proc pp*(vid: openArray[VertexID]): string = @@ -130,12 +126,12 @@ proc pp*(nd: VertexRef, db = AristoDbRef(nil)): string = of Leaf: result &= $nd.lPfx & "," & nd.lData.pp(db) of Extension: - result &= $nd.ePfx & "," & nd.eVtx.ppVid + result &= $nd.ePfx & "," & nd.eVid.ppVid of Branch: result &= "[" for n in 0..15: - if not nd.bVtx[n].isZero: - result &= nd.bVtx[n].ppVid + if not nd.bVid[n].isZero: + result &= nd.bVid[n].ppVid result &= "," result[^1] = ']' result &= ")" @@ -152,19 +148,19 @@ proc pp*(nd: NodeRef, db = AristoDbRef(nil)): string = result &= $nd.lPfx & "," & nd.lData.pp(db) of Extension: - result &= $nd.ePfx & "," & nd.eVtx.ppVid & "," & nd.key[0].ppKey + result &= $nd.ePfx & "," & nd.eVid.ppVid & "," & nd.key[0].ppKey of Branch: result &= "[" for n in 0..15: - if not nd.bVtx[n].isZero or not nd.key[n].isZero: - result &= nd.bVtx[n].ppVid - result &= db.keyVidUpdate(nd.key[n], nd.bVtx[n]) & "," + if not nd.bVid[n].isZero or not nd.key[n].isZero: + result &= nd.bVid[n].ppVid + result &= db.keyVidUpdate(nd.key[n], nd.bVid[n]) & "," result[^1] = ']' result &= ",[" for n in 0..15: - if not nd.bVtx[n].isZero or not nd.key[n].isZero: + if not nd.bVid[n].isZero or not nd.key[n].isZero: result &= nd.key[n].ppKey(db) result &= "," result[^1] = ']' diff --git a/nimbus/db/aristo/aristo_desc.nim b/nimbus/db/aristo/aristo_desc.nim index 5e1965c80..fa678592b 100644 --- a/nimbus/db/aristo/aristo_desc.nim +++ b/nimbus/db/aristo/aristo_desc.nim @@ -13,88 +13,118 @@ ## ## These data structures allows to overlay the *Patricia Trie* with *Merkel ## Trie* hashes. See the `README.md` in the `aristo` folder for documentation. - +## +## Some semantic explanations; +## +## * NodeKey, NodeRef etc. refer to the standard/legacy `Merkel Patricia Tree` +## * VertexID, VertexRef, etc. refer to the `Aristo Trie` +## {.push raises: [].} import - std/tables, + std/[sets, tables], eth/[common, trie/nibbles], stew/results, ../../sync/snap/range_desc, ./aristo_error type - VertexID* = distinct uint64 ## Tip of edge towards child, also table key + VertexID* = distinct uint64 + ## Tip of edge towards child object in the `Patricia Trie` logic. It is + ## also the key into the structural table of the `Aristo Trie`. - VertexType* = enum ## Type of Patricia Trie node + GetVtxFn* = + proc(vid: VertexID): Result[VertexRef,AristoError] + {.gcsafe, raises: [].} + ## Generic backend database retrieval function for a single structural + ## `Aristo DB` data record. + + GetKeyFn* = + proc(vid: VertexID): Result[NodeKey,AristoError] + {.gcsafe, raises: [].} + ## Generic backend database retrieval function for a single + ## `Aristo DB` hash lookup value. + + PutVtxFn* = + proc(vrps: openArray[(VertexID,VertexRef)]): AristoError + {.gcsafe, raises: [].} + ## Generic backend database bulk storage function. + + PutKeyFn* = + proc(vkps: openArray[(VertexID,NodeKey)]): AristoError + {.gcsafe, raises: [].} + ## Generic backend database bulk storage function. + + DelFn* = + proc(vids: openArray[VertexID]) + {.gcsafe, raises: [].} + ## Generic backend database delete function for both, the structural + ## `Aristo DB` data record and the hash lookup value. + + VertexType* = enum + ## Type of `Aristo Trie` vertex Leaf Extension Branch - PayloadType* = enum ## Type of leaf data (to be extended) - BlobData - AccountData + PayloadType* = enum + ## Type of leaf data (to be extended) + BlobData ## Generic data, typically RLP encoded + AccountData ## Legacy `Account` with hash references + # AristoAccount ## `Aristo account` with vertex IDs links PayloadRef* = ref object case pType*: PayloadType of BlobData: - blob*: Blob ## Opaque data value reference + blob*: Blob ## Opaque data value reference of AccountData: - account*: Account ## Expanded accounting data + account*: Account ## Expanded accounting data VertexRef* = ref object of RootRef ## Vertex for building a hexary Patricia or Merkle Patricia Trie case vType*: VertexType of Leaf: - lPfx*: NibblesSeq ## Portion of path segment - lData*: PayloadRef ## Reference to data payload + lPfx*: NibblesSeq ## Portion of path segment + lData*: PayloadRef ## Reference to data payload of Extension: - ePfx*: NibblesSeq ## Portion of path segment - eVtx*: VertexID ## Edge to vertex with ID `eVtx` + ePfx*: NibblesSeq ## Portion of path segment + eVid*: VertexID ## Edge to vertex with ID `eVid` of Branch: - bVtx*: array[16,VertexID] ## Edge list with vertex IDs + bVid*: array[16,VertexID] ## Edge list with vertex IDs NodeRef* = ref object of VertexRef ## Combined record for a *traditional* ``Merkle Patricia Tree` node merged ## with a structural `VertexRef` type object. - error*: AristoError ## Can be used for error signalling - key*: array[16,NodeKey] ## Merkle hash(es) for Branch & Extension vtx + error*: AristoError ## Can be used for error signalling + key*: array[16,NodeKey] ## Merkle hash/es for Branch & Extension - PathStep* = object - ## For constructing a tree traversal path - # key*: NodeKey ## Node label ?? - node*: VertexRef ## Referes to data record - nibble*: int8 ## Branch node selector (if any) - depth*: int ## May indicate path length (typically 64) + AristoBackendRef* = ref object + ## Backend interface. + getVtxFn*: GetVtxFn ## Read vertex record + getKeyFn*: GetKeyFn ## Read vertex hash + putVtxFn*: PutVtxFn ## Bulk store vertex records + putKeyFn*: PutKeyFn ## Bulk store vertex hashes + delFn*: DelFn ## Bulk delete vertex records and hashes - Path* = object - root*: VertexID ## Root node needed when `path.len == 0` - path*: seq[PathStep] ## Chain of nodes - tail*: NibblesSeq ## Stands for non completed leaf path - - LeafSpecs* = object - ## Temporarily stashed leaf data (as for an account.) Proper records - ## have non-empty payload. Records with empty payload are administrative - ## items, e.g. lower boundary records. - pathTag*: NodeTag ## `Patricia Trie` key path - nodeVtx*: VertexID ## Table lookup vertex ID (if any) - payload*: PayloadRef ## Reference to data payload - - GetFn* = proc(key: openArray[byte]): Blob - {.gcsafe, raises: [CatchableError].} - ## Persistent database `get()` function. For read-only cases, this - ## function can be seen as the persistent alternative to ``tab[]` on - ## a `HexaryTreeDbRef` descriptor. - - AristoDbRef* = ref object of RootObj + AristoDbRef* = ref AristoDbObj + AristoDbObj = object ## Hexary trie plus helper structures - sTab*: Table[VertexID,NodeRef] ## Structural vertex table making up a trie - kMap*: Table[VertexID,NodeKey] ## Merkle hash key mapping - pAmk*: Table[NodeKey,VertexID] ## Reverse mapper for data import - vidGen*: seq[VertexID] ## Unique vertex ID generator + sTab*: Table[VertexID,VertexRef] ## Structural vertex table making up a trie + sDel*: HashSet[VertexID] ## Deleted vertices + kMap*: Table[VertexID,NodeKey] ## Merkle hash key mapping + pAmk*: Table[NodeKey,VertexID] ## Reverse mapper for data import + + case cascaded*: bool ## Cascaded delta databases, tx layer + of true: + level*: int ## Positive number of stack layers + stack*: AristoDbRef ## Down the chain, not `nil` + base*: AristoDbRef ## Backend level descriptor + else: + vidGen*: seq[VertexID] ## Unique vertex ID generator + backend*: AristoBackendRef ## backend database (maybe `nil`) # Debugging data below, might go away in future - xMap*: Table[NodeKey,VertexID] ## Mapper for pretty printing, extends `pAmk` + xMap*: Table[NodeKey,VertexID] ## For pretty printing, extends `pAmk` static: # Not that there is no doubt about this ... @@ -109,45 +139,6 @@ proc `==`*(a, b: VertexID): bool {.borrow.} proc cmp*(a, b: VertexID): int {.borrow.} proc `$`*(a: VertexID): string = $a.uint64 -# ------------------------------------------------------------------------------ -# Public functions for `VertexID` management -# ------------------------------------------------------------------------------ - -proc new*(T: type VertexID; db: AristoDbRef): T = - ## Create a new `VertexID`. Reusable *ID*s are kept in a list where the top - ## entry *ID0* has the property that any other *ID* larger *ID0* is also not - ## not used on the database. - case db.vidGen.len: - of 0: - db.vidGen = @[2.VertexID] - result = 1.VertexID - of 1: - result = db.vidGen[^1] - db.vidGen = @[(result.uint64 + 1).VertexID] - else: - result = db.vidGen[^2] - db.vidGen[^2] = db.vidGen[^1] - db.vidGen.setLen(db.vidGen.len-1) - -proc peek*(T: type VertexID; db: AristoDbRef): T = - ## Like `new()` without consuming this *ID*. It will return the *ID* that - ## would be returned by the `new()` function. - if db.vidGen.len == 0: 1u64 else: db.vidGen[^1] - - -proc dispose*(db: AristoDbRef; vtxID: VertexID) = - ## Recycle the argument `vtxID` which is useful after deleting entries from - ## the vertex table to prevent the `VertexID` type key values small. - if db.vidGen.len == 0: - db.vidGen = @[vtxID] - else: - let topID = db.vidGen[^1] - # No need to store smaller numbers: all numberts larger than `topID` - # are free numbers - if vtxID < topID: - db.vidGen[^1] = vtxID - db.vidGen.add topID - # ------------------------------------------------------------------------------ # Public helpers: `NodeRef` and `PayloadRef` # ------------------------------------------------------------------------------ @@ -184,11 +175,11 @@ proc `==`*(a, b: VertexRef): bool = if a.lPfx != b.lPfx or a.lData != b.lData: return false of Extension: - if a.ePfx != b.ePfx or a.eVtx != b.eVtx: + if a.ePfx != b.ePfx or a.eVid != b.eVid: return false of Branch: for n in 0..15: - if a.bVtx[n] != b.bVtx[n]: + if a.bVid[n] != b.bVid[n]: return false true @@ -202,7 +193,7 @@ proc `==`*(a, b: NodeRef): bool = return false of Branch: for n in 0..15: - if a.bVtx[n] != 0.VertexID and a.key[n] != b.key[n]: + if a.bVid[n] != 0.VertexID and a.key[n] != b.key[n]: return false else: discard diff --git a/nimbus/db/aristo/aristo_get.nim b/nimbus/db/aristo/aristo_get.nim new file mode 100644 index 000000000..0e7dc2a4a --- /dev/null +++ b/nimbus/db/aristo/aristo_get.nim @@ -0,0 +1,55 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +## Read vertex recorfd on the layered Aristo DB delta architecture +## =============================================================== + +{.push raises: [].} + +import + std/tables, + stew/results, + "."/[aristo_desc, aristo_error] + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc getVtxCascaded*( + db: AristoDbRef; + vid: VertexID; + ): Result[VertexRef,AristoError] = + ## Cascaded lookup for data record down the transaction cascade. + db.sTab.withValue(vid, vtxPtr): + return ok vtxPtr[] + + # Down the rabbit hole of transaction layers + var lDb = db + while lDb.cascaded: + lDb = lDb.stack + lDb.sTab.withValue(vid, vtxPtr): + return ok vtxPtr[] + + let be = lDb.backend + if not be.isNil: + return be.getVtxFn vid + + err(GetVtxNotFound) + +proc getVtx*(db: AristoDbRef; vid: VertexID): VertexRef = + ## Variant of `getVtxCascaded()` with returning `nil` on error ignoring the + ## error type information. + let rc = db.getVtxCascaded vid + if rc.isOk: + return rc.value + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_init.nim b/nimbus/db/aristo/aristo_init.nim new file mode 100644 index 000000000..bd525170d --- /dev/null +++ b/nimbus/db/aristo/aristo_init.nim @@ -0,0 +1,42 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +## Backend or cascaded constructors for Aristo DB +## ============================================== +## +## For a backend-less constructor use `AristoDbRef.new()` + +{.push raises: [].} + +import + ./aristo_init/[aristo_memory], + ./aristo_desc + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc init*(T: type AristoDbRef): T = + ## Constructor with memory backend. + T(cascaded: false, backend: memoryBackend()) + +proc init*(T: type AristoDbRef; db: T): T = + ## Cascaded constructor, a new layer is pushed and returned. + result = T(cascaded: true, stack: db) + if db.cascaded: + result.level = db.level + 1 + result.base = db.base + else: + result.level = 1 + result.base = db + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_init/aristo_memory.nim b/nimbus/db/aristo/aristo_init/aristo_memory.nim new file mode 100644 index 000000000..5ec4caa90 --- /dev/null +++ b/nimbus/db/aristo/aristo_init/aristo_memory.nim @@ -0,0 +1,80 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +## In-memory backend for Aristo DB +## =============================== + +{.push raises: [].} + +import + std/tables, + stew/results, + ../../../sync/snap/range_desc, + ".."/[aristo_desc, aristo_error] + +type + MemBackendRef = ref object + sTab: Table[VertexID,VertexRef] ## Structural vertex table making up a trie + kMap: Table[VertexID,NodeKey] ## Merkle hash key mapping + +# ------------------------------------------------------------------------------ +# Private functions +# ------------------------------------------------------------------------------ + +proc getVtxFn(db: MemBackendRef): GetVtxFn = + result = + proc(vid: VertexID): Result[VertexRef,AristoError] = + db.sTab.withValue(vid, vtxPtr): + return ok vtxPtr[] + err(MemBeVtxNotFound) + +proc getKeyFn(db: MemBackendRef): GetKeyFn = + result = + proc(vid: VertexID): Result[NodeKey,AristoError] = + db.kMap.withValue(vid, keyPtr): + return ok keyPtr[] + err(MemBeKeyNotFound) + +proc putVtxFn(db: MemBackendRef): PutVtxFn = + result = + proc(vrps: openArray[(VertexID,VertexRef)]): AristoError = + for (vid,vtx) in vrps: + db.sTab[vid] = vtx + +proc putKeyFn(db: MemBackendRef): PutKeyFn = + result = + proc(vkps: openArray[(VertexID,NodeKey)]): AristoError = + for (vid,key) in vkps: + db.kMap[vid] = key + +proc delFn(db: MemBackendRef): DelFn = + result = + proc(vids: openArray[VertexID]) = + for vid in vids: + db.sTab.del vid + db.kMap.del vid + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc memoryBackend*(): AristoBackendRef = + let db = MemBackendRef() + + AristoBackendRef( + getVtxFn: getVtxFn db, + getKeyFn: getKeyFn db, + putVtxFn: putVtxFn db, + putKeyFn: putKeyFn db, + delFn: delFn db) + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_transcode.nim b/nimbus/db/aristo/aristo_transcode.nim index 8763807f0..fb4735c65 100644 --- a/nimbus/db/aristo/aristo_transcode.nim +++ b/nimbus/db/aristo/aristo_transcode.nim @@ -15,11 +15,7 @@ import eth/[common, trie/nibbles], stew/results, ../../sync/snap/range_desc, - "."/[aristo_desc, aristo_error] - -const - EmptyBlob = seq[byte].default - ## Useful shortcut (borrowed from `sync/snap/constants.nim`) + "."/[aristo_constants, aristo_desc, aristo_error] # ------------------------------------------------------------------------------ # Private functions @@ -165,9 +161,9 @@ proc blobify*(node: VertexRef; data: var Blob): AristoError = refs: Blob keys: Blob for n in 0..15: - if not node.bVtx[n].isZero: + if not node.bVid[n].isZero: access = access or (1u16 shl n) - refs &= node.bVtx[n].uint64.toBytesBE.toSeq + refs &= node.bVid[n].uint64.toBytesBE.toSeq data = refs & access.toBytesBE.toSeq & @[0u8] of Extension: let @@ -175,7 +171,7 @@ proc blobify*(node: VertexRef; data: var Blob): AristoError = psLen = pSegm.len.byte if psLen == 0 or 33 < pslen: return VtxExPathOverflow - data = node.eVtx.uint64.toBytesBE.toSeq & pSegm & @[0x80u8 or psLen] + data = node.eVid.uint64.toBytesBE.toSeq & pSegm & @[0x80u8 or psLen] of Leaf: let pSegm = node.lPfx.hexPrefixEncode(isleaf = true) @@ -246,7 +242,7 @@ proc deblobify*(record: Blob; vtx: var VertexRef): AristoError = # End `while` vtx = VertexRef( vType: Branch, - bVtx: vtxList) + bVid: vtxList) of 2: # `Extension` node let @@ -261,7 +257,7 @@ proc deblobify*(record: Blob; vtx: var VertexRef): AristoError = return DbrExtGotLeafPrefix vtx = VertexRef( vType: Extension, - eVtx: (uint64.fromBytesBE record[0 ..< 8]).VertexID, + eVid: (uint64.fromBytesBE record[0 ..< 8]).VertexID, ePfx: pathSegment) of 3: # `Leaf` node diff --git a/nimbus/db/aristo/aristo_vid.nim b/nimbus/db/aristo/aristo_vid.nim new file mode 100644 index 000000000..cfe0eacd5 --- /dev/null +++ b/nimbus/db/aristo/aristo_vid.nim @@ -0,0 +1,79 @@ +# nimbus-eth1 +# Copyright (c) 2021 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +## Handle vertex IDs on the layered Aristo DB delta architecture +## ============================================================= + +{.push raises: [].} + +import + ./aristo_desc + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc vidFetch*(db: AristoDbRef): VertexID = + ## Create a new `VertexID`. Reusable *ID*s are kept in a list where the top + ## entry *ID0* has the property that any other *ID* larger *ID0* is also not + ## not used on the database. + + # Down the rabbit hole of transaction layers + let xDb = if db.cascaded: db.base else: db + + case xDb.vidGen.len: + of 0: + xDb.vidGen = @[2.VertexID] + result = 1.VertexID + of 1: + result = xDb.vidGen[^1] + xDb.vidGen = @[(result.uint64 + 1).VertexID] + else: + result = xDb.vidGen[^2] + xDb.vidGen[^2] = xDb.vidGen[^1] + xDb.vidGen.setLen(xDb.vidGen.len-1) + + +proc vidPeek*(db: AristoDbRef): VertexID = + ## Like `new()` without consuming this *ID*. It will return the *ID* that + ## would be returned by the `new()` function. + + # Down the rabbit hole of transaction layers + let xDb = if db.cascaded: db.base else: db + + case xDb.vidGen.len: + of 0: + 1.VertexID + of 1: + xDb.vidGen[^1] + else: + xDb.vidGen[^2] + + +proc vidDispose*(db: AristoDbRef; vtxID: VertexID) = + ## Recycle the argument `vtxID` which is useful after deleting entries from + ## the vertex table to prevent the `VertexID` type key values small. + + # Down the rabbit hole of transaction layers + let xDb = if db.cascaded: db.base else: db + + if xDb.vidGen.len == 0: + xDb.vidGen = @[vtxID] + else: + let topID = xDb.vidGen[^1] + # No need to store smaller numbers: all numberts larger than `topID` + # are free numbers + if vtxID < topID: + xDb.vidGen[^1] = vtxID + xDb.vidGen.add topID + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ diff --git a/tests/test_aristo/test_transcode.nim b/tests/test_aristo/test_transcode.nim index 9ca0b7556..0bf21db74 100644 --- a/tests/test_aristo/test_transcode.nim +++ b/tests/test_aristo/test_transcode.nim @@ -17,7 +17,8 @@ import unittest2, ../../nimbus/db/kvstore_rocksdb, ../../nimbus/db/aristo/[ - aristo_desc, aristo_cache, aristo_debug, aristo_error, aristo_transcode], + aristo_desc, aristo_cache, aristo_debug, aristo_error, aristo_transcode, + aristo_vid], ../../nimbus/sync/snap/range_desc, ./test_helpers @@ -119,12 +120,12 @@ proc test_transcodeAccounts*( of aristo_desc.Extension: # key <-> vtx correspondence check node.key[0] == node0.key[0] - check not node.eVtx.isZero + check not node.eVid.isZero of aristo_desc.Branch: for n in 0..15: # key[n] <-> vtx[n] correspondence check node.key[n] == node0.key[n] - check node.key[n].isZero == node.bVtx[n].isZero + check node.key[n].isZero == node.bVid[n].isZero # This NIM object must match to the same RLP encoded byte stream block: @@ -179,7 +180,7 @@ proc test_transcodeVidRecycleLists*(noisy = true; seed = 42) = # Add some randum numbers block: let first = td.vidRand() - db.dispose first + db.vidDispose first var expectedVids = 1 @@ -189,7 +190,7 @@ proc test_transcodeVidRecycleLists*(noisy = true; seed = 42) = count.inc let vid = td.vidRand() expectedVids += (vid < first).ord - db.dispose vid + db.vidDispose vid check db.vidGen.len == expectedVids noisy.say "***", "vids=", db.vidGen.len, " discarded=", count-expectedVids @@ -210,20 +211,20 @@ proc test_transcodeVidRecycleLists*(noisy = true; seed = 42) = # Make sure that recycled numbers are fetched first let topVid = db.vidGen[^1] while 1 < db.vidGen.len: - let w = VertexID.new(db) + let w = db.vidFetch() check w < topVid check db.vidGen.len == 1 and db.vidGen[0] == topVid # Get some consecutive vertex IDs for n in 0 .. 5: - let w = VertexID.new(db) + let w = db.vidFetch() check w == topVid + n check db.vidGen.len == 1 # Repeat last test after clearing the cache db.vidGen.setLen(0) for n in 0 .. 5: - let w = VertexID.new(db) + let w = db.vidFetch() check w == 1.VertexID + n check db.vidGen.len == 1