Multi layer architecture 4 aristo db (#1581)
* Cosmetics, renamed fields (eVtx, bVtx) -> (eVid, bVid) * Multilayered delta architecture for Aristo DB details: Any VertexID or data retrieval needs to go down the rabbit hole and fetch/get/manipulate the bottom layer -- even without explicit backend. * Direct reference to backend from top-level layer why: Some services as the vid management needs to be synchronised among all layers. So access is optimised.
This commit is contained in:
Jordan Hrycaj
GitHub
parent
33fd8b1fae
commit
ff0fc98fdf
|
|
@ -211,14 +211,14 @@ and implemented as 64 bit values, stored *Big Endian* in the serialisation.
|
|||
| | -- first vertexID
|
||||
8 +--+--+--+--+--+--+--+--+--+
|
||||
... -- more vertexIDs
|
||||
+--+--+
|
||||
| | -- access(16) bitmap
|
||||
+--+--+
|
||||
| | -- access(16) bitmap
|
||||
+--+--+
|
||||
|| | -- marker(2) + unused(6)
|
||||
+--+
|
||||
|
||||
where
|
||||
marker(2) is the double bit array 00
|
||||
where
|
||||
marker(2) is the double bit array 00
|
||||
|
||||
For a given index *n* between *0..15*, if the bit at position *n* of the it
|
||||
vector *access(16)* is reset to zero, then there is no *n*-th structural
|
||||
|
|
@ -240,8 +240,8 @@ stored in the right byte of the serialised bitmap.
|
|||
|| | -- marker(2) + pathSegmentLen(6)
|
||||
+--+
|
||||
|
||||
where
|
||||
marker(2) is the double bit array 10
|
||||
where
|
||||
marker(2) is the double bit array 10
|
||||
|
||||
The path segment of the *Extension* record is compact encoded. So it has at
|
||||
least one byte. The first byte *P0* has bit 5 reset, i.e. *P0 and 0x20* is
|
||||
|
|
@ -260,8 +260,8 @@ of the extension record (as *recordLen - 9*.)
|
|||
|| | -- marker(2) + pathSegmentLen(6)
|
||||
+--+
|
||||
|
||||
where
|
||||
marker(2) is the double bit array 11
|
||||
where
|
||||
marker(2) is the double bit array 11
|
||||
|
||||
A *Leaf* record path segment is compact encoded. So it has at least one byte.
|
||||
The first byte *P0* has bit 5 set, i.e. *P0 and 0x20* is non-zero (bit 4 is
|
||||
|
|
@ -277,8 +277,8 @@ also set if the right nibble is the first part of the path.)
|
|||
|| | -- marker(2) + unused(6)
|
||||
+--+
|
||||
|
||||
where
|
||||
marker(2) is the double bit array 01
|
||||
where
|
||||
marker(2) is the double bit array 01
|
||||
|
||||
Currently, the descriptor record only contains data for producing unique
|
||||
vectorID values that can be used as structural keys. If this descriptor is
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import
|
|||
eth/common,
|
||||
stew/results,
|
||||
../../sync/snap/range_desc,
|
||||
"."/[aristo_desc, aristo_error, aristo_transcode]
|
||||
"."/[aristo_desc, aristo_error, aristo_transcode, aristo_vid]
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Private helpers
|
||||
|
|
@ -39,22 +39,22 @@ proc convertPartially(
|
|||
nd = NodeRef(
|
||||
vType: Extension,
|
||||
ePfx: vtx.ePfx,
|
||||
eVtx: vtx.eVtx)
|
||||
db.kMap.withValue(vtx.eVtx, keyPtr):
|
||||
eVid: vtx.eVid)
|
||||
db.kMap.withValue(vtx.eVid, keyPtr):
|
||||
nd.key[0] = keyPtr[]
|
||||
return
|
||||
result.add vtx.eVtx
|
||||
result.add vtx.eVid
|
||||
of Branch:
|
||||
nd = NodeRef(
|
||||
vType: Branch,
|
||||
bVtx: vtx.bVtx)
|
||||
bVid: vtx.bVid)
|
||||
for n in 0..15:
|
||||
if vtx.bVtx[n].isZero:
|
||||
if vtx.bVid[n].isZero:
|
||||
continue
|
||||
db.kMap.withValue(vtx.bVtx[n], kPtr):
|
||||
db.kMap.withValue(vtx.bVid[n], kPtr):
|
||||
nd.key[n] = kPtr[]
|
||||
continue
|
||||
result.add vtx.bVtx[n]
|
||||
result.add vtx.bVid[n]
|
||||
|
||||
proc convertPartiallyOk(
|
||||
db: AristoDbRef;
|
||||
|
|
@ -73,18 +73,18 @@ proc convertPartiallyOk(
|
|||
nd = NodeRef(
|
||||
vType: Extension,
|
||||
ePfx: vtx.ePfx,
|
||||
eVtx: vtx.eVtx)
|
||||
db.kMap.withValue(vtx.eVtx, keyPtr):
|
||||
eVid: vtx.eVid)
|
||||
db.kMap.withValue(vtx.eVid, keyPtr):
|
||||
nd.key[0] = keyPtr[]
|
||||
result = true
|
||||
of Branch:
|
||||
nd = NodeRef(
|
||||
vType: Branch,
|
||||
bVtx: vtx.bVtx)
|
||||
bVid: vtx.bVid)
|
||||
result = true
|
||||
for n in 0..15:
|
||||
if not vtx.bVtx[n].isZero:
|
||||
db.kMap.withValue(vtx.bVtx[n], kPtr):
|
||||
if not vtx.bVid[n].isZero:
|
||||
db.kMap.withValue(vtx.bVid[n], kPtr):
|
||||
nd.key[n] = kPtr[]
|
||||
continue
|
||||
return false
|
||||
|
|
@ -93,7 +93,7 @@ proc cachedVID(db: AristoDbRef; nodeKey: NodeKey): VertexID =
|
|||
## Get vertex ID from reverse cache
|
||||
db.pAmk.withValue(nodeKey, vidPtr):
|
||||
return vidPtr[]
|
||||
result = VertexID.new(db)
|
||||
result = db.vidFetch()
|
||||
db.pAmk[nodeKey] = result
|
||||
db.kMap[result] = nodeKey
|
||||
|
||||
|
|
@ -145,7 +145,7 @@ proc updated*(nd: NodeRef; db: AristoDbRef): NodeRef =
|
|||
vType: Extension,
|
||||
ePfx: nd.ePfx)
|
||||
if not nd.key[0].isZero:
|
||||
result.eVtx = db.cachedVID nd.key[0]
|
||||
result.eVid = db.cachedVID nd.key[0]
|
||||
result.key[0] = nd.key[0]
|
||||
of Branch:
|
||||
result = NodeRef(
|
||||
|
|
@ -153,7 +153,7 @@ proc updated*(nd: NodeRef; db: AristoDbRef): NodeRef =
|
|||
key: nd.key)
|
||||
for n in 0..15:
|
||||
if not nd.key[n].isZero:
|
||||
result.bVtx[n] = db.cachedVID nd.key[n]
|
||||
result.bVid[n] = db.cachedVID nd.key[n]
|
||||
|
||||
proc asNode*(vtx: VertexRef; db: AristoDbRef): NodeRef =
|
||||
## Return a `NodeRef` object by augmenting missing `Merkel` hashes (aka
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
# nimbus-eth1
|
||||
# Copyright (c) 2021 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import
|
||||
../../sync/snap/range_desc,
|
||||
eth/[common, trie/nibbles]
|
||||
|
||||
const
|
||||
EmptyBlob* = seq[byte].default
|
||||
## Useful shortcut (borrowed from `sync/snap/constants.nim`)
|
||||
|
||||
EmptyNibbleSeq* = EmptyBlob.initNibbleRange
|
||||
## Useful shortcut (borrowed from `sync/snap/constants.nim`)
|
||||
|
||||
EMPTY_ROOT_KEY* = EMPTY_ROOT_HASH.to(NodeKey)
|
||||
|
||||
EMPTY_CODE_KEY* = EMPTY_CODE_HASH.to(NodeKey)
|
||||
|
||||
# End
|
||||
|
|
@ -15,11 +15,7 @@ import
|
|||
eth/[common, trie/nibbles],
|
||||
stew/byteutils,
|
||||
../../sync/snap/range_desc,
|
||||
"."/aristo_desc
|
||||
|
||||
const
|
||||
EMPTY_ROOT_KEY = EMPTY_ROOT_HASH.to(NodeKey)
|
||||
EMPTY_CODE_KEY = EMPTY_CODE_HASH.to(NodeKey)
|
||||
"."/[aristo_constants, aristo_desc, aristo_vid]
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Ptivate functions
|
||||
|
|
@ -101,7 +97,7 @@ proc keyToVtxID*(db: AristoDbRef, key: NodeKey): VertexID =
|
|||
db.xMap.withValue(key, vidPtr):
|
||||
return vidPtr[]
|
||||
|
||||
result = VertexID.new db
|
||||
result = db.vidFetch()
|
||||
db.xMap[key] = result
|
||||
|
||||
proc pp*(vid: openArray[VertexID]): string =
|
||||
|
|
@ -130,12 +126,12 @@ proc pp*(nd: VertexRef, db = AristoDbRef(nil)): string =
|
|||
of Leaf:
|
||||
result &= $nd.lPfx & "," & nd.lData.pp(db)
|
||||
of Extension:
|
||||
result &= $nd.ePfx & "," & nd.eVtx.ppVid
|
||||
result &= $nd.ePfx & "," & nd.eVid.ppVid
|
||||
of Branch:
|
||||
result &= "["
|
||||
for n in 0..15:
|
||||
if not nd.bVtx[n].isZero:
|
||||
result &= nd.bVtx[n].ppVid
|
||||
if not nd.bVid[n].isZero:
|
||||
result &= nd.bVid[n].ppVid
|
||||
result &= ","
|
||||
result[^1] = ']'
|
||||
result &= ")"
|
||||
|
|
@ -152,19 +148,19 @@ proc pp*(nd: NodeRef, db = AristoDbRef(nil)): string =
|
|||
result &= $nd.lPfx & "," & nd.lData.pp(db)
|
||||
|
||||
of Extension:
|
||||
result &= $nd.ePfx & "," & nd.eVtx.ppVid & "," & nd.key[0].ppKey
|
||||
result &= $nd.ePfx & "," & nd.eVid.ppVid & "," & nd.key[0].ppKey
|
||||
|
||||
of Branch:
|
||||
result &= "["
|
||||
for n in 0..15:
|
||||
if not nd.bVtx[n].isZero or not nd.key[n].isZero:
|
||||
result &= nd.bVtx[n].ppVid
|
||||
result &= db.keyVidUpdate(nd.key[n], nd.bVtx[n]) & ","
|
||||
if not nd.bVid[n].isZero or not nd.key[n].isZero:
|
||||
result &= nd.bVid[n].ppVid
|
||||
result &= db.keyVidUpdate(nd.key[n], nd.bVid[n]) & ","
|
||||
result[^1] = ']'
|
||||
|
||||
result &= ",["
|
||||
for n in 0..15:
|
||||
if not nd.bVtx[n].isZero or not nd.key[n].isZero:
|
||||
if not nd.bVid[n].isZero or not nd.key[n].isZero:
|
||||
result &= nd.key[n].ppKey(db)
|
||||
result &= ","
|
||||
result[^1] = ']'
|
||||
|
|
|
|||
|
|
@ -13,88 +13,118 @@
|
|||
##
|
||||
## These data structures allows to overlay the *Patricia Trie* with *Merkel
|
||||
## Trie* hashes. See the `README.md` in the `aristo` folder for documentation.
|
||||
|
||||
##
|
||||
## Some semantic explanations;
|
||||
##
|
||||
## * NodeKey, NodeRef etc. refer to the standard/legacy `Merkel Patricia Tree`
|
||||
## * VertexID, VertexRef, etc. refer to the `Aristo Trie`
|
||||
##
|
||||
{.push raises: [].}
|
||||
|
||||
import
|
||||
std/tables,
|
||||
std/[sets, tables],
|
||||
eth/[common, trie/nibbles],
|
||||
stew/results,
|
||||
../../sync/snap/range_desc,
|
||||
./aristo_error
|
||||
|
||||
type
|
||||
VertexID* = distinct uint64 ## Tip of edge towards child, also table key
|
||||
VertexID* = distinct uint64
|
||||
## Tip of edge towards child object in the `Patricia Trie` logic. It is
|
||||
## also the key into the structural table of the `Aristo Trie`.
|
||||
|
||||
VertexType* = enum ## Type of Patricia Trie node
|
||||
GetVtxFn* =
|
||||
proc(vid: VertexID): Result[VertexRef,AristoError]
|
||||
{.gcsafe, raises: [].}
|
||||
## Generic backend database retrieval function for a single structural
|
||||
## `Aristo DB` data record.
|
||||
|
||||
GetKeyFn* =
|
||||
proc(vid: VertexID): Result[NodeKey,AristoError]
|
||||
{.gcsafe, raises: [].}
|
||||
## Generic backend database retrieval function for a single
|
||||
## `Aristo DB` hash lookup value.
|
||||
|
||||
PutVtxFn* =
|
||||
proc(vrps: openArray[(VertexID,VertexRef)]): AristoError
|
||||
{.gcsafe, raises: [].}
|
||||
## Generic backend database bulk storage function.
|
||||
|
||||
PutKeyFn* =
|
||||
proc(vkps: openArray[(VertexID,NodeKey)]): AristoError
|
||||
{.gcsafe, raises: [].}
|
||||
## Generic backend database bulk storage function.
|
||||
|
||||
DelFn* =
|
||||
proc(vids: openArray[VertexID])
|
||||
{.gcsafe, raises: [].}
|
||||
## Generic backend database delete function for both, the structural
|
||||
## `Aristo DB` data record and the hash lookup value.
|
||||
|
||||
VertexType* = enum
|
||||
## Type of `Aristo Trie` vertex
|
||||
Leaf
|
||||
Extension
|
||||
Branch
|
||||
|
||||
PayloadType* = enum ## Type of leaf data (to be extended)
|
||||
BlobData
|
||||
AccountData
|
||||
PayloadType* = enum
|
||||
## Type of leaf data (to be extended)
|
||||
BlobData ## Generic data, typically RLP encoded
|
||||
AccountData ## Legacy `Account` with hash references
|
||||
# AristoAccount ## `Aristo account` with vertex IDs links
|
||||
|
||||
PayloadRef* = ref object
|
||||
case pType*: PayloadType
|
||||
of BlobData:
|
||||
blob*: Blob ## Opaque data value reference
|
||||
blob*: Blob ## Opaque data value reference
|
||||
of AccountData:
|
||||
account*: Account ## Expanded accounting data
|
||||
account*: Account ## Expanded accounting data
|
||||
|
||||
VertexRef* = ref object of RootRef
|
||||
## Vertex for building a hexary Patricia or Merkle Patricia Trie
|
||||
case vType*: VertexType
|
||||
of Leaf:
|
||||
lPfx*: NibblesSeq ## Portion of path segment
|
||||
lData*: PayloadRef ## Reference to data payload
|
||||
lPfx*: NibblesSeq ## Portion of path segment
|
||||
lData*: PayloadRef ## Reference to data payload
|
||||
of Extension:
|
||||
ePfx*: NibblesSeq ## Portion of path segment
|
||||
eVtx*: VertexID ## Edge to vertex with ID `eVtx`
|
||||
ePfx*: NibblesSeq ## Portion of path segment
|
||||
eVid*: VertexID ## Edge to vertex with ID `eVid`
|
||||
of Branch:
|
||||
bVtx*: array[16,VertexID] ## Edge list with vertex IDs
|
||||
bVid*: array[16,VertexID] ## Edge list with vertex IDs
|
||||
|
||||
NodeRef* = ref object of VertexRef
|
||||
## Combined record for a *traditional* ``Merkle Patricia Tree` node merged
|
||||
## with a structural `VertexRef` type object.
|
||||
error*: AristoError ## Can be used for error signalling
|
||||
key*: array[16,NodeKey] ## Merkle hash(es) for Branch & Extension vtx
|
||||
error*: AristoError ## Can be used for error signalling
|
||||
key*: array[16,NodeKey] ## Merkle hash/es for Branch & Extension
|
||||
|
||||
PathStep* = object
|
||||
## For constructing a tree traversal path
|
||||
# key*: NodeKey ## Node label ??
|
||||
node*: VertexRef ## Referes to data record
|
||||
nibble*: int8 ## Branch node selector (if any)
|
||||
depth*: int ## May indicate path length (typically 64)
|
||||
AristoBackendRef* = ref object
|
||||
## Backend interface.
|
||||
getVtxFn*: GetVtxFn ## Read vertex record
|
||||
getKeyFn*: GetKeyFn ## Read vertex hash
|
||||
putVtxFn*: PutVtxFn ## Bulk store vertex records
|
||||
putKeyFn*: PutKeyFn ## Bulk store vertex hashes
|
||||
delFn*: DelFn ## Bulk delete vertex records and hashes
|
||||
|
||||
Path* = object
|
||||
root*: VertexID ## Root node needed when `path.len == 0`
|
||||
path*: seq[PathStep] ## Chain of nodes
|
||||
tail*: NibblesSeq ## Stands for non completed leaf path
|
||||
|
||||
LeafSpecs* = object
|
||||
## Temporarily stashed leaf data (as for an account.) Proper records
|
||||
## have non-empty payload. Records with empty payload are administrative
|
||||
## items, e.g. lower boundary records.
|
||||
pathTag*: NodeTag ## `Patricia Trie` key path
|
||||
nodeVtx*: VertexID ## Table lookup vertex ID (if any)
|
||||
payload*: PayloadRef ## Reference to data payload
|
||||
|
||||
GetFn* = proc(key: openArray[byte]): Blob
|
||||
{.gcsafe, raises: [CatchableError].}
|
||||
## Persistent database `get()` function. For read-only cases, this
|
||||
## function can be seen as the persistent alternative to ``tab[]` on
|
||||
## a `HexaryTreeDbRef` descriptor.
|
||||
|
||||
AristoDbRef* = ref object of RootObj
|
||||
AristoDbRef* = ref AristoDbObj
|
||||
AristoDbObj = object
|
||||
## Hexary trie plus helper structures
|
||||
sTab*: Table[VertexID,NodeRef] ## Structural vertex table making up a trie
|
||||
kMap*: Table[VertexID,NodeKey] ## Merkle hash key mapping
|
||||
pAmk*: Table[NodeKey,VertexID] ## Reverse mapper for data import
|
||||
vidGen*: seq[VertexID] ## Unique vertex ID generator
|
||||
sTab*: Table[VertexID,VertexRef] ## Structural vertex table making up a trie
|
||||
sDel*: HashSet[VertexID] ## Deleted vertices
|
||||
kMap*: Table[VertexID,NodeKey] ## Merkle hash key mapping
|
||||
pAmk*: Table[NodeKey,VertexID] ## Reverse mapper for data import
|
||||
|
||||
case cascaded*: bool ## Cascaded delta databases, tx layer
|
||||
of true:
|
||||
level*: int ## Positive number of stack layers
|
||||
stack*: AristoDbRef ## Down the chain, not `nil`
|
||||
base*: AristoDbRef ## Backend level descriptor
|
||||
else:
|
||||
vidGen*: seq[VertexID] ## Unique vertex ID generator
|
||||
backend*: AristoBackendRef ## backend database (maybe `nil`)
|
||||
|
||||
# Debugging data below, might go away in future
|
||||
xMap*: Table[NodeKey,VertexID] ## Mapper for pretty printing, extends `pAmk`
|
||||
xMap*: Table[NodeKey,VertexID] ## For pretty printing, extends `pAmk`
|
||||
|
||||
static:
|
||||
# Not that there is no doubt about this ...
|
||||
|
|
@ -109,45 +139,6 @@ proc `==`*(a, b: VertexID): bool {.borrow.}
|
|||
proc cmp*(a, b: VertexID): int {.borrow.}
|
||||
proc `$`*(a: VertexID): string = $a.uint64
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public functions for `VertexID` management
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc new*(T: type VertexID; db: AristoDbRef): T =
|
||||
## Create a new `VertexID`. Reusable *ID*s are kept in a list where the top
|
||||
## entry *ID0* has the property that any other *ID* larger *ID0* is also not
|
||||
## not used on the database.
|
||||
case db.vidGen.len:
|
||||
of 0:
|
||||
db.vidGen = @[2.VertexID]
|
||||
result = 1.VertexID
|
||||
of 1:
|
||||
result = db.vidGen[^1]
|
||||
db.vidGen = @[(result.uint64 + 1).VertexID]
|
||||
else:
|
||||
result = db.vidGen[^2]
|
||||
db.vidGen[^2] = db.vidGen[^1]
|
||||
db.vidGen.setLen(db.vidGen.len-1)
|
||||
|
||||
proc peek*(T: type VertexID; db: AristoDbRef): T =
|
||||
## Like `new()` without consuming this *ID*. It will return the *ID* that
|
||||
## would be returned by the `new()` function.
|
||||
if db.vidGen.len == 0: 1u64 else: db.vidGen[^1]
|
||||
|
||||
|
||||
proc dispose*(db: AristoDbRef; vtxID: VertexID) =
|
||||
## Recycle the argument `vtxID` which is useful after deleting entries from
|
||||
## the vertex table to prevent the `VertexID` type key values small.
|
||||
if db.vidGen.len == 0:
|
||||
db.vidGen = @[vtxID]
|
||||
else:
|
||||
let topID = db.vidGen[^1]
|
||||
# No need to store smaller numbers: all numberts larger than `topID`
|
||||
# are free numbers
|
||||
if vtxID < topID:
|
||||
db.vidGen[^1] = vtxID
|
||||
db.vidGen.add topID
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public helpers: `NodeRef` and `PayloadRef`
|
||||
# ------------------------------------------------------------------------------
|
||||
|
|
@ -184,11 +175,11 @@ proc `==`*(a, b: VertexRef): bool =
|
|||
if a.lPfx != b.lPfx or a.lData != b.lData:
|
||||
return false
|
||||
of Extension:
|
||||
if a.ePfx != b.ePfx or a.eVtx != b.eVtx:
|
||||
if a.ePfx != b.ePfx or a.eVid != b.eVid:
|
||||
return false
|
||||
of Branch:
|
||||
for n in 0..15:
|
||||
if a.bVtx[n] != b.bVtx[n]:
|
||||
if a.bVid[n] != b.bVid[n]:
|
||||
return false
|
||||
true
|
||||
|
||||
|
|
@ -202,7 +193,7 @@ proc `==`*(a, b: NodeRef): bool =
|
|||
return false
|
||||
of Branch:
|
||||
for n in 0..15:
|
||||
if a.bVtx[n] != 0.VertexID and a.key[n] != b.key[n]:
|
||||
if a.bVid[n] != 0.VertexID and a.key[n] != b.key[n]:
|
||||
return false
|
||||
else:
|
||||
discard
|
||||
|
|
|
|||
|
|
@ -0,0 +1,55 @@
|
|||
# nimbus-eth1
|
||||
# Copyright (c) 2021 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
## Read vertex recorfd on the layered Aristo DB delta architecture
|
||||
## ===============================================================
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import
|
||||
std/tables,
|
||||
stew/results,
|
||||
"."/[aristo_desc, aristo_error]
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public functions
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc getVtxCascaded*(
|
||||
db: AristoDbRef;
|
||||
vid: VertexID;
|
||||
): Result[VertexRef,AristoError] =
|
||||
## Cascaded lookup for data record down the transaction cascade.
|
||||
db.sTab.withValue(vid, vtxPtr):
|
||||
return ok vtxPtr[]
|
||||
|
||||
# Down the rabbit hole of transaction layers
|
||||
var lDb = db
|
||||
while lDb.cascaded:
|
||||
lDb = lDb.stack
|
||||
lDb.sTab.withValue(vid, vtxPtr):
|
||||
return ok vtxPtr[]
|
||||
|
||||
let be = lDb.backend
|
||||
if not be.isNil:
|
||||
return be.getVtxFn vid
|
||||
|
||||
err(GetVtxNotFound)
|
||||
|
||||
proc getVtx*(db: AristoDbRef; vid: VertexID): VertexRef =
|
||||
## Variant of `getVtxCascaded()` with returning `nil` on error ignoring the
|
||||
## error type information.
|
||||
let rc = db.getVtxCascaded vid
|
||||
if rc.isOk:
|
||||
return rc.value
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# End
|
||||
# ------------------------------------------------------------------------------
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
# nimbus-eth1
|
||||
# Copyright (c) 2021 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
## Backend or cascaded constructors for Aristo DB
|
||||
## ==============================================
|
||||
##
|
||||
## For a backend-less constructor use `AristoDbRef.new()`
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import
|
||||
./aristo_init/[aristo_memory],
|
||||
./aristo_desc
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public functions
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc init*(T: type AristoDbRef): T =
|
||||
## Constructor with memory backend.
|
||||
T(cascaded: false, backend: memoryBackend())
|
||||
|
||||
proc init*(T: type AristoDbRef; db: T): T =
|
||||
## Cascaded constructor, a new layer is pushed and returned.
|
||||
result = T(cascaded: true, stack: db)
|
||||
if db.cascaded:
|
||||
result.level = db.level + 1
|
||||
result.base = db.base
|
||||
else:
|
||||
result.level = 1
|
||||
result.base = db
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# End
|
||||
# ------------------------------------------------------------------------------
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
# nimbus-eth1
|
||||
# Copyright (c) 2021 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
## In-memory backend for Aristo DB
|
||||
## ===============================
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import
|
||||
std/tables,
|
||||
stew/results,
|
||||
../../../sync/snap/range_desc,
|
||||
".."/[aristo_desc, aristo_error]
|
||||
|
||||
type
|
||||
MemBackendRef = ref object
|
||||
sTab: Table[VertexID,VertexRef] ## Structural vertex table making up a trie
|
||||
kMap: Table[VertexID,NodeKey] ## Merkle hash key mapping
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Private functions
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc getVtxFn(db: MemBackendRef): GetVtxFn =
|
||||
result =
|
||||
proc(vid: VertexID): Result[VertexRef,AristoError] =
|
||||
db.sTab.withValue(vid, vtxPtr):
|
||||
return ok vtxPtr[]
|
||||
err(MemBeVtxNotFound)
|
||||
|
||||
proc getKeyFn(db: MemBackendRef): GetKeyFn =
|
||||
result =
|
||||
proc(vid: VertexID): Result[NodeKey,AristoError] =
|
||||
db.kMap.withValue(vid, keyPtr):
|
||||
return ok keyPtr[]
|
||||
err(MemBeKeyNotFound)
|
||||
|
||||
proc putVtxFn(db: MemBackendRef): PutVtxFn =
|
||||
result =
|
||||
proc(vrps: openArray[(VertexID,VertexRef)]): AristoError =
|
||||
for (vid,vtx) in vrps:
|
||||
db.sTab[vid] = vtx
|
||||
|
||||
proc putKeyFn(db: MemBackendRef): PutKeyFn =
|
||||
result =
|
||||
proc(vkps: openArray[(VertexID,NodeKey)]): AristoError =
|
||||
for (vid,key) in vkps:
|
||||
db.kMap[vid] = key
|
||||
|
||||
proc delFn(db: MemBackendRef): DelFn =
|
||||
result =
|
||||
proc(vids: openArray[VertexID]) =
|
||||
for vid in vids:
|
||||
db.sTab.del vid
|
||||
db.kMap.del vid
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public functions
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc memoryBackend*(): AristoBackendRef =
|
||||
let db = MemBackendRef()
|
||||
|
||||
AristoBackendRef(
|
||||
getVtxFn: getVtxFn db,
|
||||
getKeyFn: getKeyFn db,
|
||||
putVtxFn: putVtxFn db,
|
||||
putKeyFn: putKeyFn db,
|
||||
delFn: delFn db)
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# End
|
||||
# ------------------------------------------------------------------------------
|
||||
|
|
@ -15,11 +15,7 @@ import
|
|||
eth/[common, trie/nibbles],
|
||||
stew/results,
|
||||
../../sync/snap/range_desc,
|
||||
"."/[aristo_desc, aristo_error]
|
||||
|
||||
const
|
||||
EmptyBlob = seq[byte].default
|
||||
## Useful shortcut (borrowed from `sync/snap/constants.nim`)
|
||||
"."/[aristo_constants, aristo_desc, aristo_error]
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Private functions
|
||||
|
|
@ -165,9 +161,9 @@ proc blobify*(node: VertexRef; data: var Blob): AristoError =
|
|||
refs: Blob
|
||||
keys: Blob
|
||||
for n in 0..15:
|
||||
if not node.bVtx[n].isZero:
|
||||
if not node.bVid[n].isZero:
|
||||
access = access or (1u16 shl n)
|
||||
refs &= node.bVtx[n].uint64.toBytesBE.toSeq
|
||||
refs &= node.bVid[n].uint64.toBytesBE.toSeq
|
||||
data = refs & access.toBytesBE.toSeq & @[0u8]
|
||||
of Extension:
|
||||
let
|
||||
|
|
@ -175,7 +171,7 @@ proc blobify*(node: VertexRef; data: var Blob): AristoError =
|
|||
psLen = pSegm.len.byte
|
||||
if psLen == 0 or 33 < pslen:
|
||||
return VtxExPathOverflow
|
||||
data = node.eVtx.uint64.toBytesBE.toSeq & pSegm & @[0x80u8 or psLen]
|
||||
data = node.eVid.uint64.toBytesBE.toSeq & pSegm & @[0x80u8 or psLen]
|
||||
of Leaf:
|
||||
let
|
||||
pSegm = node.lPfx.hexPrefixEncode(isleaf = true)
|
||||
|
|
@ -246,7 +242,7 @@ proc deblobify*(record: Blob; vtx: var VertexRef): AristoError =
|
|||
# End `while`
|
||||
vtx = VertexRef(
|
||||
vType: Branch,
|
||||
bVtx: vtxList)
|
||||
bVid: vtxList)
|
||||
|
||||
of 2: # `Extension` node
|
||||
let
|
||||
|
|
@ -261,7 +257,7 @@ proc deblobify*(record: Blob; vtx: var VertexRef): AristoError =
|
|||
return DbrExtGotLeafPrefix
|
||||
vtx = VertexRef(
|
||||
vType: Extension,
|
||||
eVtx: (uint64.fromBytesBE record[0 ..< 8]).VertexID,
|
||||
eVid: (uint64.fromBytesBE record[0 ..< 8]).VertexID,
|
||||
ePfx: pathSegment)
|
||||
|
||||
of 3: # `Leaf` node
|
||||
|
|
|
|||
|
|
@ -0,0 +1,79 @@
|
|||
# nimbus-eth1
|
||||
# Copyright (c) 2021 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
## Handle vertex IDs on the layered Aristo DB delta architecture
|
||||
## =============================================================
|
||||
|
||||
{.push raises: [].}
|
||||
|
||||
import
|
||||
./aristo_desc
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public functions
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc vidFetch*(db: AristoDbRef): VertexID =
|
||||
## Create a new `VertexID`. Reusable *ID*s are kept in a list where the top
|
||||
## entry *ID0* has the property that any other *ID* larger *ID0* is also not
|
||||
## not used on the database.
|
||||
|
||||
# Down the rabbit hole of transaction layers
|
||||
let xDb = if db.cascaded: db.base else: db
|
||||
|
||||
case xDb.vidGen.len:
|
||||
of 0:
|
||||
xDb.vidGen = @[2.VertexID]
|
||||
result = 1.VertexID
|
||||
of 1:
|
||||
result = xDb.vidGen[^1]
|
||||
xDb.vidGen = @[(result.uint64 + 1).VertexID]
|
||||
else:
|
||||
result = xDb.vidGen[^2]
|
||||
xDb.vidGen[^2] = xDb.vidGen[^1]
|
||||
xDb.vidGen.setLen(xDb.vidGen.len-1)
|
||||
|
||||
|
||||
proc vidPeek*(db: AristoDbRef): VertexID =
|
||||
## Like `new()` without consuming this *ID*. It will return the *ID* that
|
||||
## would be returned by the `new()` function.
|
||||
|
||||
# Down the rabbit hole of transaction layers
|
||||
let xDb = if db.cascaded: db.base else: db
|
||||
|
||||
case xDb.vidGen.len:
|
||||
of 0:
|
||||
1.VertexID
|
||||
of 1:
|
||||
xDb.vidGen[^1]
|
||||
else:
|
||||
xDb.vidGen[^2]
|
||||
|
||||
|
||||
proc vidDispose*(db: AristoDbRef; vtxID: VertexID) =
|
||||
## Recycle the argument `vtxID` which is useful after deleting entries from
|
||||
## the vertex table to prevent the `VertexID` type key values small.
|
||||
|
||||
# Down the rabbit hole of transaction layers
|
||||
let xDb = if db.cascaded: db.base else: db
|
||||
|
||||
if xDb.vidGen.len == 0:
|
||||
xDb.vidGen = @[vtxID]
|
||||
else:
|
||||
let topID = xDb.vidGen[^1]
|
||||
# No need to store smaller numbers: all numberts larger than `topID`
|
||||
# are free numbers
|
||||
if vtxID < topID:
|
||||
xDb.vidGen[^1] = vtxID
|
||||
xDb.vidGen.add topID
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# End
|
||||
# ------------------------------------------------------------------------------
|
||||
|
|
@ -17,7 +17,8 @@ import
|
|||
unittest2,
|
||||
../../nimbus/db/kvstore_rocksdb,
|
||||
../../nimbus/db/aristo/[
|
||||
aristo_desc, aristo_cache, aristo_debug, aristo_error, aristo_transcode],
|
||||
aristo_desc, aristo_cache, aristo_debug, aristo_error, aristo_transcode,
|
||||
aristo_vid],
|
||||
../../nimbus/sync/snap/range_desc,
|
||||
./test_helpers
|
||||
|
||||
|
|
@ -119,12 +120,12 @@ proc test_transcodeAccounts*(
|
|||
of aristo_desc.Extension:
|
||||
# key <-> vtx correspondence
|
||||
check node.key[0] == node0.key[0]
|
||||
check not node.eVtx.isZero
|
||||
check not node.eVid.isZero
|
||||
of aristo_desc.Branch:
|
||||
for n in 0..15:
|
||||
# key[n] <-> vtx[n] correspondence
|
||||
check node.key[n] == node0.key[n]
|
||||
check node.key[n].isZero == node.bVtx[n].isZero
|
||||
check node.key[n].isZero == node.bVid[n].isZero
|
||||
|
||||
# This NIM object must match to the same RLP encoded byte stream
|
||||
block:
|
||||
|
|
@ -179,7 +180,7 @@ proc test_transcodeVidRecycleLists*(noisy = true; seed = 42) =
|
|||
# Add some randum numbers
|
||||
block:
|
||||
let first = td.vidRand()
|
||||
db.dispose first
|
||||
db.vidDispose first
|
||||
|
||||
var
|
||||
expectedVids = 1
|
||||
|
|
@ -189,7 +190,7 @@ proc test_transcodeVidRecycleLists*(noisy = true; seed = 42) =
|
|||
count.inc
|
||||
let vid = td.vidRand()
|
||||
expectedVids += (vid < first).ord
|
||||
db.dispose vid
|
||||
db.vidDispose vid
|
||||
|
||||
check db.vidGen.len == expectedVids
|
||||
noisy.say "***", "vids=", db.vidGen.len, " discarded=", count-expectedVids
|
||||
|
|
@ -210,20 +211,20 @@ proc test_transcodeVidRecycleLists*(noisy = true; seed = 42) =
|
|||
# Make sure that recycled numbers are fetched first
|
||||
let topVid = db.vidGen[^1]
|
||||
while 1 < db.vidGen.len:
|
||||
let w = VertexID.new(db)
|
||||
let w = db.vidFetch()
|
||||
check w < topVid
|
||||
check db.vidGen.len == 1 and db.vidGen[0] == topVid
|
||||
|
||||
# Get some consecutive vertex IDs
|
||||
for n in 0 .. 5:
|
||||
let w = VertexID.new(db)
|
||||
let w = db.vidFetch()
|
||||
check w == topVid + n
|
||||
check db.vidGen.len == 1
|
||||
|
||||
# Repeat last test after clearing the cache
|
||||
db.vidGen.setLen(0)
|
||||
for n in 0 .. 5:
|
||||
let w = VertexID.new(db)
|
||||
let w = db.vidFetch()
|
||||
check w == 1.VertexID + n
|
||||
check db.vidGen.len == 1
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue