2023-05-11 14:25:29 +00:00
|
|
|
# nimbus-eth1
|
Core db update storage root management for sub tries (#1964)
* Aristo: Re-phrase `LayerDelta` and `LayerFinal` as object references
why:
Avoids copying in some cases
* Fix copyright header
* Aristo: Verify `leafTie.root` function argument for `merge()` proc
why:
Zero root will lead to inconsistent DB entry
* Aristo: Update failure condition for hash labels compiler `hashify()`
why:
Node need not be rejected as long as links are on the schedule. In
that case, `redo[]` is to become `wff.base[]` at a later stage.
This amends an earlier fix, part of #1952 by also testing against
the target nodes of the `wff.base[]` sets.
* Aristo: Add storage root glue record to `hashify()` schedule
why:
An account leaf node might refer to a non-resolvable storage root ID.
Storage root node chains will end up at the storage root. So the link
`storage-root->account-leaf` needs an extra item in the schedule.
* Aristo: fix error code returned by `fetchPayload()`
details:
Final error code is implied by the error code form the `hikeUp()`
function.
* CoreDb: Discard `createOk` argument in API `getRoot()` function
why:
Not needed for the legacy DB. For the `Arsto` DB, a lazy approach is
implemented where a stprage root node is created on-the-fly.
* CoreDb: Prevent `$$` logging in some cases
why:
Logging the function `$$` is not useful when it is used for internal
use, i.e. retrieving an an error text for logging.
* CoreDb: Add `tryHashFn()` to API for pretty printing
why:
Pretty printing must not change the hashification status for the
`Aristo` DB. So there is an independent API wrapper for getting the
node hash which never updated the hashes.
* CoreDb: Discard `update` argument in API `hash()` function
why:
When calling the API function `hash()`, the latest state is always
wanted. For a version that uses the current state as-is without checking,
the function `tryHash()` was added to the backend.
* CoreDb: Update opaque vertex ID objects for the `Aristo` backend
why:
For `Aristo`, vID objects encapsulate a numeric `VertexID`
referencing a vertex (rather than a node hash as used on the
legacy backend.) For storage sub-tries, there might be no initial
vertex known when the descriptor is created. So opaque vertex ID
objects are supported without a valid `VertexID` which will be
initalised on-the-fly when the first item is merged.
* CoreDb: Add pretty printer for opaque vertex ID objects
* Cosmetics, printing profiling data
* CoreDb: Fix segfault in `Aristo` backend when creating MPT descriptor
why:
Missing initialisation error
* CoreDb: Allow MPT to inherit shared context on `Aristo` backend
why:
Creates descriptors with different storage roots for the same
shared `Aristo` DB descriptor.
* Cosmetics, update diagnostic message items for `Aristo` backend
* Fix Copyright year
2024-01-11 19:11:38 +00:00
|
|
|
# Copyright (c) 2023-2024 Status Research & Development GmbH
|
2023-05-11 14:25:29 +00:00
|
|
|
# Licensed under either of
|
|
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
|
|
# http://opensource.org/licenses/MIT)
|
|
|
|
# at your option. This file may not be copied, modified, or distributed
|
|
|
|
# except according to those terms.
|
|
|
|
|
|
|
|
{.push raises: [].}
|
|
|
|
|
|
|
|
import
|
2023-05-30 21:21:15 +00:00
|
|
|
std/[algorithm, sequtils, sets, strutils, tables],
|
2024-06-22 20:33:37 +00:00
|
|
|
eth/common,
|
2023-08-21 14:58:30 +00:00
|
|
|
results,
|
2024-02-22 08:24:58 +00:00
|
|
|
stew/[byteutils, interval_set],
|
2023-09-05 13:57:20 +00:00
|
|
|
./aristo_desc/desc_backend,
|
2023-09-15 15:23:53 +00:00
|
|
|
./aristo_init/[memory_db, memory_only, rocks_db],
|
2024-09-19 08:39:06 +00:00
|
|
|
"."/[aristo_desc, aristo_get, aristo_layers,
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
aristo_serialise, aristo_utils]
|
2023-05-11 14:25:29 +00:00
|
|
|
|
|
|
|
# ------------------------------------------------------------------------------
|
2023-09-05 13:57:20 +00:00
|
|
|
# Private functions
|
2023-05-11 14:25:29 +00:00
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func orDefault(db: AristoDbRef): AristoDbRef =
|
Core db update storage root management for sub tries (#1964)
* Aristo: Re-phrase `LayerDelta` and `LayerFinal` as object references
why:
Avoids copying in some cases
* Fix copyright header
* Aristo: Verify `leafTie.root` function argument for `merge()` proc
why:
Zero root will lead to inconsistent DB entry
* Aristo: Update failure condition for hash labels compiler `hashify()`
why:
Node need not be rejected as long as links are on the schedule. In
that case, `redo[]` is to become `wff.base[]` at a later stage.
This amends an earlier fix, part of #1952 by also testing against
the target nodes of the `wff.base[]` sets.
* Aristo: Add storage root glue record to `hashify()` schedule
why:
An account leaf node might refer to a non-resolvable storage root ID.
Storage root node chains will end up at the storage root. So the link
`storage-root->account-leaf` needs an extra item in the schedule.
* Aristo: fix error code returned by `fetchPayload()`
details:
Final error code is implied by the error code form the `hikeUp()`
function.
* CoreDb: Discard `createOk` argument in API `getRoot()` function
why:
Not needed for the legacy DB. For the `Arsto` DB, a lazy approach is
implemented where a stprage root node is created on-the-fly.
* CoreDb: Prevent `$$` logging in some cases
why:
Logging the function `$$` is not useful when it is used for internal
use, i.e. retrieving an an error text for logging.
* CoreDb: Add `tryHashFn()` to API for pretty printing
why:
Pretty printing must not change the hashification status for the
`Aristo` DB. So there is an independent API wrapper for getting the
node hash which never updated the hashes.
* CoreDb: Discard `update` argument in API `hash()` function
why:
When calling the API function `hash()`, the latest state is always
wanted. For a version that uses the current state as-is without checking,
the function `tryHash()` was added to the backend.
* CoreDb: Update opaque vertex ID objects for the `Aristo` backend
why:
For `Aristo`, vID objects encapsulate a numeric `VertexID`
referencing a vertex (rather than a node hash as used on the
legacy backend.) For storage sub-tries, there might be no initial
vertex known when the descriptor is created. So opaque vertex ID
objects are supported without a valid `VertexID` which will be
initalised on-the-fly when the first item is merged.
* CoreDb: Add pretty printer for opaque vertex ID objects
* Cosmetics, printing profiling data
* CoreDb: Fix segfault in `Aristo` backend when creating MPT descriptor
why:
Missing initialisation error
* CoreDb: Allow MPT to inherit shared context on `Aristo` backend
why:
Creates descriptors with different storage roots for the same
shared `Aristo` DB descriptor.
* Cosmetics, update diagnostic message items for `Aristo` backend
* Fix Copyright year
2024-01-11 19:11:38 +00:00
|
|
|
if db.isNil: AristoDbRef(top: LayerRef.init()) else: db
|
2023-12-19 12:39:23 +00:00
|
|
|
|
|
|
|
# --------------------------
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func toHex(w: VertexID): string =
|
2023-11-08 12:18:32 +00:00
|
|
|
w.uint64.toHex
|
2023-10-27 21:36:51 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func toHexLsb(w: int8): string =
|
2023-10-27 21:36:51 +00:00
|
|
|
$"0123456789abcdef"[w and 15]
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func sortedKeys(tab: Table): seq =
|
2024-02-22 08:24:58 +00:00
|
|
|
tab.keys.toSeq.sorted
|
2023-06-09 11:17:37 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func sortedKeys(pPrf: HashSet): seq =
|
2024-02-22 08:24:58 +00:00
|
|
|
pPrf.toSeq.sorted
|
2023-12-19 12:39:23 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func toPfx(indent: int; offset = 0): string =
|
2023-06-20 13:26:25 +00:00
|
|
|
if 0 < indent+offset: "\n" & " ".repeat(indent+offset) else: ""
|
2023-06-09 11:17:37 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func squeeze(s: string; hex = false; ignLen = false): string =
|
2023-05-11 14:25:29 +00:00
|
|
|
## For long strings print `begin..end` only
|
|
|
|
if hex:
|
|
|
|
let n = (s.len + 1) div 2
|
2023-05-30 21:21:15 +00:00
|
|
|
result = if s.len < 20: s else: s[0 .. 5] & ".." & s[s.len-8 .. ^1]
|
2023-05-11 14:25:29 +00:00
|
|
|
if not ignLen:
|
|
|
|
result &= "[" & (if 0 < n: "#" & $n else: "") & "]"
|
|
|
|
elif s.len <= 30:
|
|
|
|
result = s
|
|
|
|
else:
|
|
|
|
result = if (s.len and 1) == 0: s[0 ..< 8] else: "0" & s[0 ..< 7]
|
|
|
|
if not ignLen:
|
|
|
|
result &= "..(" & $s.len & ")"
|
2023-05-30 21:21:15 +00:00
|
|
|
result &= ".." & s[s.len-16 .. ^1]
|
2023-05-11 14:25:29 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func stripZeros(a: string; toExp = false): string =
|
2023-12-04 20:39:26 +00:00
|
|
|
if 0 < a.len:
|
|
|
|
result = a.strip(leading=true, trailing=false, chars={'0'})
|
|
|
|
if result.len == 0:
|
|
|
|
result = "0"
|
|
|
|
elif result[^1] == '0' and toExp:
|
|
|
|
var n = 0
|
|
|
|
while result[^1] == '0':
|
|
|
|
let w = result.len
|
|
|
|
result.setLen(w-1)
|
|
|
|
n.inc
|
|
|
|
if n == 1:
|
|
|
|
result &= "0"
|
|
|
|
elif n == 2:
|
|
|
|
result &= "00"
|
|
|
|
elif 2 < n:
|
|
|
|
result &= "↑" & $n
|
|
|
|
|
|
|
|
# ---------------------
|
2023-05-11 14:25:29 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func ppKeyOk(
|
2023-12-19 12:39:23 +00:00
|
|
|
db: AristoDbRef;
|
|
|
|
key: HashKey;
|
2024-07-04 13:46:52 +00:00
|
|
|
rvid: RootedVertexID;
|
2023-12-19 12:39:23 +00:00
|
|
|
): string =
|
2024-07-04 13:46:52 +00:00
|
|
|
if key.isValid and rvid.isValid:
|
2024-07-29 20:15:17 +00:00
|
|
|
let rv = db.xMap.getOrVoid key
|
|
|
|
if rv.isValid:
|
|
|
|
if rvid != rv:
|
|
|
|
result = "(!)"
|
|
|
|
return
|
|
|
|
db.xMap[key] = rvid
|
2023-12-19 12:39:23 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func ppVid(vid: VertexID; pfx = true): string =
|
2023-06-20 13:26:25 +00:00
|
|
|
if pfx:
|
|
|
|
result = "$"
|
|
|
|
if vid.isValid:
|
2023-10-27 21:36:51 +00:00
|
|
|
result &= vid.toHex.stripZeros.toLowerAscii
|
2023-06-20 13:26:25 +00:00
|
|
|
else:
|
|
|
|
result &= "ø"
|
2023-05-11 14:25:29 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func ppVid(sid: StorageID; pfx = true): string =
|
2024-08-07 13:28:01 +00:00
|
|
|
if sid.isValid or not sid.vid.isValid:
|
|
|
|
sid.vid.ppVid(pfx)
|
|
|
|
else:
|
|
|
|
(if pfx: "$" else: "") & "®" & sid.vid.ppVid(false)
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func ppVid(rvid: RootedVertexID; pfx = true): string =
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
if pfx:
|
|
|
|
result = "$"
|
|
|
|
result &= ppVid(rvid.root, pfx=false) & ":" & ppVid(rvid.vid, pfx=false)
|
2024-07-04 13:46:52 +00:00
|
|
|
|
2024-10-01 21:03:10 +00:00
|
|
|
func ppCodeHash(h: Hash32): string =
|
2023-12-04 20:39:26 +00:00
|
|
|
result = "¢"
|
2024-10-01 21:03:10 +00:00
|
|
|
if h == default(Hash32):
|
2023-12-04 20:39:26 +00:00
|
|
|
result &= "©"
|
|
|
|
elif h == EMPTY_CODE_HASH:
|
|
|
|
result &= "ø"
|
|
|
|
else:
|
|
|
|
result &= h.data.toHex.squeeze(hex=true,ignLen=true)
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func ppVidList(vLst: openArray[VertexID]): string =
|
2024-05-30 17:48:38 +00:00
|
|
|
result = "["
|
2024-06-04 15:05:13 +00:00
|
|
|
if vLst.len <= 250:
|
|
|
|
result &= vLst.mapIt(it.ppVid).join(",")
|
2024-05-30 17:48:38 +00:00
|
|
|
else:
|
2024-06-04 15:05:13 +00:00
|
|
|
result &= vLst[0 .. 99].mapIt(it.ppVid).join(",")
|
2024-05-30 17:48:38 +00:00
|
|
|
result &= ",.."
|
2024-06-04 15:05:13 +00:00
|
|
|
result &= vLst[^100 .. ^1].mapIt(it.ppVid).join(",")
|
2024-05-30 17:48:38 +00:00
|
|
|
result &= "]"
|
2023-06-30 22:22:33 +00:00
|
|
|
|
2024-02-14 19:11:59 +00:00
|
|
|
proc ppKey(key: HashKey; db: AristoDbRef; pfx = true): string =
|
2023-11-08 12:18:32 +00:00
|
|
|
if pfx:
|
|
|
|
result = "£"
|
2024-10-01 21:03:10 +00:00
|
|
|
if key.to(Hash32) == default(Hash32):
|
2023-12-04 20:39:26 +00:00
|
|
|
result &= "©"
|
2023-11-08 12:18:32 +00:00
|
|
|
elif not key.isValid:
|
2023-12-04 20:39:26 +00:00
|
|
|
result &= "ø"
|
2023-11-08 12:18:32 +00:00
|
|
|
else:
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
# Reverse lookup
|
2024-07-29 20:15:17 +00:00
|
|
|
let rvid = db.xMap.getOrVoid key
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
if rvid.isValid:
|
|
|
|
result &= rvid.ppVid(pfx=false)
|
2024-07-29 20:15:17 +00:00
|
|
|
let vtx = db.getVtx rvid
|
|
|
|
if vtx.isValid:
|
|
|
|
let rc = vtx.toNode(rvid.root, db)
|
|
|
|
if rc.isErr or key != rc.value.digestTo(HashKey):
|
|
|
|
result &= "≠"
|
|
|
|
else:
|
|
|
|
result &= "∞"
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
else:
|
2024-07-29 20:15:17 +00:00
|
|
|
let tag = if key.len < 32: "[#" & $key.len & "]" else: ""
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
result &= @(key.data).toHex.squeeze(hex=true,ignLen=true) & tag
|
2023-05-11 14:25:29 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func ppLeafTie(lty: LeafTie, db: AristoDbRef): string =
|
2024-06-22 20:33:37 +00:00
|
|
|
let pfx = lty.path.to(NibblesBuf)
|
2023-12-04 20:39:26 +00:00
|
|
|
"@" & lty.root.ppVid(pfx=false) & ":" &
|
|
|
|
($pfx).squeeze(hex=true,ignLen=(pfx.len==64))
|
2023-05-30 11:47:47 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func ppPathPfx(pfx: NibblesBuf): string =
|
2023-05-30 21:21:15 +00:00
|
|
|
let s = $pfx
|
|
|
|
if s.len < 20: s else: s[0 .. 5] & ".." & s[s.len-8 .. ^1] & ":" & $s.len
|
2023-05-30 11:47:47 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func ppNibble(n: int8): string =
|
2023-10-27 21:36:51 +00:00
|
|
|
if n < 0: "ø" elif n < 10: $n else: n.toHexLsb
|
2023-05-30 11:47:47 +00:00
|
|
|
|
2024-07-29 20:15:17 +00:00
|
|
|
proc ppEthAccount(a: Account, db: AristoDbRef): string =
|
|
|
|
result = "("
|
|
|
|
result &= ($a.nonce).stripZeros(toExp=true) & ","
|
|
|
|
result &= ($a.balance).stripZeros(toExp=true) & ","
|
|
|
|
result &= a.codeHash.ppCodeHash & ","
|
|
|
|
result &= a.storageRoot.to(HashKey).ppKey(db) & ")"
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func ppAriAccount(a: AristoAccount): string =
|
2024-07-29 20:15:17 +00:00
|
|
|
result = "("
|
|
|
|
result &= ($a.nonce).stripZeros(toExp=true) & ","
|
|
|
|
result &= ($a.balance).stripZeros(toExp=true) & ","
|
|
|
|
result &= a.codeHash.ppCodeHash & ")"
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func ppPayload(p: LeafPayload, db: AristoDbRef): string =
|
2024-07-14 10:02:05 +00:00
|
|
|
case p.pType:
|
|
|
|
of AccountData:
|
2024-07-29 20:15:17 +00:00
|
|
|
result = "(" & p.account.ppAriAccount() & "," & p.stoID.ppVid & ")"
|
2024-07-14 10:02:05 +00:00
|
|
|
of StoData:
|
2024-08-14 08:54:44 +00:00
|
|
|
result = ($p.stoData).squeeze
|
2023-05-11 14:25:29 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func ppVtx(nd: VertexRef, db: AristoDbRef, rvid: RootedVertexID): string =
|
2023-06-12 13:48:47 +00:00
|
|
|
if not nd.isValid:
|
2023-06-30 22:22:33 +00:00
|
|
|
result = "ø"
|
2023-05-11 14:25:29 +00:00
|
|
|
else:
|
2024-07-04 13:46:52 +00:00
|
|
|
if not rvid.isValid:
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
result = ["L(", "B("][nd.vType.ord]
|
2024-07-04 13:46:52 +00:00
|
|
|
elif db.layersGetKey(rvid).isOk:
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
result = ["l(", "b("][nd.vType.ord]
|
2023-05-30 21:21:15 +00:00
|
|
|
else:
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
result = ["ł(", "þ("][nd.vType.ord]
|
2023-05-11 14:25:29 +00:00
|
|
|
case nd.vType:
|
|
|
|
of Leaf:
|
2024-09-13 16:55:17 +00:00
|
|
|
result &= nd.pfx.ppPathPfx & "," & nd.lData.ppPayload(db)
|
2023-05-11 14:25:29 +00:00
|
|
|
of Branch:
|
2024-09-13 16:55:17 +00:00
|
|
|
result &= nd.pfx.ppPathPfx & ":"
|
Pre-allocate vids for branches (#2882)
Each branch node may have up to 16 sub-items - currently, these are
given VertexID based when they are first needed leading to a
mostly-random order of vertexid for each subitem.
Here, we pre-allocate all 16 vertex ids such that when a branch subitem
is filled, it already has a vertexid waiting for it. This brings several
important benefits:
* subitems are sorted and "close" in their id sequencing - this means
that when rocksdb stores them, they are likely to end up in the same
data block thus improving read efficiency
* because the ids are consequtive, we can store just the starting id and
a bitmap representing which subitems are in use - this reduces disk
space usage for branches allowing more of them fit into a single disk
read, further improving disk read and caching performance - disk usage
at block 18M is down from 84 to 78gb!
* the in-memory footprint of VertexRef reduced allowing more instances
to fit into caches and less memory to be used overall.
Because of the increased locality of reference, it turns out that we no
longer need to iterate over the entire database to efficiently generate
the hash key database because the normal computation is now faster -
this significantly benefits "live" chain processing as well where each
dirtied key must be accompanied by a read of all branch subitems next to
it - most of the performance benefit in this branch comes from this
locality-of-reference improvement.
On a sample resync, there's already ~20% improvement with later blocks
seeing increasing benefit (because the trie is deeper in later blocks
leading to more benefit from branch read perf improvements)
```
blocks: 18729664, baseline: 190h43m49s, contender: 153h59m0s
Time (total): -36h44m48s, -19.27%
```
Note: clients need to be resynced as the PR changes the on-disk format
R.I.P. little bloom filter - your life in the repo was short but
valuable
2024-12-04 10:42:04 +00:00
|
|
|
for n in 0'u8..15'u8:
|
|
|
|
if nd.bVid(n).isValid:
|
|
|
|
result &= nd.bVid(n).ppVid
|
2023-05-30 21:21:15 +00:00
|
|
|
if n < 15:
|
|
|
|
result &= ","
|
2023-05-11 14:25:29 +00:00
|
|
|
result &= ")"
|
|
|
|
|
2024-07-29 20:15:17 +00:00
|
|
|
|
|
|
|
proc ppNode(
|
|
|
|
nd: NodeRef;
|
|
|
|
db: AristoDbRef;
|
|
|
|
rvid = default(RootedVertexID);
|
|
|
|
): string =
|
|
|
|
if not nd.isValid:
|
|
|
|
result = "ø"
|
|
|
|
else:
|
|
|
|
if not rvid.isValid:
|
2024-09-13 16:55:17 +00:00
|
|
|
result = ["L(", "B("][nd.vtx.vType.ord]
|
2024-07-29 20:15:17 +00:00
|
|
|
elif db.layersGetKey(rvid).isOk:
|
2024-09-13 16:55:17 +00:00
|
|
|
result = ["l(", "b("][nd.vtx.vType.ord]
|
2024-07-29 20:15:17 +00:00
|
|
|
else:
|
2024-09-13 16:55:17 +00:00
|
|
|
result = ["ł(", "þ("][nd.vtx.vType.ord]
|
|
|
|
case nd.vtx.vType:
|
2024-07-29 20:15:17 +00:00
|
|
|
of Leaf:
|
2024-09-13 16:55:17 +00:00
|
|
|
result &= nd.vtx.pfx.ppPathPfx & ","
|
|
|
|
if nd.vtx.lData.pType == AccountData:
|
|
|
|
result &= "(" & nd.vtx.lData.account.ppAriAccount() & ","
|
|
|
|
if nd.vtx.lData.stoID.isValid:
|
|
|
|
let tag = db.ppKeyOk(nd.key[0],(rvid.root,nd.vtx.lData.stoID.vid))
|
|
|
|
result &= nd.vtx.lData.stoID.ppVid & tag
|
2024-07-29 20:15:17 +00:00
|
|
|
else:
|
2024-09-13 16:55:17 +00:00
|
|
|
result &= nd.vtx.lData.stoID.ppVid
|
2024-07-29 20:15:17 +00:00
|
|
|
if nd.key[0].isValid:
|
|
|
|
result &= nd.key[0].ppKey(db)
|
|
|
|
result &= ")"
|
|
|
|
else:
|
2024-09-13 16:55:17 +00:00
|
|
|
result &= nd.vtx.lData.ppPayload(db)
|
2024-07-29 20:15:17 +00:00
|
|
|
of Branch:
|
Pre-allocate vids for branches (#2882)
Each branch node may have up to 16 sub-items - currently, these are
given VertexID based when they are first needed leading to a
mostly-random order of vertexid for each subitem.
Here, we pre-allocate all 16 vertex ids such that when a branch subitem
is filled, it already has a vertexid waiting for it. This brings several
important benefits:
* subitems are sorted and "close" in their id sequencing - this means
that when rocksdb stores them, they are likely to end up in the same
data block thus improving read efficiency
* because the ids are consequtive, we can store just the starting id and
a bitmap representing which subitems are in use - this reduces disk
space usage for branches allowing more of them fit into a single disk
read, further improving disk read and caching performance - disk usage
at block 18M is down from 84 to 78gb!
* the in-memory footprint of VertexRef reduced allowing more instances
to fit into caches and less memory to be used overall.
Because of the increased locality of reference, it turns out that we no
longer need to iterate over the entire database to efficiently generate
the hash key database because the normal computation is now faster -
this significantly benefits "live" chain processing as well where each
dirtied key must be accompanied by a read of all branch subitems next to
it - most of the performance benefit in this branch comes from this
locality-of-reference improvement.
On a sample resync, there's already ~20% improvement with later blocks
seeing increasing benefit (because the trie is deeper in later blocks
leading to more benefit from branch read perf improvements)
```
blocks: 18729664, baseline: 190h43m49s, contender: 153h59m0s
Time (total): -36h44m48s, -19.27%
```
Note: clients need to be resynced as the PR changes the on-disk format
R.I.P. little bloom filter - your life in the repo was short but
valuable
2024-12-04 10:42:04 +00:00
|
|
|
let keyOnly = nd.vtx.subVids.toSeq.filterIt(it.isValid).len == 0
|
2024-09-13 16:55:17 +00:00
|
|
|
result &= nd.vtx.pfx.ppPathPfx & ":"
|
Pre-allocate vids for branches (#2882)
Each branch node may have up to 16 sub-items - currently, these are
given VertexID based when they are first needed leading to a
mostly-random order of vertexid for each subitem.
Here, we pre-allocate all 16 vertex ids such that when a branch subitem
is filled, it already has a vertexid waiting for it. This brings several
important benefits:
* subitems are sorted and "close" in their id sequencing - this means
that when rocksdb stores them, they are likely to end up in the same
data block thus improving read efficiency
* because the ids are consequtive, we can store just the starting id and
a bitmap representing which subitems are in use - this reduces disk
space usage for branches allowing more of them fit into a single disk
read, further improving disk read and caching performance - disk usage
at block 18M is down from 84 to 78gb!
* the in-memory footprint of VertexRef reduced allowing more instances
to fit into caches and less memory to be used overall.
Because of the increased locality of reference, it turns out that we no
longer need to iterate over the entire database to efficiently generate
the hash key database because the normal computation is now faster -
this significantly benefits "live" chain processing as well where each
dirtied key must be accompanied by a read of all branch subitems next to
it - most of the performance benefit in this branch comes from this
locality-of-reference improvement.
On a sample resync, there's already ~20% improvement with later blocks
seeing increasing benefit (because the trie is deeper in later blocks
leading to more benefit from branch read perf improvements)
```
blocks: 18729664, baseline: 190h43m49s, contender: 153h59m0s
Time (total): -36h44m48s, -19.27%
```
Note: clients need to be resynced as the PR changes the on-disk format
R.I.P. little bloom filter - your life in the repo was short but
valuable
2024-12-04 10:42:04 +00:00
|
|
|
for n in 0'u8..15'u8:
|
|
|
|
if nd.vtx.bVid(n).isValid:
|
|
|
|
let tag = db.ppKeyOk(nd.key[n],(rvid.root,nd.vtx.bVid(n)))
|
|
|
|
result &= nd.vtx.bVid(n).ppVid & tag
|
2024-07-29 20:15:17 +00:00
|
|
|
elif keyOnly and nd.key[n].isValid:
|
|
|
|
result &= nd.key[n].ppKey(db)
|
|
|
|
if n < 15:
|
|
|
|
result &= ","
|
|
|
|
result &= ")"
|
|
|
|
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func ppXTab[T: VertexRef|NodeRef](
|
2024-07-29 20:15:17 +00:00
|
|
|
tab: Table[RootedVertexID,T];
|
2023-12-19 12:39:23 +00:00
|
|
|
db: AristoDbRef;
|
2023-06-30 22:22:33 +00:00
|
|
|
indent = 4;
|
|
|
|
): string =
|
2024-07-29 20:15:17 +00:00
|
|
|
proc ppT(v: T; r: RootedVertexID): string =
|
|
|
|
when T is VertexRef:
|
|
|
|
v.ppVtx(db, r)
|
|
|
|
elif T is NodeRef:
|
|
|
|
v.ppNode(db, r)
|
|
|
|
"{" & tab.sortedKeys
|
|
|
|
.mapIt((it, tab.getOrDefault it))
|
|
|
|
.mapIt("(" & it[0].ppVid & "," & it[1].ppT(it[0]) & ")")
|
|
|
|
.join(indent.toPfx(1)) & "}"
|
|
|
|
|
2023-06-30 22:22:33 +00:00
|
|
|
|
2023-05-30 21:21:15 +00:00
|
|
|
proc ppXMap*(
|
2023-07-04 18:24:03 +00:00
|
|
|
db: AristoDbRef;
|
2024-07-04 13:46:52 +00:00
|
|
|
kMap: Table[RootedVertexID,HashKey];
|
2023-05-30 21:21:15 +00:00
|
|
|
indent: int;
|
|
|
|
): string =
|
2023-11-08 12:18:32 +00:00
|
|
|
let pfx = indent.toPfx(1)
|
|
|
|
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
# Sort keys by root,
|
|
|
|
# entry int: 0=no-key 1=no-vertex 2=cant-compile 3=key-mistmatch 4=key-ok
|
|
|
|
var keyLst: seq[(VertexID,seq[(VertexID,HashKey,int)])]
|
2023-12-19 12:39:23 +00:00
|
|
|
block:
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
var root = VertexID(0)
|
|
|
|
for w in kMap.sortedKeys:
|
|
|
|
if w.root != root:
|
|
|
|
keyLst.add (w.root,newSeq[typeof keyLst[0][1][0]](0))
|
|
|
|
root = w.root
|
|
|
|
let
|
|
|
|
key = kMap.getOrVoid w
|
|
|
|
mode = block:
|
|
|
|
if key == VOID_HASH_KEY:
|
|
|
|
0
|
|
|
|
else:
|
2024-07-29 20:15:17 +00:00
|
|
|
db.xMap[key] = w
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
let vtx = db.getVtx(w)
|
|
|
|
if not vtx.isValid:
|
|
|
|
1
|
|
|
|
else:
|
|
|
|
let rc = vtx.toNode(w.root, db)
|
|
|
|
if rc.isErr:
|
|
|
|
2
|
2024-07-17 13:48:21 +00:00
|
|
|
elif key != rc.value.digestTo(HashKey):
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
3
|
|
|
|
else:
|
|
|
|
4
|
|
|
|
keyLst[^1][1].add (w.vid,key,mode)
|
|
|
|
|
2024-07-29 20:15:17 +00:00
|
|
|
# Join increasing sequences for pretty printing
|
|
|
|
var keySubLst: seq[(VertexID,seq[seq[(VertexID,HashKey,int)]])]
|
|
|
|
for (root,rootQ) in keyLst:
|
|
|
|
var
|
|
|
|
q: seq[(VertexID,HashKey,int)]
|
|
|
|
subQ: seq[typeof q]
|
|
|
|
for (vid,key,state) in rootQ:
|
|
|
|
if q.len == 0:
|
|
|
|
q.add (vid,key,state)
|
|
|
|
continue
|
|
|
|
if q[^1][0]+1 == vid and q[^1][2] == state:
|
|
|
|
q.add (vid,key,state)
|
|
|
|
continue
|
|
|
|
# Otherwise new sub queue
|
|
|
|
subQ.add q
|
|
|
|
q = @[(vid,key,state)]
|
|
|
|
if 0 < q.len:
|
|
|
|
subQ.add q
|
|
|
|
keySubLst.add (root,subQ)
|
|
|
|
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
proc pp(w: (VertexID,HashKey,int)): string =
|
|
|
|
proc pp(k: HashKey): string =
|
|
|
|
result = w[1].data.toHex.squeeze(hex=true,ignLen=true)
|
|
|
|
if k.len < 32:
|
|
|
|
result &= "[#" & $k.len & "]"
|
|
|
|
w[0].ppVid(pfx=false) & (
|
|
|
|
case w[2]:
|
|
|
|
of 0: "=ø"
|
2024-07-29 20:15:17 +00:00
|
|
|
of 1: "∞"
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
of 2: "=" & w[1].pp()
|
|
|
|
of 3: "≠" & w[1].pp()
|
|
|
|
else: "")
|
|
|
|
|
2024-07-29 20:15:17 +00:00
|
|
|
result &= "{"
|
|
|
|
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
var qfx = ""
|
2024-07-29 20:15:17 +00:00
|
|
|
for (root,subQ) in keySubLst:
|
|
|
|
result &= qfx & "£" & root.ppVid(pfx=false) & ":"
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
qfx = pfx
|
2024-07-29 20:15:17 +00:00
|
|
|
var closeBracket = ""
|
|
|
|
if 1 < subQ.len or 1 < subQ[0].len:
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
result &= "["
|
2024-07-29 20:15:17 +00:00
|
|
|
closeBracket = "]"
|
|
|
|
for q in subQ:
|
|
|
|
if q.len < 3:
|
|
|
|
result &= q.mapIt(it.pp).join(",")
|
|
|
|
else:
|
|
|
|
result &= q[0].pp & ".." & q[^1].pp
|
|
|
|
result &= ","
|
|
|
|
result.setLen(result.len - 1)
|
|
|
|
result &= closeBracket
|
|
|
|
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
result &= "}"
|
2023-05-30 21:21:15 +00:00
|
|
|
|
2024-07-29 20:15:17 +00:00
|
|
|
|
2024-07-18 21:32:32 +00:00
|
|
|
proc ppBalancer(
|
|
|
|
fl: LayerRef;
|
2023-11-08 12:18:32 +00:00
|
|
|
db: AristoDbRef;
|
|
|
|
indent: int;
|
|
|
|
): string =
|
2023-08-10 20:01:28 +00:00
|
|
|
## Walk over filter tables
|
|
|
|
let
|
|
|
|
pfx = indent.toPfx
|
|
|
|
pfx1 = indent.toPfx(1)
|
|
|
|
pfx2 = indent.toPfx(2)
|
2024-07-17 19:27:33 +00:00
|
|
|
result = "<balancer>"
|
2023-08-17 13:42:01 +00:00
|
|
|
if fl.isNil:
|
2023-08-10 20:01:28 +00:00
|
|
|
result &= " n/a"
|
|
|
|
return
|
2024-06-04 15:05:13 +00:00
|
|
|
result &= pfx & "vTop=" & fl.vTop.ppVid
|
2023-08-18 19:46:55 +00:00
|
|
|
result &= pfx & "sTab" & pfx1 & "{"
|
2023-08-10 20:01:28 +00:00
|
|
|
for n,vid in fl.sTab.sortedKeys:
|
|
|
|
let vtx = fl.sTab.getOrVoid vid
|
|
|
|
if 0 < n: result &= pfx2
|
|
|
|
result &= $(1+n) & "(" & vid.ppVid & "," & vtx.ppVtx(db,vid) & ")"
|
|
|
|
result &= "}" & pfx & "kMap" & pfx1 & "{"
|
|
|
|
for n,vid in fl.kMap.sortedKeys:
|
|
|
|
let key = fl.kMap.getOrVoid vid
|
|
|
|
if 0 < n: result &= pfx2
|
2024-02-14 19:11:59 +00:00
|
|
|
result &= $(1+n) & "(" & vid.ppVid & "," & key.ppKey(db) & ")"
|
2023-08-10 20:01:28 +00:00
|
|
|
result &= "}"
|
|
|
|
|
2024-05-30 17:48:38 +00:00
|
|
|
proc ppBe[T](be: T; db: AristoDbRef; limit: int; indent: int): string =
|
2023-06-20 13:26:25 +00:00
|
|
|
## Walk over backend tables
|
2023-06-22 11:13:24 +00:00
|
|
|
let
|
|
|
|
pfx = indent.toPfx
|
|
|
|
pfx1 = indent.toPfx(1)
|
|
|
|
pfx2 = indent.toPfx(2)
|
2023-06-20 13:26:25 +00:00
|
|
|
result = "<" & $be.kind & ">"
|
2024-04-03 15:48:35 +00:00
|
|
|
var (dump,dataOk) = ("",false)
|
|
|
|
block:
|
2024-06-04 15:05:13 +00:00
|
|
|
let rc = be.getTuvFn()
|
|
|
|
if rc.isOk:
|
|
|
|
dump &= pfx & "vTop=" & rc.value.ppVid
|
2024-04-03 15:48:35 +00:00
|
|
|
dataOk = true
|
2023-12-20 16:19:00 +00:00
|
|
|
block:
|
2024-04-03 15:48:35 +00:00
|
|
|
dump &= pfx & "sTab"
|
|
|
|
var (n, data) = (0, "")
|
2023-12-20 16:19:00 +00:00
|
|
|
for (vid,vtx) in be.walkVtx:
|
|
|
|
n.inc
|
2024-05-30 17:48:38 +00:00
|
|
|
if n < limit:
|
|
|
|
if 1 < n: data &= pfx2
|
|
|
|
data &= $n & "(" & vid.ppVid & "," & vtx.ppVtx(db,vid) & ")"
|
|
|
|
elif n == limit:
|
|
|
|
data &= pfx2 & ".."
|
2024-04-03 15:48:35 +00:00
|
|
|
dump &= "(" & $n & ")"
|
|
|
|
if 0 < n:
|
|
|
|
dataOk = true
|
|
|
|
dump &= pfx1
|
|
|
|
dump &= "{" & data & "}"
|
2023-12-20 16:19:00 +00:00
|
|
|
block:
|
2024-04-03 15:48:35 +00:00
|
|
|
dump &= pfx & "kMap"
|
|
|
|
var (n, data) = (0, "")
|
2023-12-20 16:19:00 +00:00
|
|
|
for (vid,key) in be.walkKey:
|
|
|
|
n.inc
|
2024-05-30 17:48:38 +00:00
|
|
|
if n < limit:
|
|
|
|
if 1 < n: data &= pfx2
|
|
|
|
data &= $n & "(" & vid.ppVid & "," & key.ppKey(db) & ")"
|
|
|
|
elif n == limit:
|
|
|
|
data &= pfx2 & ".."
|
2024-04-03 15:48:35 +00:00
|
|
|
dump &= "(" & $n & ")"
|
|
|
|
if 0 < n:
|
|
|
|
dataOk = true
|
|
|
|
dump &= pfx1
|
|
|
|
dump &= "{" & data & "}"
|
|
|
|
if dataOk:
|
|
|
|
result &= dump
|
|
|
|
else:
|
|
|
|
result &= "[]"
|
2023-06-20 13:26:25 +00:00
|
|
|
|
2023-08-10 20:01:28 +00:00
|
|
|
proc ppLayer(
|
2023-08-18 19:46:55 +00:00
|
|
|
layer: LayerRef;
|
2023-07-04 18:24:03 +00:00
|
|
|
db: AristoDbRef;
|
2024-06-04 15:05:13 +00:00
|
|
|
vTopOk: bool;
|
2023-06-30 22:22:33 +00:00
|
|
|
sTabOk: bool;
|
|
|
|
kMapOk: bool;
|
|
|
|
indent = 4;
|
|
|
|
): string =
|
|
|
|
let
|
2023-08-17 13:42:01 +00:00
|
|
|
pfx1 = indent.toPfx(1)
|
|
|
|
pfx2 = indent.toPfx(2)
|
2024-07-03 20:21:57 +00:00
|
|
|
nOKs = vTopOk.ord + sTabOk.ord + kMapOk.ord
|
2023-08-11 17:23:57 +00:00
|
|
|
tagOk = 1 < nOKs
|
2023-06-30 22:22:33 +00:00
|
|
|
var
|
|
|
|
pfy = ""
|
|
|
|
|
|
|
|
proc doPrefix(s: string; dataOk: bool): string =
|
|
|
|
var rc: string
|
|
|
|
if tagOk:
|
2024-06-04 15:05:13 +00:00
|
|
|
rc = pfy
|
|
|
|
if 0 < s.len:
|
|
|
|
rc &= s & (if dataOk: pfx2 else: "")
|
2023-06-30 22:22:33 +00:00
|
|
|
pfy = pfx1
|
|
|
|
else:
|
|
|
|
rc = pfy
|
|
|
|
pfy = pfx2
|
|
|
|
rc
|
|
|
|
|
2023-08-10 20:01:28 +00:00
|
|
|
if not layer.isNil:
|
2023-08-17 13:42:01 +00:00
|
|
|
if 2 < nOKs:
|
|
|
|
result &= "<layer>".doPrefix(false)
|
2024-06-04 15:05:13 +00:00
|
|
|
if vTopOk:
|
2024-07-18 21:32:32 +00:00
|
|
|
result &= "".doPrefix(true) & "vTop=" & layer.vTop.ppVid
|
2023-06-30 22:22:33 +00:00
|
|
|
if sTabOk:
|
|
|
|
let
|
2024-07-18 21:32:32 +00:00
|
|
|
tLen = layer.sTab.len
|
2023-06-30 22:22:33 +00:00
|
|
|
info = "sTab(" & $tLen & ")"
|
2024-07-29 20:15:17 +00:00
|
|
|
result &= info.doPrefix(0 < tLen) & layer.sTab.ppXTab(db,indent+2)
|
2023-06-30 22:22:33 +00:00
|
|
|
if kMapOk:
|
|
|
|
let
|
2024-07-18 21:32:32 +00:00
|
|
|
tLen = layer.kMap.len
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
info = "kMap(" & $tLen & ")"
|
|
|
|
result &= info.doPrefix(0 < tLen)
|
2024-07-18 21:32:32 +00:00
|
|
|
result &= db.ppXMap(layer.kMap, indent+2)
|
2023-06-30 22:22:33 +00:00
|
|
|
|
2023-05-30 21:21:15 +00:00
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
# Public functions
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*(w: Hash32; codeHashOk: bool): string =
|
2024-04-03 15:48:35 +00:00
|
|
|
if codeHashOk:
|
|
|
|
w.ppCodeHash
|
|
|
|
elif w == EMPTY_ROOT_HASH:
|
2023-11-20 20:22:27 +00:00
|
|
|
"EMPTY_ROOT_HASH"
|
2024-10-01 21:03:10 +00:00
|
|
|
elif w == default(Hash32):
|
|
|
|
"default(Hash32)"
|
2023-11-20 20:22:27 +00:00
|
|
|
else:
|
|
|
|
w.data.toHex.squeeze(hex=true,ignLen=true)
|
2023-11-08 12:18:32 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*(n: NibblesBuf): string =
|
2024-07-29 20:15:17 +00:00
|
|
|
n.ppPathPfx()
|
|
|
|
|
2024-02-14 19:11:59 +00:00
|
|
|
proc pp*(w: HashKey; db = AristoDbRef(nil)): string =
|
|
|
|
w.ppKey(db.orDefault)
|
2023-05-30 21:21:15 +00:00
|
|
|
|
2024-10-01 21:03:10 +00:00
|
|
|
proc pp*(w: Hash32; db = AristoDbRef(nil)): string =
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
w.to(HashKey).ppKey(db.orDefault)
|
|
|
|
|
2024-04-03 15:48:35 +00:00
|
|
|
proc pp*(w: openArray[HashKey]; db = AristoDbRef(nil)): string =
|
|
|
|
"[" & @w.mapIt(it.ppKey(db.orDefault)).join(",") & "]"
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*(lty: LeafTie, db = AristoDbRef(nil)): string =
|
2023-12-19 12:39:23 +00:00
|
|
|
lty.ppLeafTie(db.orDefault)
|
2023-06-09 11:17:37 +00:00
|
|
|
|
2024-07-29 20:15:17 +00:00
|
|
|
proc pp*(a: Account, db = AristoDbRef(nil)): string =
|
|
|
|
a.ppEthAccount(db.orDefault)
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*(vid: VertexID): string =
|
2023-05-30 21:21:15 +00:00
|
|
|
vid.ppVid
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*(rvid: RootedVertexID): string =
|
No ext update (#2494)
* Imported/rebase from `no-ext`, PR #2485
Store extension nodes together with the branch
Extension nodes must be followed by a branch - as such, it makes sense
to store the two together both in the database and in memory:
* fewer reads, writes and updates to traverse the tree
* simpler logic for maintaining the node structure
* less space used, both memory and storage, because there are fewer
nodes overall
There is also a downside: hashes can no longer be cached for an
extension - instead, only the extension+branch hash can be cached - this
seems like a fine tradeoff since computing it should be fast.
TODO: fix commented code
* Fix merge functions and `toNode()`
* Update `merkleSignCommit()` prototype
why:
Result is always a 32bit hash
* Update short Merkle hash key generation
details:
Ethereum reference MPTs use Keccak hashes as node links if the size of
an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded
node value is used as a pseudo node link (rather than a hash.) This is
specified in the yellow paper, appendix D.
Different to the `Aristo` implementation, the reference MPT would not
store such a node on the key-value database. Rather the RLP encoded node value is stored instead of a node link in a parent node
is stored as a node link on the parent database.
Only for the root hash, the top level node is always referred to by the
hash.
* Fix/update `Extension` sections
why:
Were commented out after removal of a dedicated `Extension` type which
left the system disfunctional.
* Clean up unused error codes
* Update unit tests
* Update docu
---------
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-07-16 19:47:59 +00:00
|
|
|
rvid.ppVid
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*(vLst: openArray[VertexID]): string =
|
2024-06-04 15:05:13 +00:00
|
|
|
vLst.ppVidList
|
2023-05-30 21:21:15 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*(p: LeafPayload, db = AristoDbRef(nil)): string =
|
2023-12-19 12:39:23 +00:00
|
|
|
p.ppPayload(db.orDefault)
|
2023-05-30 21:21:15 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*(nd: VertexRef, db = AristoDbRef(nil)): string =
|
2024-07-04 13:46:52 +00:00
|
|
|
nd.ppVtx(db.orDefault, default(RootedVertexID))
|
|
|
|
|
2024-07-29 20:15:17 +00:00
|
|
|
proc pp*(nd: NodeRef, db = AristoDbRef(nil)): string =
|
|
|
|
nd.ppNode(db.orDefault, default(RootedVertexID))
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*(e: (VertexID,AristoError)): string =
|
2024-07-29 20:15:17 +00:00
|
|
|
"(" & e[0].pp & "," & $e[1] & ")"
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*[T](rc: Result[T,(VertexID,AristoError)]): string =
|
2024-02-08 16:32:16 +00:00
|
|
|
if rc.isOk:
|
|
|
|
result = "ok("
|
|
|
|
when T isnot void:
|
|
|
|
result &= ".."
|
|
|
|
result &= ")"
|
|
|
|
else:
|
2024-07-29 20:15:17 +00:00
|
|
|
result = "err(" & rc.error.pp & ")"
|
2024-02-08 16:32:16 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*(
|
2024-07-04 13:46:52 +00:00
|
|
|
sTab: Table[RootedVertexID,VertexRef];
|
2023-12-19 12:39:23 +00:00
|
|
|
db = AristoDbRef(nil);
|
2023-07-04 18:24:03 +00:00
|
|
|
indent = 4;
|
|
|
|
): string =
|
2024-07-29 20:15:17 +00:00
|
|
|
sTab.ppXTab(db.orDefault)
|
2023-06-09 11:17:37 +00:00
|
|
|
|
2024-08-01 10:41:20 +00:00
|
|
|
proc pp*(leg: Leg; root: VertexID; db = AristoDbRef(nil)): string =
|
2023-12-19 12:39:23 +00:00
|
|
|
let db = db.orDefault()
|
2023-06-09 11:17:37 +00:00
|
|
|
result = "(" & leg.wp.vid.ppVid & ","
|
2023-12-19 12:39:23 +00:00
|
|
|
block:
|
2024-07-04 13:46:52 +00:00
|
|
|
let key = db.layersGetKeyOrVoid (root, leg.wp.vid)
|
2024-02-14 19:11:59 +00:00
|
|
|
if not key.isValid:
|
2023-06-20 13:26:25 +00:00
|
|
|
result &= "ø"
|
2024-07-29 20:15:17 +00:00
|
|
|
elif (root, leg.wp.vid) != db.xMap.getOrVoid key:
|
2024-02-14 19:11:59 +00:00
|
|
|
result &= key.ppKey(db)
|
2023-06-20 13:26:25 +00:00
|
|
|
result &= ","
|
|
|
|
if 0 <= leg.nibble:
|
|
|
|
result &= $leg.nibble.ppNibble
|
|
|
|
result &= "," & leg.wp.vtx.pp(db) & ")"
|
2023-05-30 21:21:15 +00:00
|
|
|
|
2023-12-19 12:39:23 +00:00
|
|
|
proc pp*(hike: Hike; db = AristoDbRef(nil); indent = 4): string =
|
|
|
|
let
|
|
|
|
db = db.orDefault()
|
|
|
|
pfx = indent.toPfx(1)
|
2023-06-09 11:17:37 +00:00
|
|
|
result = "["
|
|
|
|
if hike.legs.len == 0:
|
|
|
|
result &= "(" & hike.root.ppVid & ")"
|
|
|
|
else:
|
|
|
|
if hike.legs[0].wp.vid != hike.root:
|
|
|
|
result &= "(" & hike.root.ppVid & ")" & pfx
|
2024-08-01 10:41:20 +00:00
|
|
|
result &= hike.legs.mapIt(it.pp(hike.root, db)).join(pfx)
|
2023-06-09 11:17:37 +00:00
|
|
|
result &= pfx & "(" & hike.tail.ppPathPfx & ")"
|
2023-05-30 11:47:47 +00:00
|
|
|
result &= "]"
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*[T: NodeRef|VertexRef|HashKey](
|
2024-07-29 20:15:17 +00:00
|
|
|
q: seq[(HashKey,T)];
|
|
|
|
db = AristoDbRef(nil);
|
|
|
|
indent = 4;
|
|
|
|
): string =
|
|
|
|
let db = db.orDefault
|
|
|
|
proc ppT(v: T): string =
|
|
|
|
when T is VertexID or T is RootedVertexID:
|
|
|
|
v.pp()
|
|
|
|
else:
|
|
|
|
v.pp(db)
|
|
|
|
"{" & q.mapIt("(" & it[0].ppKey(db) & "," & it[1].ppT & ")")
|
|
|
|
.join("," & indent.toPfx(1)) & "}"
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*[T: NodeRef|VertexRef|HashKey](
|
2024-07-29 20:15:17 +00:00
|
|
|
t: Table[HashKey,T];
|
|
|
|
db = AristoDbRef(nil);
|
|
|
|
indent = 4;
|
|
|
|
): string =
|
|
|
|
## Sort hash keys by associated vertex ID were possible
|
|
|
|
let db = db.orDefault
|
|
|
|
var
|
|
|
|
t0: Table[RootedVertexID,(HashKey,T)]
|
|
|
|
t1: Table[HashKey,T]
|
|
|
|
for (key,val) in t.pairs:
|
|
|
|
db.xMap.withValue(key,rv):
|
|
|
|
t0[rv[]] = (key,val)
|
|
|
|
do:
|
|
|
|
t1[key] = val
|
|
|
|
let
|
|
|
|
q0 = t0.sortedKeys.mapIt(t0.getOrDefault it)
|
|
|
|
q1 = t1.sortedKeys.mapIt((it, t1.getOrDefault it))
|
|
|
|
(q0 & q1).pp(db,indent)
|
|
|
|
|
|
|
|
proc pp*[T: HashKey](
|
|
|
|
t: Table[T,RootedVertexID];
|
|
|
|
db = AristoDbRef(nil);
|
|
|
|
indent = 4;
|
|
|
|
): string =
|
|
|
|
## Sort by second tab item vertex ID
|
|
|
|
let db = db.orDefault
|
|
|
|
proc ppT(v: T): string =
|
|
|
|
when T is VertexID or T is RootedVertexID:
|
|
|
|
v.pp()
|
|
|
|
else:
|
|
|
|
v.pp(db)
|
|
|
|
var rev: Table[RootedVertexID,seq[T]]
|
|
|
|
for (key,rvid) in t.pairs:
|
|
|
|
rev.withValue(rvid,val):
|
|
|
|
val[].add key
|
|
|
|
do:
|
|
|
|
rev[rvid] = @[key]
|
|
|
|
var flat: seq[(HashKey,RootedVertexID)]
|
|
|
|
for rvid in rev.keys.toSeq.sorted:
|
|
|
|
rev.withValue(rvid,keysPtr):
|
|
|
|
for key in keysPtr[]:
|
|
|
|
flat.add (key,rvid)
|
|
|
|
# Now sorted vy values
|
|
|
|
"{" & flat.mapIt("(" & it[0].ppT & "," & it[1].pp & ")")
|
2023-06-09 11:17:37 +00:00
|
|
|
.join("," & indent.toPfx(1)) & "}"
|
2023-05-30 11:47:47 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*[T: HashKey](
|
2024-07-29 20:15:17 +00:00
|
|
|
t: TableRef[HashKey,T];
|
|
|
|
db = AristoDbRef(nil);
|
|
|
|
indent = 4;
|
|
|
|
): string =
|
|
|
|
pp(t[],db,indent)
|
|
|
|
|
|
|
|
proc pp*(
|
|
|
|
kMap: Table[RootedVertexID,HashKey];
|
|
|
|
db: AristoDbRef;
|
|
|
|
indent = 4;
|
|
|
|
): string =
|
2024-02-22 08:24:58 +00:00
|
|
|
db.ppXMap(kMap, indent)
|
2023-12-19 12:39:23 +00:00
|
|
|
|
|
|
|
# ---------------------
|
2023-05-30 11:47:47 +00:00
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*(tx: AristoTxRef): string =
|
2023-12-19 12:39:23 +00:00
|
|
|
result = "(uid=" & $tx.txUid & ",level=" & $tx.level
|
Core db and aristo updates for destructor and tx logic (#1894)
* Disable `TransactionID` related functions from `state_db.nim`
why:
Functions `getCommittedStorage()` and `updateOriginalRoot()` from
the `state_db` module are nowhere used. The emulation of a legacy
`TransactionID` type functionality is administratively expensive to
provide by `Aristo` (the legacy DB version is only partially
implemented, anyway).
As there is no other place where `TransactionID`s are used, they will
not be provided by the `Aristo` variant of the `CoreDb`. For the
legacy DB API, nothing will change.
* Fix copyright headers in source code
* Get rid of compiler warning
* Update Aristo code, remove unused `merge()` variant, export `hashify()`
why:
Adapt to upcoming `CoreDb` wrapper
* Remove synced tx feature from `Aristo`
why:
+ This feature allowed to synchronise transaction methods like begin,
commit, and rollback for a group of descriptors.
+ The feature is over engineered and not needed for `CoreDb`, neither
is it complete (some convergence features missing.)
* Add debugging helpers to `Kvt`
also:
Update database iterator, add count variable yield argument similar
to `Aristo`.
* Provide optional destructors for `CoreDb` API
why;
For the upcoming Aristo wrapper, this allows to control when certain
smart destruction and update can take place. The auto destructor works
fine in general when the storage/cache strategy is known and acceptable
when creating descriptors.
* Add update option for `CoreDb` API function `hash()`
why;
The hash function is typically used to get the state root of the MPT.
Due to lazy hashing, this might be not available on the `Aristo` DB.
So the `update` function asks for re-hashing the gurrent state changes
if needed.
* Update API tracking log mode: `info` => `debug
* Use shared `Kvt` descriptor in new Ledger API
why:
No need to create a new descriptor all the time
2023-11-16 19:35:03 +00:00
|
|
|
if not tx.parent.isNil:
|
|
|
|
result &= ", par=" & $tx.parent.txUid
|
|
|
|
result &= ")"
|
|
|
|
|
2024-10-26 05:10:06 +00:00
|
|
|
func pp*(wp: VidVtxPair; db: AristoDbRef): string =
|
2023-12-04 20:39:26 +00:00
|
|
|
"(" & wp.vid.pp & "," & wp.vtx.pp(db) & ")"
|
|
|
|
|
2023-05-30 21:21:15 +00:00
|
|
|
|
2023-08-10 20:01:28 +00:00
|
|
|
proc pp*(
|
2023-08-18 19:46:55 +00:00
|
|
|
layer: LayerRef;
|
2023-08-10 20:01:28 +00:00
|
|
|
db: AristoDbRef;
|
|
|
|
indent = 4;
|
2024-07-18 21:32:32 +00:00
|
|
|
sTabOk = true,
|
|
|
|
kMapOk = true,
|
2024-07-29 20:15:17 +00:00
|
|
|
vTopOk = true,
|
2023-08-10 20:01:28 +00:00
|
|
|
): string =
|
2024-07-29 20:15:17 +00:00
|
|
|
layer.ppLayer(
|
|
|
|
db.orDefault(), vTopOk=vTopOk, sTabOk=sTabOk, kMapOk=kMapOk, indent=indent)
|
2023-05-30 21:21:15 +00:00
|
|
|
|
2023-06-20 13:26:25 +00:00
|
|
|
proc pp*(
|
2023-09-05 13:57:20 +00:00
|
|
|
be: BackendRef;
|
2023-07-04 18:24:03 +00:00
|
|
|
db: AristoDbRef;
|
2024-05-30 17:48:38 +00:00
|
|
|
limit = 100;
|
2023-06-20 13:26:25 +00:00
|
|
|
indent = 4;
|
|
|
|
): string =
|
2024-07-18 21:32:32 +00:00
|
|
|
result = db.balancer.ppBalancer(db, indent+1) & indent.toPfx
|
2023-09-05 13:57:20 +00:00
|
|
|
case be.kind:
|
2023-06-20 13:26:25 +00:00
|
|
|
of BackendMemory:
|
2024-05-30 17:48:38 +00:00
|
|
|
result &= be.MemBackendRef.ppBe(db, limit, indent+1)
|
2024-06-13 18:15:11 +00:00
|
|
|
of BackendRocksDB, BackendRdbHosting:
|
2024-05-30 17:48:38 +00:00
|
|
|
result &= be.RdbBackendRef.ppBe(db, limit, indent+1)
|
2023-08-11 17:23:57 +00:00
|
|
|
of BackendVoid:
|
2023-09-15 15:23:53 +00:00
|
|
|
result &= "<NoBackend>"
|
2023-06-20 13:26:25 +00:00
|
|
|
|
2023-09-11 20:38:49 +00:00
|
|
|
proc pp*(
|
|
|
|
db: AristoDbRef;
|
|
|
|
indent = 4;
|
2023-12-04 20:39:26 +00:00
|
|
|
backendOk = false;
|
2024-06-03 20:10:35 +00:00
|
|
|
balancerOk = true;
|
2024-03-22 17:31:56 +00:00
|
|
|
topOk = true;
|
|
|
|
stackOk = true;
|
2024-04-03 15:48:35 +00:00
|
|
|
kMapOk = true;
|
2024-07-18 21:32:32 +00:00
|
|
|
sTabOk = true;
|
2024-05-30 17:48:38 +00:00
|
|
|
limit = 100;
|
2023-09-11 20:38:49 +00:00
|
|
|
): string =
|
2024-03-22 17:31:56 +00:00
|
|
|
if topOk:
|
2024-07-29 20:15:17 +00:00
|
|
|
result = db.layersCc.ppLayer(
|
|
|
|
db, sTabOk=sTabOk, kMapOk=kMapOk, vTopOk=true, indent=indent)
|
2024-06-03 20:10:35 +00:00
|
|
|
let stackOnlyOk = stackOk and not (topOk or balancerOk or backendOk)
|
2024-03-22 17:31:56 +00:00
|
|
|
if not stackOnlyOk:
|
2024-07-29 20:15:17 +00:00
|
|
|
result &= indent.toPfx(1) & "level=" & $db.stack.len
|
2024-03-22 17:31:56 +00:00
|
|
|
if (stackOk and 0 < db.stack.len) or stackOnlyOk:
|
|
|
|
let layers = @[db.top] & db.stack.reversed
|
|
|
|
var lStr = ""
|
|
|
|
for n,w in layers:
|
|
|
|
let
|
|
|
|
m = layers.len - n - 1
|
|
|
|
l = db.layersCc m
|
2024-07-18 21:32:32 +00:00
|
|
|
a = w.kMap.values.toSeq.filterIt(not it.isValid).len
|
|
|
|
c = l.kMap.values.toSeq.filterIt(not it.isValid).len
|
|
|
|
result &= "(" & $(w.kMap.len - a) & "," & $a & ")"
|
|
|
|
lStr &= " " & $m & "=(" & $(l.kMap.len - c) & "," & $c & ")"
|
2024-03-22 17:31:56 +00:00
|
|
|
result &= " =>" & lStr
|
2023-09-11 20:38:49 +00:00
|
|
|
if backendOk:
|
2024-05-30 17:48:38 +00:00
|
|
|
result &= indent.toPfx & db.backend.pp(db, limit=limit, indent)
|
2024-06-03 20:10:35 +00:00
|
|
|
elif balancerOk:
|
2024-07-18 21:32:32 +00:00
|
|
|
result &= indent.toPfx & db.balancer.ppBalancer(db, indent+1)
|
2023-11-08 12:18:32 +00:00
|
|
|
|
2023-05-11 14:25:29 +00:00
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
# End
|
|
|
|
# ------------------------------------------------------------------------------
|