2023-05-11 14:25:29 +00:00
|
|
|
# Nimbus - Types, data structures and shared utilities used in network sync
|
|
|
|
#
|
|
|
|
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
|
|
|
# Licensed under either of
|
|
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
|
|
# http://opensource.org/licenses/MIT)
|
|
|
|
# at your option. This file may not be copied, modified, or
|
|
|
|
# distributed except according to those terms.
|
|
|
|
|
|
|
|
import
|
2023-08-22 18:44:54 +00:00
|
|
|
std/[hashes, os, sequtils],
|
2023-05-11 14:25:29 +00:00
|
|
|
eth/common,
|
|
|
|
rocksdb,
|
2023-06-09 11:17:37 +00:00
|
|
|
../../nimbus/db/aristo/[
|
2023-09-05 13:57:20 +00:00
|
|
|
aristo_constants, aristo_debug, aristo_desc,
|
|
|
|
aristo_filter/filter_scheduler, aristo_merge],
|
2023-05-11 14:25:29 +00:00
|
|
|
../../nimbus/db/kvstore_rocksdb,
|
2023-06-02 10:04:29 +00:00
|
|
|
../../nimbus/sync/protocol/snap/snap_types,
|
|
|
|
../test_sync_snap/test_types,
|
|
|
|
../replay/[pp, undump_accounts, undump_storages]
|
|
|
|
|
2023-06-12 18:16:03 +00:00
|
|
|
from ../../nimbus/sync/snap/range_desc
|
|
|
|
import NodeKey
|
|
|
|
|
2023-06-02 10:04:29 +00:00
|
|
|
type
|
|
|
|
ProofTrieData* = object
|
2023-06-12 18:16:03 +00:00
|
|
|
root*: HashKey
|
2023-06-02 10:04:29 +00:00
|
|
|
id*: int
|
|
|
|
proof*: seq[SnapProof]
|
2023-06-12 13:48:47 +00:00
|
|
|
kvpLst*: seq[LeafTiePayload]
|
2023-06-02 10:04:29 +00:00
|
|
|
|
2023-09-05 13:57:20 +00:00
|
|
|
const
|
|
|
|
QidSlotLyo* = [(4,0,10),(3,3,10),(3,4,10),(3,5,10)]
|
|
|
|
QidSample* = (3 * QidSlotLyo.stats.minCovered) div 2
|
|
|
|
|
2023-06-02 10:04:29 +00:00
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
# Private helpers
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
|
2023-06-09 11:17:37 +00:00
|
|
|
proc toPfx(indent: int): string =
|
|
|
|
"\n" & " ".repeat(indent)
|
2023-05-11 14:25:29 +00:00
|
|
|
|
2023-06-12 18:16:03 +00:00
|
|
|
proc to(a: NodeKey; T: type HashKey): T =
|
|
|
|
a.T
|
|
|
|
|
2023-05-11 14:25:29 +00:00
|
|
|
# ------------------------------------------------------------------------------
|
2023-06-09 11:17:37 +00:00
|
|
|
# Public pretty printing
|
2023-05-11 14:25:29 +00:00
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
|
2023-06-12 13:48:47 +00:00
|
|
|
proc pp*(
|
|
|
|
w: ProofTrieData;
|
|
|
|
rootID: VertexID;
|
2023-07-04 18:24:03 +00:00
|
|
|
db: AristoDbRef;
|
2023-06-12 13:48:47 +00:00
|
|
|
indent = 4;
|
|
|
|
): string =
|
2023-06-09 11:17:37 +00:00
|
|
|
let pfx = indent.toPfx
|
2023-06-12 13:48:47 +00:00
|
|
|
result = "(" & HashLabel(root: rootID, key: w.root).pp(db)
|
|
|
|
result &= "," & $w.id & ",[" & $w.proof.len & "],"
|
2023-06-09 11:17:37 +00:00
|
|
|
result &= pfx & " ["
|
|
|
|
for n,kvp in w.kvpLst:
|
|
|
|
if 0 < n:
|
|
|
|
result &= "," & pfx & " "
|
2023-06-12 13:48:47 +00:00
|
|
|
result &= "(" & kvp.leafTie.pp(db) & "," & $kvp.payload.pType & ")"
|
2023-06-09 11:17:37 +00:00
|
|
|
result &= "])"
|
|
|
|
|
|
|
|
proc pp*(w: ProofTrieData; indent = 4): string =
|
2023-07-04 18:24:03 +00:00
|
|
|
var db = AristoDbRef()
|
2023-06-12 13:48:47 +00:00
|
|
|
w.pp(VertexID(1), db, indent)
|
2023-06-09 11:17:37 +00:00
|
|
|
|
2023-06-12 13:48:47 +00:00
|
|
|
proc pp*(
|
|
|
|
w: openArray[ProofTrieData];
|
|
|
|
rootID: VertexID;
|
2023-07-04 18:24:03 +00:00
|
|
|
db: AristoDbRef;
|
2023-06-12 13:48:47 +00:00
|
|
|
indent = 4): string =
|
2023-06-09 11:17:37 +00:00
|
|
|
let pfx = indent.toPfx
|
2023-06-12 13:48:47 +00:00
|
|
|
"[" & w.mapIt(it.pp(rootID, db, indent + 1)).join("," & pfx & " ") & "]"
|
2023-06-09 11:17:37 +00:00
|
|
|
|
|
|
|
proc pp*(w: openArray[ProofTrieData]; indent = 4): string =
|
|
|
|
let pfx = indent.toPfx
|
|
|
|
"[" & w.mapIt(it.pp(indent + 1)).join("," & pfx & " ") & "]"
|
|
|
|
|
2023-07-04 18:24:03 +00:00
|
|
|
proc pp*(ltp: LeafTiePayload; db: AristoDbRef): string =
|
2023-06-22 11:13:24 +00:00
|
|
|
"(" & ltp.leafTie.pp(db) & "," & ltp.payload.pp(db) & ")"
|
|
|
|
|
2023-06-09 11:17:37 +00:00
|
|
|
# ----------
|
|
|
|
|
2023-05-11 14:25:29 +00:00
|
|
|
proc say*(noisy = false; pfx = "***"; args: varargs[string, `$`]) =
|
|
|
|
if noisy:
|
|
|
|
if args.len == 0:
|
|
|
|
echo "*** ", pfx
|
|
|
|
elif 0 < pfx.len and pfx[^1] != ' ':
|
|
|
|
echo pfx, " ", args.toSeq.join
|
|
|
|
else:
|
|
|
|
echo pfx, args.toSeq.join
|
|
|
|
|
2023-06-09 11:17:37 +00:00
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
# Public helpers
|
|
|
|
# ------------------------------------------------------------------------------
|
2023-06-02 10:04:29 +00:00
|
|
|
|
2023-06-30 22:22:33 +00:00
|
|
|
proc `==`*[T: AristoError|VertexID](a: T, b: int): bool =
|
|
|
|
a == T(b)
|
|
|
|
|
2023-07-05 13:50:11 +00:00
|
|
|
proc `==`*(a: (VertexID,AristoError), b: (int,int)): bool =
|
|
|
|
(a[0].int,a[1].int) == b
|
|
|
|
|
|
|
|
proc `==`*(a: (VertexID,AristoError), b: (int,AristoError)): bool =
|
|
|
|
(a[0].int,a[1]) == b
|
|
|
|
|
|
|
|
proc `==`*(a: (int,AristoError), b: (int,int)): bool =
|
|
|
|
(a[0],a[1].int) == b
|
|
|
|
|
2023-08-18 19:46:55 +00:00
|
|
|
proc `==`*(a: (int,VertexID,AristoError), b: (int,int,int)): bool =
|
|
|
|
(a[0], a[1].int, a[2].int) == b
|
2023-06-30 22:22:33 +00:00
|
|
|
|
2023-08-25 22:53:59 +00:00
|
|
|
proc `==`*(a: (QueueID,Hash), b: (int,Hash)): bool =
|
2023-08-22 18:44:54 +00:00
|
|
|
(a[0].int,a[1]) == b
|
|
|
|
|
2023-06-02 10:04:29 +00:00
|
|
|
proc to*(sample: AccountsSample; T: type seq[UndumpAccounts]): T =
|
|
|
|
## Convert test data into usable in-memory format
|
|
|
|
let file = sample.file.findFilePath.value
|
|
|
|
var root: Hash256
|
|
|
|
for w in file.undumpNextAccount:
|
|
|
|
let n = w.seenAccounts - 1
|
|
|
|
if n < sample.firstItem:
|
|
|
|
continue
|
|
|
|
if sample.lastItem < n:
|
|
|
|
break
|
|
|
|
if sample.firstItem == n:
|
|
|
|
root = w.root
|
|
|
|
elif w.root != root:
|
|
|
|
break
|
|
|
|
result.add w
|
|
|
|
|
|
|
|
proc to*(sample: AccountsSample; T: type seq[UndumpStorages]): T =
|
|
|
|
## Convert test data into usable in-memory format
|
|
|
|
let file = sample.file.findFilePath.value
|
|
|
|
var root: Hash256
|
|
|
|
for w in file.undumpNextStorages:
|
|
|
|
let n = w.seenAccounts - 1 # storages selector based on accounts
|
|
|
|
if n < sample.firstItem:
|
|
|
|
continue
|
|
|
|
if sample.lastItem < n:
|
|
|
|
break
|
|
|
|
if sample.firstItem == n:
|
|
|
|
root = w.root
|
|
|
|
elif w.root != root:
|
|
|
|
break
|
|
|
|
result.add w
|
|
|
|
|
2023-06-09 11:17:37 +00:00
|
|
|
proc to*(ua: seq[UndumpAccounts]; T: type seq[ProofTrieData]): T =
|
2023-06-12 18:16:03 +00:00
|
|
|
var (rootKey, rootVid) = (VOID_HASH_KEY, VertexID(0))
|
2023-06-09 11:17:37 +00:00
|
|
|
for w in ua:
|
2023-06-12 18:16:03 +00:00
|
|
|
let thisRoot = w.root.to(HashKey)
|
2023-06-09 11:17:37 +00:00
|
|
|
if rootKey != thisRoot:
|
|
|
|
(rootKey, rootVid) = (thisRoot, VertexID(rootVid.uint64 + 1))
|
2023-06-12 13:48:47 +00:00
|
|
|
if 0 < w.data.accounts.len:
|
|
|
|
result.add ProofTrieData(
|
|
|
|
root: rootKey,
|
|
|
|
proof: w.data.proof,
|
|
|
|
kvpLst: w.data.accounts.mapIt(LeafTiePayload(
|
2023-06-12 18:16:03 +00:00
|
|
|
leafTie: LeafTie(
|
|
|
|
root: rootVid,
|
2023-10-27 21:36:51 +00:00
|
|
|
path: it.accKey.to(HashKey).to(PathID)),
|
2023-07-05 20:27:48 +00:00
|
|
|
payload: PayloadRef(pType: RawData, rawBlob: it.accBlob))))
|
2023-06-09 11:17:37 +00:00
|
|
|
|
|
|
|
proc to*(us: seq[UndumpStorages]; T: type seq[ProofTrieData]): T =
|
2023-06-12 18:16:03 +00:00
|
|
|
var (rootKey, rootVid) = (VOID_HASH_KEY, VertexID(0))
|
2023-06-09 11:17:37 +00:00
|
|
|
for n,s in us:
|
|
|
|
for w in s.data.storages:
|
2023-06-12 18:16:03 +00:00
|
|
|
let thisRoot = w.account.storageRoot.to(HashKey)
|
2023-06-09 11:17:37 +00:00
|
|
|
if rootKey != thisRoot:
|
|
|
|
(rootKey, rootVid) = (thisRoot, VertexID(rootVid.uint64 + 1))
|
2023-06-12 13:48:47 +00:00
|
|
|
if 0 < w.data.len:
|
|
|
|
result.add ProofTrieData(
|
|
|
|
root: thisRoot,
|
|
|
|
id: n + 1,
|
|
|
|
kvpLst: w.data.mapIt(LeafTiePayload(
|
2023-06-12 18:16:03 +00:00
|
|
|
leafTie: LeafTie(
|
|
|
|
root: rootVid,
|
2023-10-27 21:36:51 +00:00
|
|
|
path: it.slotHash.to(HashKey).to(PathID)),
|
2023-07-05 20:27:48 +00:00
|
|
|
payload: PayloadRef(pType: RawData, rawBlob: it.slotData))))
|
2023-06-09 11:17:37 +00:00
|
|
|
if 0 < result.len:
|
|
|
|
result[^1].proof = s.data.proof
|
2023-06-02 10:04:29 +00:00
|
|
|
|
2023-06-12 13:48:47 +00:00
|
|
|
proc mapRootVid*(
|
|
|
|
a: openArray[LeafTiePayload];
|
|
|
|
toVid: VertexID;
|
|
|
|
): seq[LeafTiePayload] =
|
|
|
|
a.mapIt(LeafTiePayload(
|
|
|
|
leafTie: LeafTie(root: toVid, path: it.leafTie.path),
|
2023-06-09 11:17:37 +00:00
|
|
|
payload: it.payload))
|
2023-06-02 10:04:29 +00:00
|
|
|
|
2023-05-11 14:25:29 +00:00
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
# End
|
|
|
|
# ------------------------------------------------------------------------------
|