status-im/nimbus-eth1
2
0
Fork 0
mirror of https://github.com/status-im/nimbus-eth1.git synced 2025-02-24 17:58:30 +00:00
Code Issues Projects Releases Wiki Activity

Remove RawData from possible leaf payload types (#2794)

Browse Source 
This kind of data is not used except in tests where it is used only to
create databases that don't match actual usage of aristo.

Removing simplifies future optimizations that can focus on processing
specific leaf types more efficiently.

A casualty of this removal is some test code as well as some proof
generation code that is unused - on the surface, it looks like it should
be possible to port both of these to the more specific data types -
doing so would ensure that a database written by one part of the
codebase can interact with the other - as it stands, there is confusion
on this point since using the proof generation code will result in a
database of a shape that is incompatible with the rest of eth1.
This commit is contained in:
Jacek Sieka 2024-11-02 10:29:16 +01:00 committed by GitHub
parent a5541a5a4f
commit 58cde36656
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 321 additions and 463 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
TracerTests.md
View File

@ -2,16 +2,16 @@ TracerTests
=== ===
## TracerTests ## TracerTests
```diff ```diff
+ block46147.json OK block46147.json Skip
+ block46400.json OK block46400.json Skip
+ block46402.json OK block46402.json Skip
+ block47205.json OK block47205.json Skip
+ block48712.json OK block48712.json Skip
+ block48915.json OK block48915.json Skip
+ block49018.json OK block49018.json Skip
+ block97.json OK block97.json Skip
``` ```
OK: 8/8 Fail: 0/8 Skip: 0/8 OK: 0/8 Fail: 0/8 Skip: 8/8
---TOTAL--- ---TOTAL---
OK: 8/8 Fail: 0/8 Skip: 0/8 OK: 0/8 Fail: 0/8 Skip: 8/8

1
nimbus/db/aristo.nim
View File

@ -32,7 +32,6 @@ export
leftPairs, # iterators leftPairs, # iterators
rightPairs, rightPairs,
rightPairsAccount, rightPairsAccount,
rightPairsGeneric,
rightPairsStorage rightPairsStorage
import import

55
nimbus/db/aristo/aristo_blobify.nim
View File

@ -124,10 +124,6 @@ proc load256(data: openArray[byte]; start: var int, len: int): Result[UInt256,Ar
proc blobifyTo*(pyl: LeafPayload, data: var seq[byte]) = proc blobifyTo*(pyl: LeafPayload, data: var seq[byte]) =
case pyl.pType case pyl.pType
of RawData:
data &= pyl.rawBlob
data &= [0x10.byte]
of AccountData: of AccountData:
# `lens` holds `len-1` since `mask` filters out the zero-length case (which # `lens` holds `len-1` since `mask` filters out the zero-length case (which
# allows saving 1 bit per length) # allows saving 1 bit per length)
@ -248,45 +244,42 @@ proc deblobify(
pyl: var LeafPayload; pyl: var LeafPayload;
): Result[void,AristoError] = ): Result[void,AristoError] =
if data.len == 0: if data.len == 0:
pyl = LeafPayload(pType: RawData) return err(DeblobVtxTooShort)
return ok()
let mask = data[^1] let mask = data[^1]
if (mask and 0x10) > 0: # unstructured payload
pyl = LeafPayload(pType: RawData, rawBlob: data[0 .. ^2])
return ok()
if (mask and 0x20) > 0: # Slot storage data if (mask and 0x20) > 0: # Slot storage data
pyl = LeafPayload( pyl = LeafPayload(
pType: StoData, pType: StoData,
stoData: ?deblobify(data.toOpenArray(0, data.len - 2), UInt256)) stoData: ?deblobify(data.toOpenArray(0, data.len - 2), UInt256))
return ok() ok()
elif (mask and 0xf0) == 0: # Only account fields set
pyl = LeafPayload(pType: AccountData)
var
start = 0
lens = uint16.fromBytesBE(data.toOpenArray(data.len - 3, data.len - 2))
pyl = LeafPayload(pType: AccountData) if (mask and 0x01) > 0:
var let len = lens and 0b111
start = 0 pyl.account.nonce = ? load64(data, start, int(len + 1))
lens = uint16.fromBytesBE(data.toOpenArray(data.len - 3, data.len - 2))
if (mask and 0x01) > 0: if (mask and 0x02) > 0:
let len = lens and 0b111 let len = (lens shr 3) and 0b11111
pyl.account.nonce = ? load64(data, start, int(len + 1)) pyl.account.balance = ? load256(data, start, int(len + 1))
if (mask and 0x02) > 0: if (mask and 0x04) > 0:
let len = (lens shr 3) and 0b11111 let len = (lens shr 8) and 0b111
pyl.account.balance = ? load256(data, start, int(len + 1)) pyl.stoID = (true, VertexID(? load64(data, start, int(len + 1))))
if (mask and 0x04) > 0: if (mask and 0x08) > 0:
let len = (lens shr 8) and 0b111 if data.len() < start + 32:
pyl.stoID = (true, VertexID(? load64(data, start, int(len + 1)))) return err(DeblobCodeLenUnsupported)
discard pyl.account.codeHash.data.copyFrom(data.toOpenArray(start, start + 31))
else:
pyl.account.codeHash = EMPTY_CODE_HASH
if (mask and 0x08) > 0: ok()
if data.len() < start + 32:
return err(DeblobCodeLenUnsupported)
discard pyl.account.codeHash.data.copyFrom(data.toOpenArray(start, start + 31))
else: else:
pyl.account.codeHash = EMPTY_CODE_HASH err(DeblobUnknown)
ok()
proc deblobifyType*(record: openArray[byte]; T: type VertexRef): proc deblobifyType*(record: openArray[byte]; T: type VertexRef):
Result[VertexType, AristoError] = Result[VertexType, AristoError] =

4
nimbus/db/aristo/aristo_compute.nim
View File

@ -249,8 +249,6 @@ proc computeKeyImpl(
storageRoot: skey.to(Hash32), storageRoot: skey.to(Hash32),
codeHash: vtx.lData.account.codeHash, codeHash: vtx.lData.account.codeHash,
) )
of RawData:
vtx.lData.rawBlob
of StoData: of StoData:
# TODO avoid memory allocation when encoding storage data # TODO avoid memory allocation when encoding storage data
rlp.encode(vtx.lData.stoData) rlp.encode(vtx.lData.stoData)
@ -371,8 +369,6 @@ proc computeLeafKeysImpl(
codeHash: vtx.lData.account.codeHash, codeHash: vtx.lData.account.codeHash,
) )
writer2.finish() writer2.finish()
of RawData:
vtx.lData.rawBlob
of StoData: of StoData:
writer2.clear() writer2.clear()
writer2.append(vtx.lData.stoData) writer2.append(vtx.lData.stoData)

2
nimbus/db/aristo/aristo_debug.nim
View File

@ -180,8 +180,6 @@ func ppAriAccount(a: AristoAccount): string =
func ppPayload(p: LeafPayload, db: AristoDbRef): string = func ppPayload(p: LeafPayload, db: AristoDbRef): string =
case p.pType: case p.pType:
of RawData:
result &= p.rawBlob.toHex.squeeze(hex=true)
of AccountData: of AccountData:
result = "(" & p.account.ppAriAccount() & "," & p.stoID.ppVid & ")" result = "(" & p.account.ppAriAccount() & "," & p.stoID.ppVid & ")"
of StoData: of StoData:

11
nimbus/db/aristo/aristo_desc/desc_structural.nim
View File

@ -43,7 +43,6 @@ type
PayloadType* = enum PayloadType* = enum
## Type of leaf data. ## Type of leaf data.
RawData ## Generic data
AccountData ## `Aristo account` with vertex IDs links AccountData ## `Aristo account` with vertex IDs links
StoData ## Slot storage data StoData ## Slot storage data
@ -58,10 +57,7 @@ type
LeafPayload* = object LeafPayload* = object
## The payload type depends on the sub-tree used. The `VertexID(1)` rooted ## The payload type depends on the sub-tree used. The `VertexID(1)` rooted
## sub-tree only has `AccountData` type payload, stoID-based have StoData ## sub-tree only has `AccountData` type payload, stoID-based have StoData
## while generic have RawData
case pType*: PayloadType case pType*: PayloadType
of RawData:
rawBlob*: seq[byte] ## Opaque data, default value
of AccountData: of AccountData:
account*: AristoAccount account*: AristoAccount
stoID*: StorageID ## Storage vertex ID (if any) stoID*: StorageID ## Storage vertex ID (if any)
@ -157,9 +153,6 @@ proc `==`*(a, b: LeafPayload): bool =
if a.pType != b.pType: if a.pType != b.pType:
return false return false
case a.pType: case a.pType:
of RawData:
if a.rawBlob != b.rawBlob:
return false
of AccountData: of AccountData:
if a.account != b.account or if a.account != b.account or
a.stoID != b.stoID: a.stoID != b.stoID:
@ -208,10 +201,6 @@ proc `==`*(a, b: NodeRef): bool =
func dup*(pld: LeafPayload): LeafPayload = func dup*(pld: LeafPayload): LeafPayload =
## Duplicate payload. ## Duplicate payload.
case pld.pType: case pld.pType:
of RawData:
LeafPayload(
pType: RawData,
rawBlob: pld.rawBlob)
of AccountData: of AccountData:
LeafPayload( LeafPayload(
pType: AccountData, pType: AccountData,

44
nimbus/db/aristo/aristo_fetch.nim
View File

@ -23,18 +23,6 @@ import
# Private functions # Private functions
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
func mustBeGeneric(
root: VertexID;
): Result[void,AristoError] =
## Verify that `root` is neither from an accounts tree nor a strorage tree.
if not root.isValid:
return err(FetchRootVidMissing)
elif root == VertexID(1):
return err(FetchAccRootNotAccepted)
elif LEAST_FREE_VID <= root.distinctBase:
return err(FetchStoRootNotAccepted)
ok()
proc retrieveLeaf( proc retrieveLeaf(
db: AristoDbRef; db: AristoDbRef;
root: VertexID; root: VertexID;
@ -260,38 +248,6 @@ proc hasPathAccount*(
## ##
db.hasAccountPayload(accPath) db.hasAccountPayload(accPath)
proc fetchGenericData*(
db: AristoDbRef;
root: VertexID;
path: openArray[byte];
): Result[seq[byte],AristoError] =
## For a generic sub-tree starting at `root`, fetch the data record
## indexed by `path`.
##
? root.mustBeGeneric()
let pyl = ? db.retrieveLeaf(root, path)
assert pyl.lData.pType == RawData # debugging only
ok pyl.lData.rawBlob
proc fetchGenericState*(
db: AristoDbRef;
root: VertexID;
updateOk: bool;
): Result[Hash32,AristoError] =
## Fetch the Merkle hash of the argument `root`.
db.retrieveMerkleHash(root, updateOk)
proc hasPathGeneric*(
db: AristoDbRef;
root: VertexID;
path: openArray[byte];
): Result[bool,AristoError] =
## For a generic sub-tree starting at `root` and indexed by `path`, query
## whether this record exists on the database.
##
? root.mustBeGeneric()
db.hasPayload(root, path)
proc fetchStorageData*( proc fetchStorageData*(
db: AristoDbRef; db: AristoDbRef;
accPath: Hash32; accPath: Hash32;

31
nimbus/db/aristo/aristo_merge.nim
View File

@ -202,37 +202,6 @@ proc mergeAccountRecord*(
ok true ok true
proc mergeGenericData*(
db: AristoDbRef; # Database, top layer
root: VertexID; # MPT state root
path: openArray[byte]; # Leaf item to add to the database
data: openArray[byte]; # Raw data payload value
): Result[bool,AristoError] =
## Variant of `mergeXXX()` for generic sub-trees, i.e. for arguments
## `root` greater than `VertexID(1)` and smaller than `LEAST_FREE_VID`.
##
## On success, the function returns `true` if the `data` argument was merged
## into the database ot updated, and `false` if it was on the database
## already.
##
# Verify that `root` is neither an accounts tree nor a strorage tree.
if not root.isValid:
return err(MergeRootVidMissing)
elif root == VertexID(1):
return err(MergeAccRootNotAccepted)
elif LEAST_FREE_VID <= root.distinctBase:
return err(MergeStoRootNotAccepted)
let
pyl = LeafPayload(pType: RawData, rawBlob: @data)
discard db.mergePayloadImpl(root, path, Opt.none(VertexRef), pyl).valueOr:
if error == MergeNoAction:
return ok false
return err error
ok true
proc mergeStorageData*( proc mergeStorageData*(
db: AristoDbRef; # Database, top layer db: AristoDbRef; # Database, top layer
accPath: Hash32; # Needed for accounts payload accPath: Hash32; # Needed for accounts payload

11
nimbus/db/aristo/aristo_nearby.nim
View File

@ -439,17 +439,6 @@ iterator rightPairsAccount*(
for (lty,pyl) in db.rightPairs LeafTie(root: VertexID(1), path: start): for (lty,pyl) in db.rightPairs LeafTie(root: VertexID(1), path: start):
yield (lty.path, pyl.account) yield (lty.path, pyl.account)
iterator rightPairsGeneric*(
db: AristoDbRef; # Database layer
root: VertexID; # Generic root (different from VertexID)
start = low(PathID); # Before or at first value
): (PathID,seq[byte]) =
## Variant of `rightPairs()` for a generic tree
# Verify that `root` is neither from an accounts tree nor a strorage tree.
if VertexID(1) < root and root.distinctBase < LEAST_FREE_VID:
for (lty,pyl) in db.rightPairs LeafTie(root: VertexID(1), path: start):
yield (lty.path, pyl.rawBlob)
iterator rightPairsStorage*( iterator rightPairsStorage*(
db: AristoDbRef; # Database layer db: AristoDbRef; # Database layer
accPath: Hash32; # Account the storage data belong to accPath: Hash32; # Account the storage data belong to

31
nimbus/db/aristo/aristo_part.nim
View File

@ -366,37 +366,6 @@ proc partReRoot*(
# Public merge functions on partial tree database # Public merge functions on partial tree database
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
proc partMergeGenericData*(
ps: PartStateRef;
root: VertexID; # MPT state root
path: openArray[byte]; # Leaf item to add to the database
data: openArray[byte]; # Raw data payload value
): Result[bool,AristoError] =
## ..
let mergeError = block:
# Opportunistically try whether it just works
let rc = ps.db.mergeGenericData(root, path, data)
if rc.isOk or rc.error != GetVtxNotFound:
return rc
rc.error
# Otherwise clean the way removing blind link and retry
let
ctx = ps.ctxMergeBegin(root, path).valueOr:
let ctxErr = if error == PartCtxNotAvailable: mergeError else: error
return err(ctxErr)
rc = ps.db.mergeGenericData(root, path, data)
# Evaluate result => commit/rollback
if rc.isErr:
? ctx.ctxMergeRollback()
return rc
if not ? ctx.ctxMergeCommit():
return err(PartVtxSlotWasNotModified)
ok(rc.value)
proc partMergeAccountRecord*( proc partMergeAccountRecord*(
ps: PartStateRef; ps: PartStateRef;
accPath: Hash32; # Even nibbled byte path accPath: Hash32; # Even nibbled byte path

19
nimbus/db/aristo/aristo_part/part_helpers.nim
View File

@ -64,14 +64,15 @@ proc read(rlp: var Rlp; T: type PrfNode): T {.gcsafe, raises: [RlpError].} =
let (isLeaf, pathSegment) = NibblesBuf.fromHexPrefix blobs[0] let (isLeaf, pathSegment) = NibblesBuf.fromHexPrefix blobs[0]
if isLeaf: if isLeaf:
return PrfNode( return PrfNode(
prfType: ignore, prfType: ignore, )
vtx: VertexRef( # TODO interpret the blob (?)
vType: Leaf, # vtx: VertexRef(
pfx: pathSegment, # vType: Leaf,
lData: LeafPayload( # pfx: pathSegment,
pType: RawData, # lData: LeafPayload(
rawBlob: blobs[1]))) # pType: RawData,
# rawBlob: blobs[1])))
else: else:
var node = PrfNode( var node = PrfNode(
prfType: isExtension, prfType: isExtension,
@ -145,7 +146,9 @@ func toNodesTab*(
# Decode payload to deficated format for storage or accounts # Decode payload to deficated format for storage or accounts
var pyl: PrfPayload var pyl: PrfPayload
try: try:
pyl = rlp.decode(nd.vtx.lData.rawBlob, PrfPayload) # TODO interpret the blob
# pyl = rlp.decode(nd.vtx.lData.rawBlob, PrfPayload)
pyl = PrfPayload(prfType: isError, error: PartRlpPayloadException)
except RlpError: except RlpError:
pyl = PrfPayload(prfType: isError, error: PartRlpPayloadException) pyl = PrfPayload(prfType: isError, error: PartRlpPayloadException)

2
nimbus/db/aristo/aristo_serialise.nim
View File

@ -34,8 +34,6 @@ proc serialise(
## of account type, otherwise pass the data as is. ## of account type, otherwise pass the data as is.
## ##
case pyl.pType: case pyl.pType:
of RawData:
ok pyl.rawBlob
of AccountData: of AccountData:
let key = block: let key = block:
if pyl.stoID.isValid: if pyl.stoID.isValid:

2
tests/test_aristo/test_blobify.nim
View File

@ -15,7 +15,6 @@ import unittest2, ../../nimbus/db/aristo/aristo_blobify
suite "Aristo blobify": suite "Aristo blobify":
test "VertexRef roundtrip": test "VertexRef roundtrip":
let let
leafRawData = VertexRef(vType: Leaf, lData: LeafPayload(pType: RawData))
leafAccount = VertexRef(vType: Leaf, lData: LeafPayload(pType: AccountData)) leafAccount = VertexRef(vType: Leaf, lData: LeafPayload(pType: AccountData))
leafStoData = leafStoData =
VertexRef(vType: Leaf, lData: LeafPayload(pType: StoData, stoData: 42.u256)) VertexRef(vType: Leaf, lData: LeafPayload(pType: StoData, stoData: 42.u256))
@ -65,7 +64,6 @@ suite "Aristo blobify":
) )
check: check:
deblobify(blobify(leafRawData), VertexRef)[] == leafRawData
deblobify(blobify(leafAccount), VertexRef)[] == leafAccount deblobify(blobify(leafAccount), VertexRef)[] == leafAccount
deblobify(blobify(leafStoData), VertexRef)[] == leafStoData deblobify(blobify(leafStoData), VertexRef)[] == leafStoData
deblobify(blobify(branch), VertexRef)[] == branch deblobify(blobify(branch), VertexRef)[] == branch

70
tests/test_aristo/test_helpers.nim
View File

@ -160,15 +160,16 @@ func to*(ua: seq[UndumpAccounts]; T: type seq[ProofTrieData]): T =
let thisRoot = w.root let thisRoot = w.root
if rootKey != thisRoot: if rootKey != thisRoot:
(rootKey, rootVid) = (thisRoot, VertexID(rootVid.uint64 + 1)) (rootKey, rootVid) = (thisRoot, VertexID(rootVid.uint64 + 1))
if 0 < w.data.accounts.len: # TODO rewrite as account leaves
result.add ProofTrieData( # if 0 < w.data.accounts.len:
root: rootKey, # result.add ProofTrieData(
proof: cast[seq[seq[byte]]](w.data.proof), # root: rootKey,
kvpLst: w.data.accounts.mapIt(LeafTiePayload( # proof: cast[seq[seq[byte]]](w.data.proof),
leafTie: LeafTie( # kvpLst: w.data.accounts.mapIt(LeafTiePayload(
root: rootVid, # leafTie: LeafTie(
path: it.accKey.to(PathID)), # root: rootVid,
payload: LeafPayload(pType: RawData, rawBlob: it.accBlob)))) # path: it.accKey.to(PathID)),
# payload: LeafPayload(pType: RawData, rawBlob: it.accBlob))))
func to*(us: seq[UndumpStorages]; T: type seq[ProofTrieData]): T = func to*(us: seq[UndumpStorages]; T: type seq[ProofTrieData]): T =
var (rootKey, rootVid) = (default(Hash32), VertexID(0)) var (rootKey, rootVid) = (default(Hash32), VertexID(0))
@ -177,15 +178,17 @@ func to*(us: seq[UndumpStorages]; T: type seq[ProofTrieData]): T =
let thisRoot = w.account.storageRoot let thisRoot = w.account.storageRoot
if rootKey != thisRoot: if rootKey != thisRoot:
(rootKey, rootVid) = (thisRoot, VertexID(rootVid.uint64 + 1)) (rootKey, rootVid) = (thisRoot, VertexID(rootVid.uint64 + 1))
if 0 < w.data.len: # TODO rewrite as account leaves
result.add ProofTrieData(
root: thisRoot, # if 0 < w.data.len:
id: n + 1, # result.add ProofTrieData(
kvpLst: w.data.mapIt(LeafTiePayload( # root: thisRoot,
leafTie: LeafTie( # id: n + 1,
root: rootVid, # kvpLst: w.data.mapIt(LeafTiePayload(
path: it.slotHash.to(PathID)), # leafTie: LeafTie(
payload: LeafPayload(pType: RawData, rawBlob: it.slotData)))) # root: rootVid,
# path: it.slotHash.to(PathID)),
# payload: LeafPayload(pType: RawData, rawBlob: it.slotData))))
if 0 < result.len: if 0 < result.len:
result[^1].proof = cast[seq[seq[byte]]](s.data.proof) result[^1].proof = cast[seq[seq[byte]]](s.data.proof)
@ -217,14 +220,6 @@ proc schedStow*(
# ------------------ # ------------------
proc mergeGenericData*(
db: AristoDbRef; # Database, top layer
leaf: LeafTiePayload; # Leaf item to add to the database
): Result[bool,AristoError] =
## Variant of `mergeGenericData()`.
db.mergeGenericData(
leaf.leafTie.root, @(leaf.leafTie.path), leaf.payload.rawBlob)
proc mergeList*( proc mergeList*(
db: AristoDbRef; # Database, top layer db: AristoDbRef; # Database, top layer
leafs: openArray[LeafTiePayload]; # Leaf items to add to the database leafs: openArray[LeafTiePayload]; # Leaf items to add to the database
@ -235,17 +230,18 @@ proc mergeList*(
for n,w in leafs: for n,w in leafs:
noisy.say "*** mergeList", noisy.say "*** mergeList",
" n=", n, "/", leafs.len " n=", n, "/", leafs.len
let rc = db.mergeGenericData w # TODO refactor to not use generic data
noisy.say "*** mergeList", # let rc = db.mergeGenericData w
" n=", n, "/", leafs.len, # noisy.say "*** mergeList",
" rc=", (if rc.isOk: "ok" else: $rc.error), # " n=", n, "/", leafs.len,
"\n -------------\n" # " rc=", (if rc.isOk: "ok" else: $rc.error),
if rc.isErr: # "\n -------------\n"
return (n,dups,rc.error) # if rc.isErr:
elif rc.value: # return (n,dups,rc.error)
merged.inc # elif rc.value:
else: # merged.inc
dups.inc # else:
# dups.inc
(merged, dups, AristoError(0)) (merged, dups, AristoError(0))

89
tests/test_aristo/test_merge_proof.nim
View File

@ -93,58 +93,59 @@ proc testMergeProofAndKvpList*(
list: openArray[ProofTrieData]; list: openArray[ProofTrieData];
rdbPath: string; # Rocks DB storage directory rdbPath: string; # Rocks DB storage directory
idPfx = ""; idPfx = "";
): bool = ): bool {.deprecated.} =
var # TODO update for non-generic data
ps = PartStateRef(nil) # var
tx = AristoTxRef(nil) # ps = PartStateRef(nil)
rootKey: Hash32 # tx = AristoTxRef(nil)
defer: # rootKey: Hash32
if not ps.isNil: # defer:
ps.db.finish(eradicate=true) # if not ps.isNil:
# ps.db.finish(eradicate=true)
for n,w in list: # for n,w in list:
# Start new database upon request # # Start new database upon request
if w.root != rootKey or w.proof.len == 0: # if w.root != rootKey or w.proof.len == 0:
ps.innerCleanUp() # ps.innerCleanUp()
let db = block: # let db = block:
# New DB with disabled filter slots management # # New DB with disabled filter slots management
if 0 < rdbPath.len: # if 0 < rdbPath.len:
let (dbOpts, cfOpts) = DbOptions.init().toRocksDb() # let (dbOpts, cfOpts) = DbOptions.init().toRocksDb()
let rc = AristoDbRef.init(RdbBackendRef, rdbPath, DbOptions.init(), dbOpts, cfOpts, []) # let rc = AristoDbRef.init(RdbBackendRef, rdbPath, DbOptions.init(), dbOpts, cfOpts, [])
xCheckRc rc.error == 0 # xCheckRc rc.error == 0
rc.value()[0] # rc.value()[0]
else: # else:
AristoDbRef.init(MemBackendRef) # AristoDbRef.init(MemBackendRef)
ps = PartStateRef.init(db) # ps = PartStateRef.init(db)
# Start transaction (double frame for testing) # # Start transaction (double frame for testing)
tx = ps.db.txBegin().value.to(AristoDbRef).txBegin().value # tx = ps.db.txBegin().value.to(AristoDbRef).txBegin().value
xCheck tx.isTop() # xCheck tx.isTop()
# Update root # # Update root
rootKey = w.root # rootKey = w.root
if 0 < w.proof.len: # if 0 < w.proof.len:
let rc = ps.partPut(w.proof, ForceGenericPayload) # let rc = ps.partPut(w.proof, ForceGenericPayload)
xCheckRc rc.error == 0 # xCheckRc rc.error == 0
block: # block:
let rc = ps.check() # let rc = ps.check()
xCheckRc rc.error == (0,0) # xCheckRc rc.error == (0,0)
for ltp in w.kvpLst: # for ltp in w.kvpLst:
block: # block:
let rc = ps.partMergeGenericData( # let rc = ps.partMergeGenericData(
testRootVid, @(ltp.leafTie.path), ltp.payload.rawBlob) # testRootVid, @(ltp.leafTie.path), ltp.payload.rawBlob)
xCheckRc rc.error == 0 # xCheckRc rc.error == 0
block: # block:
let rc = ps.check() # let rc = ps.check()
xCheckRc rc.error == (0,0) # xCheckRc rc.error == (0,0)
block: # block:
let saveBeOk = tx.saveToBackend(noisy=noisy, debugID=n) # let saveBeOk = tx.saveToBackend(noisy=noisy, debugID=n)
xCheck saveBeOk # xCheck saveBeOk
true true

7
tests/test_aristo/test_portal_proof.nim
View File

@ -88,8 +88,6 @@ proc payloadAsBlob(pyl: LeafPayload; ps: PartStateRef): seq[byte] =
## ##
const info = "payloadAsBlob" const info = "payloadAsBlob"
case pyl.pType: case pyl.pType:
of RawData:
pyl.rawBlob
of AccountData: of AccountData:
let key = block: let key = block:
if pyl.stoID.isValid: if pyl.stoID.isValid:
@ -135,7 +133,10 @@ when false:
# Private test functions # Private test functions
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
proc testCreatePortalProof(node: JsonNode, testStatusIMPL: var TestStatus) = proc testCreatePortalProof(node: JsonNode, testStatusIMPL: var TestStatus) {.deprecated: "need to be rewritten to use non-generic data".} =
block: # TODO remove after rewrite
skip
return
const info = "testCreateProofTwig" const info = "testCreateProofTwig"
# Create partial database # Create partial database

304
tests/test_aristo/test_tx.nim
View File

@ -247,106 +247,106 @@ proc testTxMergeAndDeleteOneByOne*(
noisy: bool; noisy: bool;
list: openArray[ProofTrieData]; list: openArray[ProofTrieData];
rdbPath: string; # Rocks DB storage directory rdbPath: string; # Rocks DB storage directory
): bool = ): bool {.deprecated: "rewrite to use non-generic data".} =
var # var
prng = PrngDesc.init 42 # prng = PrngDesc.init 42
db = AristoDbRef(nil) # db = AristoDbRef(nil)
fwdRevVfyToggle = true # fwdRevVfyToggle = true
defer: # defer:
if not db.isNil: # if not db.isNil:
db.finish(eradicate=true) # db.finish(eradicate=true)
for n,w in list: # for n,w in list:
# Start with brand new persistent database. # # Start with brand new persistent database.
db = block: # db = block:
if 0 < rdbPath.len: # if 0 < rdbPath.len:
let (dbOpts, cfOpts) = DbOptions.init().toRocksDb() # let (dbOpts, cfOpts) = DbOptions.init().toRocksDb()
let rc = AristoDbRef.init(RdbBackendRef, rdbPath, DbOptions.init(), dbOpts, cfOpts, []) # let rc = AristoDbRef.init(RdbBackendRef, rdbPath, DbOptions.init(), dbOpts, cfOpts, [])
xCheckRc rc.error == 0 # xCheckRc rc.error == 0
rc.value()[0] # rc.value()[0]
else: # else:
AristoDbRef.init(MemBackendRef) # AristoDbRef.init(MemBackendRef)
# Start transaction (double frame for testing) # # Start transaction (double frame for testing)
xCheck db.txTop.isErr # xCheck db.txTop.isErr
var tx = db.txBegin().value.to(AristoDbRef).txBegin().value # var tx = db.txBegin().value.to(AristoDbRef).txBegin().value
xCheck tx.isTop() # xCheck tx.isTop()
xCheck tx.level == 2 # xCheck tx.level == 2
# Reset database so that the next round has a clean setup # # Reset database so that the next round has a clean setup
defer: db.innerCleanUp # defer: db.innerCleanUp
# Merge leaf data into main trie # # Merge leaf data into main trie
let kvpLeafs = block: # let kvpLeafs = block:
var lst = w.kvpLst.mapRootVid testRootVid # var lst = w.kvpLst.mapRootVid testRootVid
# The list might be reduced for isolation of particular properties, # # The list might be reduced for isolation of particular properties,
# e.g. lst.setLen(min(5,lst.len)) # # e.g. lst.setLen(min(5,lst.len))
lst # lst
for i,leaf in kvpLeafs: # for i,leaf in kvpLeafs:
let rc = db.mergeGenericData leaf # let rc = db.mergeGenericData leaf
xCheckRc rc.error == 0 # xCheckRc rc.error == 0
# List of all leaf entries that should be on the database # # List of all leaf entries that should be on the database
var leafsLeft = kvpLeafs.mapIt(it.leafTie).toHashSet # var leafsLeft = kvpLeafs.mapIt(it.leafTie).toHashSet
# Provide a (reproducible) peudo-random copy of the leafs list # # Provide a (reproducible) peudo-random copy of the leafs list
let leafVidPairs = block: # let leafVidPairs = block:
let rc = db.randomisedLeafs(leafsLeft, prng) # let rc = db.randomisedLeafs(leafsLeft, prng)
xCheckRc rc.error == (0,0) # xCheckRc rc.error == (0,0)
rc.value # rc.value
# Trigger subsequent saving tasks in loop below # # Trigger subsequent saving tasks in loop below
let (saveMod, saveRest, relax) = block: # let (saveMod, saveRest, relax) = block:
if leafVidPairs.len < 17: (7, 3, false) # if leafVidPairs.len < 17: (7, 3, false)
elif leafVidPairs.len < 31: (11, 7, false) # elif leafVidPairs.len < 31: (11, 7, false)
else: (leafVidPairs.len div 5, 11, true) # else: (leafVidPairs.len div 5, 11, true)
# === Loop over leafs === # # === Loop over leafs ===
for u,lvp in leafVidPairs: # for u,lvp in leafVidPairs:
let # let
runID = n + list.len * u # runID = n + list.len * u
tailWalkVerify = 7 # + 999 # tailWalkVerify = 7 # + 999
doSaveBeOk = ((u mod saveMod) == saveRest) # doSaveBeOk = ((u mod saveMod) == saveRest)
(leaf, lid) = lvp # (leaf, lid) = lvp
if doSaveBeOk: # if doSaveBeOk:
let saveBeOk = tx.saveToBackend(relax=relax, noisy=noisy, runID) # let saveBeOk = tx.saveToBackend(relax=relax, noisy=noisy, runID)
xCheck saveBeOk: # xCheck saveBeOk:
noisy.say "***", "del1by1(2)", # noisy.say "***", "del1by1(2)",
" u=", u, # " u=", u,
" n=", n, "/", list.len, # " n=", n, "/", list.len,
"\n db\n ", db.pp(backendOk=true), # "\n db\n ", db.pp(backendOk=true),
"" # ""
# Delete leaf # # Delete leaf
block: # block:
let rc = db.deleteGenericData(leaf.root, @(leaf.path)) # let rc = db.deleteGenericData(leaf.root, @(leaf.path))
xCheckRc rc.error == 0 # xCheckRc rc.error == 0
# Update list of remaininf leafs # # Update list of remaininf leafs
leafsLeft.excl leaf # leafsLeft.excl leaf
let deletedVtx = tx.db.getVtx lid # let deletedVtx = tx.db.getVtx lid
xCheck deletedVtx.isValid == false: # xCheck deletedVtx.isValid == false:
noisy.say "***", "del1by1(8)" # noisy.say "***", "del1by1(8)"
# Walking the database is too slow for large tables. So the hope is that # # Walking the database is too slow for large tables. So the hope is that
# potential errors will not go away and rather pop up later, as well. # # potential errors will not go away and rather pop up later, as well.
if leafsLeft.len <= tailWalkVerify: # if leafsLeft.len <= tailWalkVerify:
if u < leafVidPairs.len-1: # if u < leafVidPairs.len-1:
if fwdRevVfyToggle: # if fwdRevVfyToggle:
fwdRevVfyToggle = false # fwdRevVfyToggle = false
if not db.fwdWalkVerify(leaf.root, leafsLeft, noisy, runID): # if not db.fwdWalkVerify(leaf.root, leafsLeft, noisy, runID):
return # return
else: # else:
fwdRevVfyToggle = true # fwdRevVfyToggle = true
if not db.revWalkVerify(leaf.root, leafsLeft, noisy, runID): # if not db.revWalkVerify(leaf.root, leafsLeft, noisy, runID):
return # return
when true and false: # when true and false:
noisy.say "***", "del1by1(9)", # noisy.say "***", "del1by1(9)",
" n=", n, "/", list.len, # " n=", n, "/", list.len,
" nLeafs=", kvpLeafs.len # " nLeafs=", kvpLeafs.len
true true
@ -355,79 +355,79 @@ proc testTxMergeAndDeleteSubTree*(
noisy: bool; noisy: bool;
list: openArray[ProofTrieData]; list: openArray[ProofTrieData];
rdbPath: string; # Rocks DB storage directory rdbPath: string; # Rocks DB storage directory
): bool = ): bool {.deprecated: "rewrite to use non-generic data".} =
var # var
prng = PrngDesc.init 42 # prng = PrngDesc.init 42
db = AristoDbRef(nil) # db = AristoDbRef(nil)
defer: # defer:
if not db.isNil: # if not db.isNil:
db.finish(eradicate=true) # db.finish(eradicate=true)
for n,w in list: # for n,w in list:
# Start with brand new persistent database. # # Start with brand new persistent database.
db = block: # db = block:
if 0 < rdbPath.len: # if 0 < rdbPath.len:
let (dbOpts, cfOpts) = DbOptions.init().toRocksDb() # let (dbOpts, cfOpts) = DbOptions.init().toRocksDb()
let rc = AristoDbRef.init(RdbBackendRef, rdbPath, DbOptions.init(), dbOpts, cfOpts, []) # let rc = AristoDbRef.init(RdbBackendRef, rdbPath, DbOptions.init(), dbOpts, cfOpts, [])
xCheckRc rc.error == 0 # xCheckRc rc.error == 0
rc.value()[0] # rc.value()[0]
else: # else:
AristoDbRef.init(MemBackendRef) # AristoDbRef.init(MemBackendRef)
# Start transaction (double frame for testing) # # Start transaction (double frame for testing)
xCheck db.txTop.isErr # xCheck db.txTop.isErr
var tx = db.txBegin().value.to(AristoDbRef).txBegin().value # var tx = db.txBegin().value.to(AristoDbRef).txBegin().value
xCheck tx.isTop() # xCheck tx.isTop()
xCheck tx.level == 2 # xCheck tx.level == 2
# Reset database so that the next round has a clean setup # # Reset database so that the next round has a clean setup
defer: db.innerCleanUp # defer: db.innerCleanUp
# Merge leaf data into main trie (w/vertex ID 2) # # Merge leaf data into main trie (w/vertex ID 2)
let kvpLeafs = block: # let kvpLeafs = block:
var lst = w.kvpLst.mapRootVid testRootVid # var lst = w.kvpLst.mapRootVid testRootVid
# The list might be reduced for isolation of particular properties, # # The list might be reduced for isolation of particular properties,
# e.g. lst.setLen(min(5,lst.len)) # # e.g. lst.setLen(min(5,lst.len))
lst # lst
for i,leaf in kvpLeafs: # for i,leaf in kvpLeafs:
let rc = db.mergeGenericData leaf # let rc = db.mergeGenericData leaf
xCheckRc rc.error == 0 # xCheckRc rc.error == 0
# List of all leaf entries that should be on the database # # List of all leaf entries that should be on the database
var leafsLeft = kvpLeafs.mapIt(it.leafTie).toHashSet # var leafsLeft = kvpLeafs.mapIt(it.leafTie).toHashSet
# Provide a (reproducible) peudo-random copy of the leafs list # # Provide a (reproducible) peudo-random copy of the leafs list
let leafVidPairs = block: # let leafVidPairs = block:
let rc = db.randomisedLeafs(leafsLeft, prng) # let rc = db.randomisedLeafs(leafsLeft, prng)
xCheckRc rc.error == (0,0) # xCheckRc rc.error == (0,0)
rc.value # rc.value
discard leafVidPairs # discard leafVidPairs
# === delete sub-tree === # # === delete sub-tree ===
block: # block:
let saveBeOk = tx.saveToBackend(relax=false, noisy=noisy, 1+list.len*n) # let saveBeOk = tx.saveToBackend(relax=false, noisy=noisy, 1+list.len*n)
xCheck saveBeOk: # xCheck saveBeOk:
noisy.say "***", "del(1)", # noisy.say "***", "del(1)",
" n=", n, "/", list.len, # " n=", n, "/", list.len,
"\n db\n ", db.pp(backendOk=true), # "\n db\n ", db.pp(backendOk=true),
"" # ""
# Delete sub-tree # # Delete sub-tree
block: # block:
let rc = db.deleteGenericTree testRootVid # let rc = db.deleteGenericTree testRootVid
xCheckRc rc.error == 0: # xCheckRc rc.error == 0:
noisy.say "***", "del(2)", # noisy.say "***", "del(2)",
" n=", n, "/", list.len, # " n=", n, "/", list.len,
"\n db\n ", db.pp(backendOk=true), # "\n db\n ", db.pp(backendOk=true),
"" # ""
block: # block:
let saveBeOk = tx.saveToBackend(relax=false, noisy=noisy, 2+list.len*n) # let saveBeOk = tx.saveToBackend(relax=false, noisy=noisy, 2+list.len*n)
xCheck saveBeOk: # xCheck saveBeOk:
noisy.say "***", "del(3)", # noisy.say "***", "del(3)",
" n=", n, "/", list.len, # " n=", n, "/", list.len,
"\n db\n ", db.pp(backendOk=true), # "\n db\n ", db.pp(backendOk=true),
"" # ""
when true and false: # when true and false:
noisy.say "***", "del(9) n=", n, "/", list.len, " nLeafs=", kvpLeafs.len # noisy.say "***", "del(9) n=", n, "/", list.len, " nLeafs=", kvpLeafs.len
true true

81
tests/test_tracer_json.nim
View File

@ -67,45 +67,46 @@ proc preLoadAristoDb(cdb: CoreDbRef; jKvp: JsonNode; num: BlockNumber) =
ps.partPut(proof, AutomaticPayload).isOkOr: ps.partPut(proof, AutomaticPayload).isOkOr:
raiseAssert info & ": partPut => " & $error raiseAssert info & ": partPut => " & $error
# Handle transaction sub-tree # TODO code needs updating after removal of generic payloads
if txRoot.isValid: # # Handle transaction sub-tree
var txs: seq[Transaction] # if txRoot.isValid:
for (key,pyl) in adb.rightPairs LeafTie(root: ps.partGetSubTree txRoot): # var txs: seq[Transaction]
let # for (key,pyl) in adb.rightPairs LeafTie(root: ps.partGetSubTree txRoot):
inx = key.path.to(UInt256).truncate(uint) # let
tx = rlp.decode(pyl.rawBlob, Transaction) # inx = key.path.to(UInt256).truncate(uint)
# # tx = rlp.decode(pyl.rawBlob, Transaction)
# FIXME: Is this might be a bug in the test data? # #
# # # FIXME: Is this might be a bug in the test data?
# The single item test key is always `128`. For non-single test # #
# lists, the keys are `1`,`2`, ..,`N`, `128` (some single digit # # The single item test key is always `128`. For non-single test
# number `N`.) # # lists, the keys are `1`,`2`, ..,`N`, `128` (some single digit
# # # number `N`.)
# Unless the `128` item value is put at the start of the argument # #
# list `txs[]` for `persistTransactions()`, the `tracer` module # # Unless the `128` item value is put at the start of the argument
# will throw an exception at # # list `txs[]` for `persistTransactions()`, the `tracer` module
# `doAssert(transactions.calcTxRoot == header.txRoot)` in the # # will throw an exception at
# function `traceTransactionImpl()`. # # `doAssert(transactions.calcTxRoot == header.txRoot)` in the
# # # function `traceTransactionImpl()`.
if (inx and 0x80) != 0: # #
txs = @[tx] & txs # if (inx and 0x80) != 0:
else: # txs = @[tx] & txs
txs.add tx # else:
cdb.persistTransactions(num, txRoot, txs) # txs.add tx
# cdb.persistTransactions(num, txRoot, txs)
# Handle receipts sub-tree # # Handle receipts sub-tree
if rcptRoot.isValid: # if rcptRoot.isValid:
var rcpts: seq[Receipt] # var rcpts: seq[Receipt]
for (key,pyl) in adb.rightPairs LeafTie(root: ps.partGetSubTree rcptRoot): # for (key,pyl) in adb.rightPairs LeafTie(root: ps.partGetSubTree rcptRoot):
let # let
inx = key.path.to(UInt256).truncate(uint) # inx = key.path.to(UInt256).truncate(uint)
rcpt = rlp.decode(pyl.rawBlob, Receipt) # rcpt = rlp.decode(pyl.rawBlob, Receipt)
# FIXME: See comment at `txRoot` section. # # FIXME: See comment at `txRoot` section.
if (inx and 0x80) != 0: # if (inx and 0x80) != 0:
rcpts = @[rcpt] & rcpts # rcpts = @[rcpt] & rcpts
else: # else:
rcpts.add rcpt # rcpts.add rcpt
cdb.persistReceipts(rcptRoot, rcpts) # cdb.persistReceipts(rcptRoot, rcpts)
# Save keys to database # Save keys to database
for (rvid,key) in ps.vkPairs: for (rvid,key) in ps.vkPairs:
@ -121,7 +122,9 @@ proc preLoadAristoDb(cdb: CoreDbRef; jKvp: JsonNode; num: BlockNumber) =
#if true: quit() #if true: quit()
# use tracerTestGen.nim to generate additional test data # use tracerTestGen.nim to generate additional test data
proc testFixtureImpl(node: JsonNode, testStatusIMPL: var TestStatus, memoryDB: CoreDbRef) = proc testFixtureImpl(node: JsonNode, testStatusIMPL: var TestStatus, memoryDB: CoreDbRef) {.deprecated: "needs fixing for non-generic payloads".} =
block:
return
setErrorLevel() setErrorLevel()
var var