status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity
nimbus-eth1/tests/test_aristo/test_misc.nim

518 lines
15 KiB
Nim
Raw Blame History

# Nimbus
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or
# distributed except according to those terms.
## Aristo (aka Patricia) DB trancoder test
import
std/[algorithm, sequtils, sets, strutils],
eth/common,
results,
stew/byteutils,
stew/endians2,
unittest2,
../../nimbus/db/aristo,
../../nimbus/db/aristo/[
aristo_check, aristo_debug, aristo_desc, aristo_blobify, aristo_layers,
aristo_vid],
../../nimbus/db/aristo/aristo_filter/filter_scheduler,
../replay/xcheck,
./test_helpers
type
TesterDesc = object
prng: uint32 ## random state
QValRef = ref object
fid: FilterID
width: uint32
QTabRef = TableRef[QueueID,QValRef]
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
proc posixPrngRand(state: var uint32): byte =
## POSIX.1-2001 example of a rand() implementation, see manual page rand(3).
state = state * 1103515245 + 12345;
let val = (state shr 16) and 32767 # mod 2^31
(val shr 8).byte # Extract second byte
proc rand[W: SomeInteger|VertexID](ap: var TesterDesc; T: type W): T =
var a: array[sizeof T,byte]
for n in 0 ..< sizeof T:
a[n] = ap.prng.posixPrngRand().byte
when sizeof(T) == 1:
let w = uint8.fromBytesBE(a).T
when sizeof(T) == 2:
let w = uint16.fromBytesBE(a).T
when sizeof(T) == 4:
let w = uint32.fromBytesBE(a).T
else:
let w = uint64.fromBytesBE(a).T
when T is SomeUnsignedInt:
# That way, `fromBytesBE()` can be applied to `uint`
result = w
else:
# That way the result is independent of endianness
(addr result).copyMem(unsafeAddr w, sizeof w)
proc vidRand(td: var TesterDesc; bits = 19): VertexID =
if bits < 64:
let
mask = (1u64 shl max(1,bits)) - 1
rval = td.rand uint64
(rval and mask).VertexID
else:
td.rand VertexID
proc init(T: type TesterDesc; seed: int): TesterDesc =
result.prng = (seed and 0x7fffffff).uint32
proc `+`(a: VertexID, b: int): VertexID =
(a.uint64 + b.uint64).VertexID
# ---------------------
iterator walkFifo(qt: QTabRef;scd: QidSchedRef): (QueueID,QValRef) =
## ...
proc kvp(chn: int, qid: QueueID): (QueueID,QValRef) =
let cid = QueueID((chn.uint64 shl 62) or qid.uint64)
(cid, qt.getOrDefault(cid, QValRef(nil)))
if not scd.isNil:
for i in 0 ..< scd.state.len:
let (left, right) = scd.state[i]
if left == 0:
discard
elif left <= right:
for j in right.countDown left:
yield kvp(i, j)
else:
for j in right.countDown QueueID(1):
yield kvp(i, j)
for j in scd.ctx.q[i].wrap.countDown left:
yield kvp(i, j)
proc fifos(qt: QTabRef; scd: QidSchedRef): seq[seq[(QueueID,QValRef)]] =
## ..
var lastChn = -1
for (qid,val) in qt.walkFifo scd:
let chn = (qid.uint64 shr 62).int
while lastChn < chn:
lastChn.inc
result.add newSeq[(QueueID,QValRef)](0)
result[^1].add (qid,val)
func sortedPairs(qt: QTabRef): seq[(QueueID,QValRef)] =
qt.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.QueueID).mapIt((it,qt[it]))
func flatten(a: seq[seq[(QueueID,QValRef)]]): seq[(QueueID,QValRef)] =
for w in a:
result &= w
func pp(val: QValRef): string =
if val.isNil:
return "ø"
result = $val.fid.uint64
if 0 < val.width:
result &= ":" & $val.width
func pp(kvp: (QueueID,QValRef)): string =
kvp[0].pp & "=" & kvp[1].pp
func pp(qt: QTabRef): string =
"{" & qt.sortedPairs.mapIt(it.pp).join(",") & "}"
func pp(qt: QTabRef; scd: QidSchedRef): string =
result = "["
for w in qt.fifos scd:
if w.len == 0:
result &= "ø"
else:
result &= w.mapIt(it.pp).join(",")
result &= ","
if result[^1] == ',':
result[^1] = ']'
else:
result &= "]"
# ------------------
proc exec(db: QTabRef; serial: int; instr: seq[QidAction]; relax: bool): bool =
## ..
var
saved: bool
hold: seq[(QueueID,QueueID)]
for act in instr:
case act.op:
of Oops:
xCheck act.op != Oops
of SaveQid:
xCheck not saved
db[act.qid] = QValRef(fid: FilterID(serial))
saved = true
of DelQid:
let val = db.getOrDefault(act.qid, QValRef(nil))
xCheck not val.isNil
db.del act.qid
of HoldQid:
hold.add (act.qid, act.xid)
of DequQid:
var merged = QValRef(nil)
for w in hold:
for qid in w[0] .. w[1]:
let val = db.getOrDefault(qid, QValRef(nil))
if not relax:
xCheck not val.isNil
if not val.isNil:
if merged.isNil:
merged = val
else:
if relax:
xCheck merged.fid + merged.width + 1 <= val.fid
else:
xCheck merged.fid + merged.width + 1 == val.fid
merged.width += val.width + 1
db.del qid
if not relax:
xCheck not merged.isNil
if not merged.isNil:
db[act.qid] = merged
hold.setLen(0)
xCheck saved
xCheck hold.len == 0
true
proc validate(db: QTabRef; scd: QidSchedRef; serial: int; relax: bool): bool =
## Verify that the round-robin queues in `db` are consecutive and in the
## right order.
var
step = 1u
lastVal = FilterID(serial+1)
for chn,queue in db.fifos scd:
step *= scd.ctx.q[chn].width + 1 # defined by schedule layout
for kvp in queue:
let (qid,val) = (kvp[0], kvp[1])
if not relax:
xCheck not val.isNil # Entries must exist
xCheck val.fid + step == lastVal # Item distances must match
if not val.isNil:
xCheck val.fid + step <= lastVal # Item distances must decrease
xCheck val.width + 1 == step # Must correspond to `step` size
lastVal = val.fid
# Compare database against expected fill state
if relax:
xCheck db.len <= scd.len
else:
xCheck db.len == scd.len
proc qFn(qid: QueueID): FilterID =
let val = db.getOrDefault(qid, QValRef(nil))
if not val.isNil:
return val.fid
# Test filter ID selection
var lastFid = FilterID(serial + 1)
xCheck scd.le(lastFid + 0, qFn) == scd[0] # Test fringe condition
xCheck scd.le(lastFid + 1, qFn) == scd[0] # Test fringe condition
for (qid,val) in db.fifos(scd).flatten:
xCheck scd.eq(val.fid, qFn) == qid
xCheck scd.le(val.fid, qFn) == qid
for w in val.fid+1 ..< lastFid:
xCheck scd.le(w, qFn) == qid
xCheck scd.eq(w, qFn) == QueueID(0)
lastFid = val.fid
if FilterID(1) < lastFid: # Test fringe condition
xCheck scd.le(lastFid - 1, qFn) == QueueID(0)
if FilterID(2) < lastFid: # Test fringe condition
xCheck scd.le(lastFid - 2, qFn) == QueueID(0)
true
# ------------------------------------------------------------------------------
# Public test function
# ------------------------------------------------------------------------------
proc testVidRecycleLists*(noisy = true; seed = 42): bool =
## Transcode VID lists held in `AristoDb` descriptor
##
var td = TesterDesc.init seed
let db = AristoDbRef.init()
# Add some randum numbers
block:
let first = td.vidRand()
db.vidDispose first
var
expectedVids = 1
count = 1
# Feed some numbers used and some discaded
while expectedVids < 5 or count < 5 + expectedVids:
count.inc
let vid = td.vidRand()
expectedVids += (vid < first).ord
db.vidDispose vid
xCheck db.vGen.len == expectedVids
noisy.say "***", "vids=", db.vGen.len, " discarded=", count-expectedVids
# Serialise/deserialise
block:
let dbBlob = db.vGen.blobify
# Deserialise
let
db1 = AristoDbRef.init()
rc = dbBlob.deblobify seq[VertexID]
xCheckRc rc.error == 0
db1.top.final.vGen = rc.value
xCheck db.vGen == db1.vGen
# Make sure that recycled numbers are fetched first
let topVid = db.vGen[^1]
while 1 < db.vGen.len:
let w = db.vidFetch()
xCheck w < topVid
xCheck db.vGen.len == 1 and db.vGen[0] == topVid
# Get some consecutive vertex IDs
for n in 0 .. 5:
let w = db.vidFetch()
xCheck w == topVid + n
xCheck db.vGen.len == 1
# Repeat last test after clearing the cache
db.top.final.vGen.setLen(0)
for n in 0 .. 5:
let w = db.vidFetch()
xCheck w == VertexID(2) + n # VertexID(1) is default root ID
xCheck db.vGen.len == 1
# Recycling and re-org tests
func toVQ(a: seq[int]): seq[VertexID] = a.mapIt(VertexID(it))
xCheck @[8, 7, 3, 4, 5, 9] .toVQ.vidReorg == @[3, 4, 5, 7] .toVQ
xCheck @[8, 7, 6, 3, 4, 5, 9] .toVQ.vidReorg == @[3] .toVQ
xCheck @[5, 4, 3, 7] .toVQ.vidReorg == @[5, 4, 3, 7] .toVQ
xCheck @[5] .toVQ.vidReorg == @[5] .toVQ
xCheck @[3, 5] .toVQ.vidReorg == @[3, 5] .toVQ
xCheck @[4, 5] .toVQ.vidReorg == @[4] .toVQ
xCheck newSeq[VertexID](0).vidReorg().len == 0
true
proc testQidScheduler*(
noisy = true;
layout = QidSlotLyo;
sampleSize = QidSample;
reorgPercent = 40
): bool =
##
## Example table for `QidSlotLyo` layout after 10_000 cycles
## ::
## QueueID | QValRef |
## | FilterID | width | comment
## --------+----------+-------+----------------------------------
## %a | 10000 | 0 | %a stands for QueueID(10)
## %9 | 9999 | 0 |
## %8 | 9998 | 0 |
## %7 | 9997 | 0 |
## | | |
## %1:9 | 9993 | 3 | 9993 + 3 + 1 => 9997, see %7
## %1:8 | 9989 | 3 |
## %1:7 | 9985 | 3 |
## %1:6 | 9981 | 3 | %1:6 stands for QueueID((1 shl 62) + 6)
## | | |
## %2:9 | 9961 | 19 | 9961 + 19 + 1 => 9981, see %1:6
## %2:8 | 9941 | 19 |
## %2:7 | 9921 | 19 |
## %2:6 | 9901 | 19 |
## %2:5 | 9881 | 19 |
## %2:4 | 9861 | 19 |
## %2:3 | 9841 | 19 |
## | | |
## %3:2 | 9721 | 119 | 9721 + 119 + 1 => 9871, see %2:3
## %3:1 | 9601 | 119 |
## %3:a | 9481 | 119 |
##
var
debug = false # or true
let
list = newTable[QueueID,QValRef]()
scd = QidSchedRef.init layout
ctx = scd.ctx.q
proc show(serial = 0; exec: seq[QidAction] = @[]) =
var s = ""
if 0 < serial:
s &= "n=" & $serial
if 0 < exec.len:
s &= " exec=" & exec.pp
s &= "" &
"\n state=" & scd.state.pp &
"\n list=" & list.pp &
"\n fifo=" & list.pp(scd) &
"\n"
noisy.say "***", s
if debug:
noisy.say "***", "sampleSize=", sampleSize,
" ctx=", ctx, " stats=", scd.ctx.stats
for n in 1 .. sampleSize:
let w = scd.addItem()
let execOk = list.exec(serial=n, instr=w.exec, relax=false)
xCheck execOk
scd[] = w.fifo[]
let validateOk = list.validate(scd, serial=n, relax=false)
xCheck validateOk:
show(serial=n, exec=w.exec)
let fifoID = list.fifos(scd).flatten.mapIt(it[0])
for j in 0 ..< list.len:
# Check fifo order
xCheck fifoID[j] == scd[j]:
noisy.say "***", "n=", n, " exec=", w.exec.pp,
" fifoID[", j, "]=", fifoID[j].pp,
" scd[", j, "]=", scd[j].pp,
"\n fifo=", list.pp scd
# Check random access and reverse
let qid = scd[j]
xCheck j == scd[qid]
if debug:
show(exec=w.exec)
# -------------------
# Mark deleted some entries from database
var
nDel = (list.len * reorgPercent) div 100
delIDs: HashSet[QueueID]
for n in 0 ..< nDel:
delIDs.incl scd[n]
# Delete these entries
let fetch = scd.fetchItems nDel
for act in fetch.exec:
xCheck act.op == HoldQid
for qid in act.qid .. act.xid:
xCheck qid in delIDs
xCheck list.hasKey qid
delIDs.excl qid
list.del qid
xCheck delIDs.len == 0
scd[] = fetch.fifo[]
# -------------------
# Continue adding items
for n in sampleSize + 1 .. 2 * sampleSize:
let w = scd.addItem()
let execOk = list.exec(serial=n, instr=w.exec, relax=true)
xCheck execOk
scd[] = w.fifo[]
let validateOk = list.validate(scd, serial=n, relax=true)
xCheck validateOk:
show(serial=n, exec=w.exec)
# Continue adding items, now strictly
for n in 2 * sampleSize + 1 .. 3 * sampleSize:
let w = scd.addItem()
let execOk = list.exec(serial=n, instr=w.exec, relax=false)
xCheck execOk
scd[] = w.fifo[]
let validateOk = list.validate(scd, serial=n, relax=false)
xCheck validateOk
if debug:
show()
true
proc testShortKeys*(
noisy = true;
): bool =
## Check for some pathological cases
func x(s: string): Blob = s.hexToSeqByte
func k(s: string): HashKey = HashKey.fromBytes(s.x).value
let samples = [
# From InvalidBlocks/bc4895-withdrawals/twoIdenticalIndex.json
[("80".x,
"da808094c94f5374fce5edbc8e2a8697c15331677e6ebf0b822710".x,
"27f166f1d7c789251299535cb176ba34116e44894476a7886fe5d73d9be5c973".k),
("01".x,
"da028094c94f5374fce5edbc8e2a8697c15331677e6ebf0b822710".x,
"81eac5f476f48feb289af40ee764015f6b49036760438ea45df90d5342b6ae61".k),
("02".x,
"da018094c94f5374fce5edbc8e2a8697c15331677e6ebf0b822710".x,
"463769ae507fcc6d6231c8888425191c5622f330fdd4b78a7b24c4521137b573".k),
("03".x,
"da028094c94f5374fce5edbc8e2a8697c15331677e6ebf0b822710".x,
"a95b9a7b58a6b3cb4001eb0be67951c5517141cb0183a255b5cae027a7b10b36".k)]]
let gossip = false # or noisy
for n,sample in samples:
let sig = merkleSignBegin()
var inx = -1
for (k,v,r) in sample:
inx.inc
sig.merkleSignAdd(k,v)
gossip.say "*** testShortkeys (1)", "n=", n, " inx=", inx,
"\n k=", k.toHex, " v=", v.toHex,
"\n r=", r.pp(sig),
"\n ", sig.pp(),
"\n",
"\n pAmk=", sig.db.layersWalkLebal.toSeq.toTable.pp(sig.db),
"\n"
let w = sig.merkleSignCommit().value
gossip.say "*** testShortkeys (2)", "n=", n, " inx=", inx,
"\n k=", k.toHex, " v=", v.toHex,
"\n r=", r.pp(sig),
"\n R=", w.pp(sig),
"\n ", sig.pp(),
"\n",
"\n pAmk=", sig.db.layersWalkLebal.toSeq.toTable.pp(sig.db),
"\n",
"\n ----------------",
"\n"
let rc = sig.db.check
xCheckRc rc.error == (0,0)
xCheck r == w
true
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------
Reference in New Issue View Git Blame Copy Permalink