status-im/nimbus-eth1
2
0
Fork 0
mirror of https://github.com/status-im/nimbus-eth1.git synced 2025-01-12 05:14:14 +00:00
Code Issues Projects Releases Wiki Activity

Snap sync accounts db code reorg (#1189)

Browse Source 
* Extracted functionality into sub-modules for maintainability

* Setting SST bulk load as default in `accounts_db`

details:
+ currently, the same data are stored via rocksdb if available, and
  the same via embedded `storage_type` with (non-standard) prefix 200
  for time comparisons
+ fallback to normal `put()` unless rocksdb is accessible
This commit is contained in:
Jordan Hrycaj 2022-08-15 16:51:50 +01:00 committed by GitHub
parent 7d7e26d45f
commit 7489784ba8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 1446 additions and 1219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1260
nimbus/sync/snap/worker/accounts_db.nim
View File

File diff suppressed because it is too large Load Diff

177
nimbus/sync/snap/worker/db/bulk_storage.nim Normal file
View File

@ -0,0 +1,177 @@
# nimbus-eth1
# Copyright (c) 2021 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
import
std/[algorithm, strutils, tables],
chronicles,
eth/[common/eth_types, trie/db],
../../../../db/[kvstore_rocksdb, storage_types],
../../../types,
../../range_desc,
"."/[hexary_defs, hexary_desc, rocky_bulk_load]
{.push raises: [Defect].}
logScope:
topics = "snap-db"
type
BulkStorageDbXKeyKind = enum
## Extends `storage_types.DbDBKeyKind` for testing/debugging
ChainDbHexaryPfx = 200 # <hash-key> on trie db layer
const
RockyBulkCache = "accounts.sst"
static:
# Make sure that `DBKeyKind` extension does not overlap
doAssert high(DBKeyKind).int < low(BulkStorageDbXKeyKind).int
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
proc to(tag: NodeTag; T: type RepairKey): T =
tag.to(NodeKey).to(RepairKey)
proc convertTo(key: RepairKey; T: type NodeKey): T =
if key.isNodeKey:
discard result.init(key.ByteArray33[1 .. 32])
proc convertTo(key: RepairKey; T: type NodeTag): T =
if key.isNodeKey:
result = UInt256.fromBytesBE(key.ByteArray33[1 .. 32]).T
# ------------------------------------------------------------------------------
# Private helpers for bulk load testing
# ------------------------------------------------------------------------------
proc chainDbHexaryKey(a: RepairKey): ByteArray32 =
a.convertTo(NodeKey).ByteArray32
proc chainDbHexaryKey(a: NodeTag): ByteArray32 =
a.to(NodeKey).ByteArray32
proc bulkStorageChainDbHexaryXKey*(a: RepairKey): DbKey =
result.data[0] = byte ord(ChainDbHexaryPfx)
result.data[1 .. 32] = a.convertTo(NodeKey).ByteArray32
result.dataEndPos = uint8 32
template toOpenArray*(k: ByteArray32): openArray[byte] =
k.toOpenArray(0, 31)
# ------------------------------------------------------------------------------
# Public helperd
# ------------------------------------------------------------------------------
proc bulkStorageChainDbHexaryXKey*(a: NodeTag): DbKey =
result.data[0] = byte ord(ChainDbHexaryPfx)
result.data[1 .. 32] = a.to(NodeKey).ByteArray32
result.dataEndPos = uint8 32
proc bulkStorageClearRockyCacheFile*(rocky: RocksStoreRef): bool =
if not rocky.isNil:
# A cache file might hang about from a previous crash
try:
discard rocky.clearCacheFile(RockyBulkCache)
return true
except OSError as e:
error "Cannot clear rocksdb cache", exception=($e.name), msg=e.msg
# ------------------------------------------------------------------------------
# Public bulk store examples
# ------------------------------------------------------------------------------
proc bulkStorageHexaryNodesOnChainDb*(
db: HexaryTreeDB;
base: TrieDatabaseRef
): Result[void,HexaryDbError] =
## Bulk load using transactional `put()`
let dbTx = base.beginTransaction
defer: dbTx.commit
for (key,value) in db.tab.pairs:
if not key.isNodeKey:
let error = UnresolvedRepairNode
trace "Unresolved node in repair table", error
return err(error)
base.put(key.chainDbHexaryKey.toOpenArray, value.convertTo(Blob))
ok()
proc bulkStorageHexaryNodesOnXChainDb*(
db: HexaryTreeDB;
base: TrieDatabaseRef
): Result[void,HexaryDbError] =
## Bulk load using transactional `put()` on a sub-table
let dbTx = base.beginTransaction
defer: dbTx.commit
for (key,value) in db.tab.pairs:
if not key.isNodeKey:
let error = UnresolvedRepairNode
trace "Unresolved node in repair table", error
return err(error)
base.put(
key.bulkStorageChainDbHexaryXKey.toOpenArray, value.convertTo(Blob))
ok()
proc bulkStorageHexaryNodesOnRockyDb*(
db: HexaryTreeDB;
rocky: RocksStoreRef
): Result[void,HexaryDbError]
{.gcsafe, raises: [Defect,OSError,KeyError,ValueError].} =
## SST based bulk load on `rocksdb`.
if rocky.isNil:
return err(NoRocksDbBackend)
let bulker = RockyBulkLoadRef.init(rocky)
defer: bulker.destroy()
if not bulker.begin(RockyBulkCache):
let error = CannotOpenRocksDbBulkSession
trace "Rocky hexary session initiation failed",
error, info=bulker.lastError()
return err(error)
#let keyList = toSeq(db.tab.keys)
# .filterIt(it.isNodeKey)
# .mapIt(it.convertTo(NodeTag))
# .sorted(cmp)
var
keyList = newSeq[NodeTag](db.tab.len)
inx = 0
for repairKey in db.tab.keys:
if repairKey.isNodeKey:
keyList[inx] = repairKey.convertTo(NodeTag)
inx.inc
if inx < db.tab.len:
return err(UnresolvedRepairNode)
keyList.sort(cmp)
for n,nodeTag in keyList:
let
key = nodeTag.chainDbHexaryKey()
data = db.tab[nodeTag.to(RepairKey)].convertTo(Blob)
if not bulker.add(key.toOpenArray, data):
let error = AddBulkItemFailed
trace "Rocky hexary bulk load failure",
n, len=db.tab.len, error, info=bulker.lastError()
return err(error)
if bulker.finish().isErr:
let error = CommitBulkItemsFailed
trace "Rocky hexary commit failure",
len=db.acc.len, error, info=bulker.lastError()
return err(error)
ok()
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

36
nimbus/sync/snap/worker/db/hexary_defs.nim Normal file
View File

@ -0,0 +1,36 @@
# nimbus-eth1
# Copyright (c) 2021 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
type
HexaryDbError* = enum
NothingSerious = 0
AccountNotFound
AccountSmallerThanBase
AccountsNotSrictlyIncreasing
AccountRangesOverlap
AccountRepairBlocked
BoundaryProofFailed
Rlp2Or17ListEntries
RlpBlobExpected
RlpBranchLinkExpected
RlpEncoding
RlpExtPathEncoding
RlpNonEmptyBlobExpected
# bulk storage
AddBulkItemFailed
CannotOpenRocksDbBulkSession
CommitBulkItemsFailed
NoRocksDbBackend
UnresolvedRepairNode
# End

282
nimbus/sync/snap/worker/db/hexary_desc.nim Normal file
View File

@ -0,0 +1,282 @@
# nimbus-eth1
# Copyright (c) 2021 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
import
std/[hashes, sequtils, strformat, strutils, tables],
eth/[common/eth_types, p2p, trie/nibbles],
nimcrypto/keccak,
stint,
../../range_desc
{.push raises: [Defect].}
type
HexaryPpFn* = proc(key: RepairKey): string {.gcsafe.}
## For testing/debugging: key pretty printer function
ByteArray32* = array[32,byte]
## Used for 32 byte database keys
ByteArray33* = array[33,byte]
## Used for 31 byte database keys, i.e. <marker> + <32-byte-key>
NodeKey* = distinct ByteArray32
## Hash key without the hash wrapper
RepairKey* = distinct ByteArray33
## Byte prefixed `NodeKey` for internal DB records
RNodeKind* = enum
Branch
Extension
Leaf
RNodeState* = enum
Static = 0 ## Inserted as proof record
Locked ## Like `Static`, only added on-the-fly
Mutable ## Open for modification
# Example trie from https://eth.wiki/en/fundamentals/patricia-tree
#
# lookup data:
# "do": "verb"
# "dog": "puppy"
# "dodge": "coin"
# "horse": "stallion"
#
# trie DB:
# root: [16 A]
# A: [* * * * B * * * [20+"orse" "stallion"] * * * * * * * *]
# B: [00+"o" D]
# D: [* * * * * * E * * * * * * * * * "verb"]
# E: [17 [* * * * * * [35 "coin"] * * * * * * * * * "puppy"]]
#
# with first nibble of two-column rows:
# hex bits | node type length
# ---------+------------------
# 0 0000 | extension even
# 1 0001 | extension odd
# 2 0010 | leaf even
# 3 0011 | leaf odd
#
# and key path:
# "do": 6 4 6 f
# "dog": 6 4 6 f 6 7
# "dodge": 6 4 6 f 6 7 6 5
# "horse": 6 8 6 f 7 2 7 3 6 5
RNodeRef* = ref object
## For building a temporary repair tree
state*: RNodeState ## `Static` if added as proof data
case kind*: RNodeKind
of Leaf:
lPfx*: NibblesSeq ## Portion of path segment
lData*: Blob
of Extension:
ePfx*: NibblesSeq ## Portion of path segment
eLink*: RepairKey ## Single down link
of Branch:
bLink*: array[16,RepairKey] ## Down links
bData*: Blob
RPathStep* = object
## For constructing tree traversal `seq[RPathStep]` path
key*: RepairKey ## Tree label, node hash
node*: RNodeRef ## Referes to data record
nibble*: int8 ## Branch node selector (if any)
RPathXStep* = object
## Extended `RPathStep` needed for `NodeKey` assignmant
pos*: int ## Some position into `seq[RPathStep]`
step*: RPathStep ## Modified copy of an `RPathStep`
canLock*: bool ## Can set `Locked` state
RPath* = object
path*: seq[RPathStep]
tail*: NibblesSeq ## Stands for non completed leaf path
RLeafSpecs* = object
## Temporarily stashed leaf data (as for an account.) Proper records
## have non-empty payload. Records with empty payload are administrative
## items, e.g. lower boundary records.
pathTag*: NodeTag ## Equivalent to account hash
nodeKey*: RepairKey ## Leaf hash into hexary repair table
payload*: Blob ## Data payload
HexaryTreeDB* = object
rootKey*: NodeKey ## Current root node
tab*: Table[RepairKey,RNodeRef] ## Repair table
acc*: seq[RLeafSpecs] ## Accounts to appprove of
repairKeyGen*: uint64 ## Unique tmp key generator
keyPp*: HexaryPpFn ## For debugging
const
EmptyNodeBlob* = seq[byte].default
static:
# Not that there is no doubt about this ...
doAssert NodeKey.default.ByteArray32.initNibbleRange.len == 64
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
proc pp(key: RepairKey): string =
key.ByteArray33.toSeq.mapIt(it.toHex(2)).join.toLowerAscii
# ------------------------------------------------------------------------------
# Public debugging helpers
# ------------------------------------------------------------------------------
proc pp*(s: string; hex = false): string =
## For long strings print `begin..end` only
if hex:
let n = (s.len + 1) div 2
(if s.len < 20: s else: s[0 .. 5] & ".." & s[s.len-8 .. s.len-1]) &
"[" & (if 0 < n: "#" & $n else: "") & "]"
elif s.len <= 30:
s
else:
(if (s.len and 1) == 0: s[0 ..< 8] else: "0" & s[0 ..< 7]) &
"..(" & $s.len & ").." & s[s.len-16 ..< s.len]
proc pp*(w: NibblesSeq): string =
$w
proc pp*(key: RepairKey; db: HexaryTreeDB): string =
try:
if not db.keyPp.isNil:
return db.keyPp(key)
except:
discard
key.pp
proc pp*(w: openArray[RepairKey]; db: HexaryTreeDB): string =
"<" & w.mapIt(it.pp(db)).join(",") & ">"
proc pp*(n: RNodeRef; db: HexaryTreeDB): string
{.gcsafe, raises: [Defect, ValueError].} =
proc ppStr(blob: Blob): string =
if blob.len == 0: ""
else: blob.mapIt(it.toHex(2)).join.toLowerAscii.pp(hex = true)
let so = n.state.ord
case n.kind:
of Leaf:
result = ["l","ł","L"][so] & &"({n.lPfx.pp},{n.lData.ppStr})"
of Extension:
result = ["e","€","E"][so] & &"({n.ePfx.pp},{n.eLink.pp(db)})"
of Branch:
result = ["b","þ","B"][so] & &"({n.bLink.pp(db)},{n.bData.ppStr})"
# ------------------------------------------------------------------------------
# Public constructor (or similar)
# ------------------------------------------------------------------------------
proc init*(key: var NodeKey; data: openArray[byte]): bool =
key.reset
if data.len <= 32:
if 0 < data.len:
let trg = addr key.ByteArray32[32 - data.len]
trg.copyMem(unsafeAddr data[0], data.len)
return true
proc init*(key: var RepairKey; data: openArray[byte]): bool =
key.reset
if data.len <= 33:
if 0 < data.len:
let trg = addr key.ByteArray33[33 - data.len]
trg.copyMem(unsafeAddr data[0], data.len)
return true
proc newRepairKey*(db: var HexaryTreeDB): RepairKey =
db.repairKeyGen.inc
var src = db.repairKeyGen.toBytesBE
(addr result.ByteArray33[25]).copyMem(addr src[0], 8)
result.ByteArray33[0] = 1
# ------------------------------------------------------------------------------
# Public functions
# ------------------------------------------------------------------------------
proc hash*(a: NodeKey): Hash =
## Tables mixin
a.ByteArray32.hash
proc hash*(a: RepairKey): Hash =
## Tables mixin
a.ByteArray33.hash
proc `==`*(a, b: NodeKey): bool =
## Tables mixin
a.ByteArray32 == b.ByteArray32
proc `==`*(a, b: RepairKey): bool =
## Tables mixin
a.ByteArray33 == b.ByteArray33
proc to*(tag: NodeTag; T: type NodeKey): T =
tag.UInt256.toBytesBE.T
proc to*(key: NodeKey; T: type NibblesSeq): T =
key.ByteArray32.initNibbleRange
proc to*(key: NodeKey; T: type RepairKey): T =
(addr result.ByteArray33[1]).copyMem(unsafeAddr key.ByteArray32[0], 32)
proc isZero*[T: NodeTag|NodeKey|RepairKey](a: T): bool =
a == T.default
proc isNodeKey*(a: RepairKey): bool =
a.ByteArray33[0] == 0
proc digestTo*(data: Blob; T: type NodeKey): T =
keccak256.digest(data).data.T
proc convertTo*[W: NodeKey|RepairKey](data: openArray[byte]; T: type W): T =
## Probably lossy conversion, use `init()` for safe conversion
discard result.init(data)
proc convertTo*(node: RNodeRef; T: type Blob): T =
## Write the node as an RLP-encoded blob
var writer = initRlpWriter()
proc appendOk(writer: var RlpWriter; key: RepairKey): bool =
if key.isZero:
writer.append(EmptyNodeBlob)
elif key.isNodeKey:
var hash: Hash256
(addr hash.data[0]).copyMem(unsafeAddr key.ByteArray33[1], 32)
writer.append(hash)
else:
return false
true
case node.kind:
of Branch:
writer.startList(17)
for n in 0 ..< 16:
if not writer.appendOk(node.bLink[n]):
return # empty `Blob`
writer.append(node.bData)
of Extension:
writer.startList(2)
writer.append(node.ePfx.hexPrefixEncode(isleaf = false))
if not writer.appendOk(node.eLink):
return # empty `Blob`
of Leaf:
writer.startList(2)
writer.append(node.lPfx.hexPrefixEncode(isleaf = true))
writer.append(node.lData)
writer.finish()
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

146
nimbus/sync/snap/worker/db/hexary_follow.nim Normal file
View File

@ -0,0 +1,146 @@
# nimbus-eth1
# Copyright (c) 2021 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
## This module is sort of a customised rewrite of the function
## `eth/trie/hexary.getAux()`, `getkeysAux()`, etc.
import
std/sequtils,
chronicles,
eth/[common/eth_types, trie/nibbles],
./hexary_desc
{.push raises: [Defect].}
const
HexaryFollowDebugging = false or true
type
HexaryGetFn* = proc(key: Blob): Blob {.gcsafe.}
## Fortesting/debugging: database get() function
# ------------------------------------------------------------------------------
# Public walk along hexary trie records
# ------------------------------------------------------------------------------
proc hexaryFollow*(
db: HexaryTreeDB;
root: NodeKey;
path: NibblesSeq;
getFn: HexaryGetFn
): (int, bool, Blob)
{.gcsafe, raises: [Defect,RlpError]} =
## Returns the number of matching digits/nibbles from the argument `path`
## found in the proofs trie.
let
nNibbles = path.len
var
inPath = path
recKey = root.ByteArray32.toSeq
leafBlob: Blob
emptyRef = false
when HexaryFollowDebugging:
trace "follow", rootKey=root.to(RepairKey).pp(db), path
while true:
let value = recKey.getFn()
if value.len == 0:
break
var nodeRlp = rlpFromBytes value
case nodeRlp.listLen:
of 2:
let
(isLeaf, pathSegment) = hexPrefixDecode nodeRlp.listElem(0).toBytes
sharedNibbles = inPath.sharedPrefixLen(pathSegment)
fullPath = sharedNibbles == pathSegment.len
inPathLen = inPath.len
inPath = inPath.slice(sharedNibbles)
# Leaf node
if isLeaf:
let leafMode = sharedNibbles == inPathLen
if fullPath and leafMode:
leafBlob = nodeRlp.listElem(1).toBytes
when HexaryFollowDebugging:
let nibblesLeft = inPathLen - sharedNibbles
trace "follow leaf",
fullPath, leafMode, sharedNibbles, nibblesLeft,
pathSegment, newPath=inPath
break
# Extension node
if fullPath:
let branch = nodeRlp.listElem(1)
if branch.isEmpty:
when HexaryFollowDebugging:
trace "follow extension", newKey="n/a"
emptyRef = true
break
recKey = branch.toBytes
when HexaryFollowDebugging:
trace "follow extension",
newKey=recKey.convertTo(RepairKey).pp(db), newPath=inPath
else:
when HexaryFollowDebugging:
trace "follow extension",
fullPath, sharedNibbles, pathSegment, inPathLen, newPath=inPath
break
of 17:
# Branch node
if inPath.len == 0:
leafBlob = nodeRlp.listElem(1).toBytes
break
let
inx = inPath[0].int
branch = nodeRlp.listElem(inx)
if branch.isEmpty:
when HexaryFollowDebugging:
trace "follow branch", newKey="n/a"
emptyRef = true
break
inPath = inPath.slice(1)
recKey = branch.toBytes
when HexaryFollowDebugging:
trace "follow branch",
newKey=recKey.convertTo(RepairKey).pp(db), inx, newPath=inPath
else:
when HexaryFollowDebugging:
trace "follow oops",
nColumns = nodeRlp.listLen
break
# end while
let pathLen = nNibbles - inPath.len
when HexaryFollowDebugging:
trace "follow done",
recKey, emptyRef, pathLen, leafSize=leafBlob.len
(pathLen, emptyRef, leafBlob)
proc hexaryFollow*(
db: HexaryTreeDB;
root: NodeKey;
path: NodeKey;
getFn: HexaryGetFn;
): (int, bool, Blob)
{.gcsafe, raises: [Defect,RlpError]} =
## Variant of `hexaryFollow()`
db.hexaryFollow(root, path.to(NibblesSeq), getFn)
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

127
nimbus/sync/snap/worker/db/hexary_import.nim Normal file
View File

@ -0,0 +1,127 @@
# nimbus-eth1
# Copyright (c) 2021 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
import
std/[sequtils, strutils, tables],
eth/[common/eth_types, trie/nibbles],
stew/results,
"."/[hexary_defs, hexary_desc]
{.push raises: [Defect].}
const
HexaryImportDebugging = false or true
# ------------------------------------------------------------------------------
# Private debugging helpers
# ------------------------------------------------------------------------------
proc pp(q: openArray[byte]): string =
q.toSeq.mapIt(it.toHex(2)).join.toLowerAscii.pp(hex = true)
# ------------------------------------------------------------------------------
# Public
# ------------------------------------------------------------------------------
proc hexaryImport*(
db: var HexaryTreeDB;
recData: Blob
): Result[void,HexaryDbError]
{.gcsafe, raises: [Defect, RlpError].} =
## Decode a single trie item for adding to the table and add it to the
## database. Branch and exrension record links are collected.
let
nodeKey = recData.digestTo(NodeKey)
repairKey = nodeKey.to(RepairKey) # for repair table
var
rlp = recData.rlpFromBytes
blobs = newSeq[Blob](2) # temporary, cache
links: array[16,RepairKey] # reconstruct branch node
blob16: Blob # reconstruct branch node
top = 0 # count entries
rNode: RNodeRef # repair tree node
# Collect lists of either 2 or 17 blob entries.
for w in rlp.items:
case top
of 0, 1:
if not w.isBlob:
return err(RlpBlobExpected)
blobs[top] = rlp.read(Blob)
of 2 .. 15:
var key: NodeKey
if not key.init(rlp.read(Blob)):
return err(RlpBranchLinkExpected)
# Update ref pool
links[top] = key.to(RepairKey)
of 16:
if not w.isBlob:
return err(RlpBlobExpected)
blob16 = rlp.read(Blob)
else:
return err(Rlp2Or17ListEntries)
top.inc
# Verify extension data
case top
of 2:
if blobs[0].len == 0:
return err(RlpNonEmptyBlobExpected)
let (isLeaf, pathSegment) = hexPrefixDecode blobs[0]
if isLeaf:
rNode = RNodeRef(
kind: Leaf,
lPfx: pathSegment,
lData: blobs[1])
else:
var key: NodeKey
if not key.init(blobs[1]):
return err(RlpExtPathEncoding)
# Update ref pool
rNode = RNodeRef(
kind: Extension,
ePfx: pathSegment,
eLink: key.to(RepairKey))
of 17:
for n in [0,1]:
var key: NodeKey
if not key.init(blobs[n]):
return err(RlpBranchLinkExpected)
# Update ref pool
links[n] = key.to(RepairKey)
rNode = RNodeRef(
kind: Branch,
bLink: links,
bData: blob16)
else:
discard
# Add to repair database
db.tab[repairKey] = rNode
# Add to hexary trie database -- disabled, using bulk import later
#ps.base.db.put(nodeKey.ByteArray32, recData)
when HexaryImportDebugging:
# Rebuild blob from repair record
let nodeBlob = rNode.convertTo(Blob)
if nodeBlob != recData:
echo "*** hexaryImport oops:",
" kind=", rNode.kind,
" key=", repairKey.pp(db),
" nodeBlob=", nodeBlob.pp,
" recData=", recData.pp
doAssert nodeBlob == recData
ok()
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

528
nimbus/sync/snap/worker/db/hexary_interpolate.nim Normal file
View File

@ -0,0 +1,528 @@
# nimbus-eth1
# Copyright (c) 2021 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed
# except according to those terms.
## For a given path, sdd missing nodes to a hexary trie.
##
## This module function is temporary and proof-of-concept. for production
## purposes, it should be replaced by the new facility of the upcoming
## re-factored database layer.
import
std/[sequtils, strformat, strutils, tables],
eth/[common/eth_types, trie/nibbles],
stew/results,
../../range_desc,
"."/[hexary_defs, hexary_desc]
{.push raises: [Defect].}
const
RepairTreeDebugging = false
EmptyNibbleRange = EmptyNodeBlob.initNibbleRange
# ------------------------------------------------------------------------------
# Private debugging helpers
# ------------------------------------------------------------------------------
template noPpError(info: static[string]; code: untyped) =
try:
code
except ValueError as e:
raiseAssert "Inconveivable (" & info & "): " & e.msg
except KeyError as e:
raiseAssert "Not possible (" & info & "): " & e.msg
except Defect as e:
raise e
except Exception as e:
raiseAssert "Ooops (" & info & ") " & $e.name & ": " & e.msg
proc pp(w: RPathStep; db: HexaryTreeDB): string =
noPpError("pp(RPathStep)])"):
let nibble = if 0 <= w.nibble: &"{w.nibble:x}" else: "ø"
result = &"({w.key.pp(db)},{nibble},{w.node.pp(db)})"
proc pp(w: openArray[RPathStep]; db: HexaryTreeDB; indent = 4): string =
let pfx = "\n" & " ".repeat(indent)
noPpError("pp(seq[RPathStep])"):
result = w.toSeq.mapIt(it.pp(db)).join(pfx)
proc pp(w: RPath; db: HexaryTreeDB; indent = 4): string =
let pfx = "\n" & " ".repeat(indent)
noPpError("pp(RPath)"):
result = w.path.pp(db,indent) & &"{pfx}({w.tail.pp})"
proc pp(w: RPathXStep; db: HexaryTreeDB): string =
noPpError("pp(RPathXStep)"):
let y = if w.canLock: "lockOk" else: "noLock"
result = &"({w.pos},{y},{w.step.pp(db)})"
proc pp(w: seq[RPathXStep]; db: HexaryTreeDB; indent = 4): string =
let pfx = "\n" & " ".repeat(indent)
noPpError("pp(seq[RPathXStep])"):
result = w.mapIt(it.pp(db)).join(pfx)
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
proc dup(node: RNodeRef): RNodeRef =
new result
result[] = node[]
template noKeyError(info: static[string]; code: untyped) =
try:
code
except KeyError as e:
raiseAssert "Not possible (" & info & "): " & e.msg
# ------------------------------------------------------------------------------
# Private getters & setters
# ------------------------------------------------------------------------------
proc xPfx(node: RNodeRef): NibblesSeq =
case node.kind:
of Leaf:
return node.lPfx
of Extension:
return node.ePfx
of Branch:
doAssert node.kind != Branch # Ooops
proc `xPfx=`(node: RNodeRef, val: NibblesSeq) =
case node.kind:
of Leaf:
node.lPfx = val
of Extension:
node.ePfx = val
of Branch:
doAssert node.kind != Branch # Ooops
proc xData(node: RNodeRef): Blob =
case node.kind:
of Branch:
return node.bData
of Leaf:
return node.lData
of Extension:
doAssert node.kind != Extension # Ooops
proc `xData=`(node: RNodeRef; val: Blob) =
case node.kind:
of Branch:
node.bData = val
of Leaf:
node.lData = val
of Extension:
doAssert node.kind != Extension # Ooops
# ------------------------------------------------------------------------------
# Private functions, repair tree action helpers
# ------------------------------------------------------------------------------
proc rTreeExtendLeaf(
db: var HexaryTreeDB;
rPath: RPath;
key: RepairKey
): RPath =
## Append a `Leaf` node to a `Branch` node (see `rTreeExtend()`.)
if 0 < rPath.tail.len:
let
nibble = rPath.path[^1].nibble
leaf = RNodeRef(
state: Mutable,
kind: Leaf,
lPfx: rPath.tail)
db.tab[key] = leaf
if not key.isNodeKey:
rPath.path[^1].node.bLink[nibble] = key
return RPath(
path: rPath.path & RPathStep(key: key, node: leaf, nibble: -1),
tail: EmptyNibbleRange)
proc rTreeExtendLeaf(
db: var HexaryTreeDB;
rPath: RPath;
key: RepairKey;
node: RNodeRef
): RPath =
## Register `node` and append/link a `Leaf` node to a `Branch` node (see
## `rTreeExtend()`.)
if 1 < rPath.tail.len and node.state == Mutable:
let
nibble = rPath.tail[0].int8
xStep = RPathStep(key: key, node: node, nibble: nibble)
xPath = RPath(path: rPath.path & xStep, tail: rPath.tail.slice(1))
return db.rTreeExtendLeaf(xPath, db.newRepairKey())
proc rTreeSplitNode(
db: var HexaryTreeDB;
rPath: RPath;
key: RepairKey;
node: RNodeRef
): RPath =
## Replace `Leaf` or `Extension` node in tuple `(key,node)` by parts (see
## `rTreeExtend()`):
##
## left(Extension) -> middle(Branch) -> right(Extension or Leaf)
## ^ ^
## | |
## added-to-path added-to-path
##
## where either `left()` or `right()` extensions might be missing.
##
let
nibbles = node.xPfx
lLen = rPath.tail.sharedPrefixLen(nibbles)
if nibbles.len == 0 or rPath.tail.len <= lLen:
return # Ooops (^^^^^ otherwise `rPath` was not the longest)
var
mKey = key
let
mNibble = nibbles[lLen] # exists as `lLen < tail.len`
rPfx = nibbles.slice(lLen + 1) # might be empty OK
result = rPath
# Insert node (if any): left(Extension)
if 0 < lLen:
let lNode = RNodeRef(
state: Mutable,
kind: Extension,
ePfx: result.tail.slice(0,lLen),
eLink: db.newRepairKey())
db.tab[key] = lNode
result.path.add RPathStep(key: key, node: lNode, nibble: -1)
result.tail = result.tail.slice(lLen)
mKey = lNode.eLink
# Insert node: middle(Branch)
let mNode = RNodeRef(
state: Mutable,
kind: Branch)
db.tab[mKey] = mNode
result.path.add RPathStep(key: mKey, node: mNode, nibble: -1) # no nibble yet
# Insert node (if any): right(Extension) -- not to be registered in `rPath`
if 0 < rPfx.len:
let rKey = db.newRepairKey()
# Re-use argument node
mNode.bLink[mNibble] = rKey
db.tab[rKey] = node
node.xPfx = rPfx
# Otherwise merge argument node
elif node.kind == Extension:
mNode.bLink[mNibble] = node.eLink
else:
# Oops, does it make sense, at all?
mNode.bData = node.lData
# ------------------------------------------------------------------------------
# Private functions, repair tree actions
# ------------------------------------------------------------------------------
proc rTreeFollow(
nodeKey: NodeKey;
db: var HexaryTreeDB
): RPath =
## Compute logest possible path matching the `nodeKey` nibbles.
result.tail = nodeKey.to(NibblesSeq)
noKeyError("rTreeFollow"):
var key = db.rootKey.to(RepairKey)
while db.tab.hasKey(key) and 0 < result.tail.len:
let node = db.tab[key]
case node.kind:
of Leaf:
if result.tail.len == result.tail.sharedPrefixLen(node.lPfx):
# Bingo, got full path
result.path.add RPathStep(key: key, node: node, nibble: -1)
result.tail = EmptyNibbleRange
return
of Branch:
let nibble = result.tail[0].int8
if node.bLink[nibble].isZero:
return
result.path.add RPathStep(key: key, node: node, nibble: nibble)
result.tail = result.tail.slice(1)
key = node.bLink[nibble]
of Extension:
if node.ePfx.len != result.tail.sharedPrefixLen(node.ePfx):
return
result.path.add RPathStep(key: key, node: node, nibble: -1)
result.tail = result.tail.slice(node.ePfx.len)
key = node.eLink
proc rTreeFollow(
nodeTag: NodeTag;
db: var HexaryTreeDB
): RPath =
## Variant of `rTreeFollow()`
nodeTag.to(NodeKey).rTreeFollow(db)
proc rTreeInterpolate(
rPath: RPath;
db: var HexaryTreeDB
): RPath =
## Extend path, add missing nodes to tree. The last node added will be
## a `Leaf` node if this function succeeds.
##
## The function assumed that the `RPath` argument is the longest possible
## as just constructed by `rTreeFollow()`
if 0 < rPath.path.len and 0 < rPath.tail.len:
noKeyError("rTreeExtend"):
let step = rPath.path[^1]
case step.node.kind:
of Branch:
# Now, the slot must not be empty. An empty slot would lead to a
# rejection of this record as last valid step, contrary to the
# assumption `path` is the longest one.
if step.nibble < 0:
return # sanitary check failed
let key = step.node.bLink[step.nibble]
if key.isZero:
return # sanitary check failed
# Case: unused slot => add leaf record
if not db.tab.hasKey(key):
return db.rTreeExtendLeaf(rPath, key)
# So a `child` node exits but it is something that could not be used to
# extend the argument `path` which is assumed the longest possible one.
let child = db.tab[key]
case child.kind:
of Branch:
# So a `Leaf` node can be linked into the `child` branch
return db.rTreeExtendLeaf(rPath, key, child)
# Need to split the right `grandChild` in `child -> grandChild`
# into parts:
#
# left(Extension) -> middle(Branch)
# | |
# | +-----> right(Extension or Leaf) ...
# +---------> new Leaf record
#
# where either `left()` or `right()` extensions might be missing
of Extension, Leaf:
var xPath = db.rTreeSplitNode(rPath, key, child)
if 0 < xPath.path.len:
# Append `Leaf` node
xPath.path[^1].nibble = xPath.tail[0].int8
xPath.tail = xPath.tail.slice(1)
return db.rTreeExtendLeaf(xPath, db.newRepairKey())
of Leaf:
return # Oops
of Extension:
let key = step.node.eLink
var child: RNodeRef
if db.tab.hasKey(key):
child = db.tab[key]
# `Extension` can only be followed by a `Branch` node
if child.kind != Branch:
return
else:
# Case: unused slot => add `Branch` and `Leaf` record
child = RNodeRef(
state: Mutable,
kind: Branch)
db.tab[key] = child
# So a `Leaf` node can be linked into the `child` branch
return db.rTreeExtendLeaf(rPath, key, child)
proc rTreeInterpolate(
rPath: RPath;
db: var HexaryTreeDB;
payload: Blob
): RPath =
## Variant of `rTreeExtend()` which completes a `Leaf` record.
result = rPath.rTreeInterpolate(db)
if 0 < result.path.len and result.tail.len == 0:
let node = result.path[^1].node
if node.kind != Extension and node.state == Mutable:
node.xData = payload
proc rTreeUpdateKeys(
rPath: RPath;
db: var HexaryTreeDB
): Result[void,int] =
## The argument `rPath` is assumed to organise database nodes as
##
## root -> ... -> () -> () -> ... -> () -> () ...
## |-------------| |------------| |------
## static nodes locked nodes mutable nodes
##
## Where
## * Static nodes are read-only nodes provided by the proof database
## * Locked nodes are added read-only nodes that satisfy the proof condition
## * Mutable nodes are incomplete nodes
##
## Then update nodes from the right end and set all the mutable nodes
## locked if possible.
var
rTop = rPath.path.len
stack: seq[RPathXStep]
if 0 < rTop and
rPath.path[^1].node.state == Mutable and
rPath.path[0].node.state != Mutable:
# Set `Leaf` entry
let leafNode = rPath.path[^1].node.dup
stack.add RPathXStep(
pos: rTop - 1,
canLock: true,
step: RPathStep(
node: leafNode,
key: leafNode.convertTo(Blob).digestTo(NodeKey).to(RepairKey),
nibble: -1))
while true:
rTop.dec
# Update parent node (note that `2 <= rPath.path.len`)
let
thisKey = stack[^1].step.key
preStep = rPath.path[rTop-1]
preNibble = preStep.nibble
# End reached
if preStep.node.state != Mutable:
# Verify the tail matches
var key = RepairKey.default
case preStep.node.kind:
of Branch:
key = preStep.node.bLink[preNibble]
of Extension:
key = preStep.node.eLink
of Leaf:
discard
if key != thisKey:
return err(rTop-1)
when RepairTreeDebugging:
echo "*** rTreeUpdateKeys",
" rPath\n ", rPath.pp(ps),
"\n stack\n ", stack.pp(ps)
# Ok, replace database records by stack entries
var lockOk = true
for n in countDown(stack.len-1,0):
let item = stack[n]
db.tab.del(rPath.path[item.pos].key)
db.tab[item.step.key] = item.step.node
if lockOk:
if item.canLock:
item.step.node.state = Locked
else:
lockOk = false
if not lockOk:
return err(rTop-1) # repeat
break # Done ok()
stack.add RPathXStep(
pos: rTop - 1,
step: RPathStep(
node: preStep.node.dup, # (!)
nibble: preNibble,
key: preStep.key))
case stack[^1].step.node.kind:
of Branch:
stack[^1].step.node.bLink[preNibble] = thisKey
# Check whether all keys are proper, non-temporary keys
stack[^1].canLock = true
for n in 0 ..< 16:
if not stack[^1].step.node.bLink[n].isNodeKey:
stack[^1].canLock = false
break
of Extension:
stack[^1].step.node.eLink = thisKey
stack[^1].canLock = thisKey.isNodeKey
of Leaf:
return err(rTop-1)
# Must not overwrite a non-temprary key
if stack[^1].canLock:
stack[^1].step.key =
stack[^1].step.node.convertTo(Blob).digestTo(NodeKey).to(RepairKey)
ok()
# ------------------------------------------------------------------------------
# Public fuctions
# ------------------------------------------------------------------------------
proc hexary_interpolate*(db: var HexaryTreeDB): Result[void,HexaryDbError] =
## Verifiy accounts by interpolating the collected accounts on the hexary
## trie of the repair database. If all accounts can be represented in the
## hexary trie, they are vonsidered validated.
##
# Walk top down and insert/complete missing account access nodes
for n in countDown(db.acc.len-1,0):
let acc = db.acc[n]
if acc.payload.len != 0:
let rPath = acc.pathTag.rTreeFollow(db)
var repairKey = acc.nodeKey
if repairKey.isZero and 0 < rPath.path.len and rPath.tail.len == 0:
repairKey = rPath.path[^1].key
db.acc[n].nodeKey = repairKey
if repairKey.isZero:
let
update = rPath.rTreeInterpolate(db, acc.payload)
final = acc.pathTag.rTreeFollow(db)
if update != final:
return err(AccountRepairBlocked)
db.acc[n].nodeKey = rPath.path[^1].key
# Replace temporary repair keys by proper hash based node keys.
var reVisit: seq[NodeTag]
for n in countDown(db.acc.len-1,0):
let acc = db.acc[n]
if not acc.nodeKey.isZero:
let rPath = acc.pathTag.rTreeFollow(db)
if rPath.path[^1].node.state == Mutable:
let rc = rPath.rTreeUpdateKeys(db)
if rc.isErr:
reVisit.add acc.pathTag
while 0 < reVisit.len:
var again: seq[NodeTag]
for nodeTag in reVisit:
let rc = nodeTag.rTreeFollow(db).rTreeUpdateKeys(db)
if rc.isErr:
again.add nodeTag
if reVisit.len <= again.len:
return err(BoundaryProofFailed)
reVisit = again
ok()
# ------------------------------------------------------------------------------
# Debugging
# ------------------------------------------------------------------------------
proc dumpPath*(db: var HexaryTreeDB; key: NodeTag): seq[string] =
## Pretty print helper compiling the path into the repair tree for the
## argument `key`.
let rPath = key.rTreeFollow(db)
rPath.path.mapIt(it.pp(db)) & @["(" & rPath.tail.pp & ")"]
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------

7
nimbus/sync/snap/worker/rocky_bulk_load.nim → nimbus/sync/snap/worker/db/rocky_bulk_load.nim
View File

@ -1,5 +1,4 @@
# Nimbus - Types, data structures and shared utilities used in network sync
#
# nimbus-eth1
# Copyright (c) 2018-2021 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
@ -15,7 +14,7 @@ import
std/os, # std/[sequtils, strutils],
eth/common/eth_types,
rocksdb,
../../../db/[kvstore_rocksdb, select_backend]
../../../../db/[kvstore_rocksdb, select_backend]
{.push raises: [Defect].}
@ -165,7 +164,7 @@ proc finish*(
## `destroy()` if successful. Otherwise `destroy()` must be called
## explicitely, e.g. after error analysis.
##
## If successful, the return value is the size of the SST file used if
## If successful, the return value is the size of the SST file used if
## that value is available. Otherwise, `0` is returned.
when select_backend.dbBackend == select_backend.rocksdb:
var csError: cstring

102
tests/test_sync_snap.nim
View File

@ -25,7 +25,7 @@ import
../nimbus/p2p/chain,
../nimbus/sync/[types, protocol],
../nimbus/sync/snap/range_desc,
../nimbus/sync/snap/worker/[accounts_db, rocky_bulk_load],
../nimbus/sync/snap/worker/[accounts_db, db/hexary_desc, db/rocky_bulk_load],
../nimbus/utils/prettify,
./replay/[pp, undump],
./test_sync_snap/sample0
@ -87,7 +87,7 @@ const
let
# Forces `check()` to print the error (as opposed when using `isOk()`)
OkAccDb = Result[void,AccountsDbError].ok()
OkHexDb = Result[void,HexaryDbError].ok()
# There was a problem with the Github/CI which results in spurious crashes
# when leaving the `runner()` if the persistent BaseChainDB initialisation
@ -135,7 +135,7 @@ proc pp(d: AccountLoadStats): string =
"[" & d.size.toSeq.mapIt(it.toSI).join(",") & "," &
d.dura.toSeq.mapIt(it.pp).join(",") & "]"
proc pp(rc: Result[Account,AccountsDbError]): string =
proc pp(rc: Result[Account,HexaryDbError]): string =
if rc.isErr: $rc.error else: rc.value.pp
proc ppKvPc(w: openArray[(string,int)]): string =
@ -268,52 +268,52 @@ proc accountsRunner(noisy = true; persistent = true; sample = accSample0) =
tmpDir = getTmpDir()
db = if persistent: tmpDir.testDbs(sample.name) else: testDbs()
dbDir = db.dbDir.split($DirSep).lastTwo.join($DirSep)
info = if db.persistent: &"persistent db on \"{dbDir}\"" else: "in-memory db"
info = if db.persistent: &"persistent db on \"{dbDir}\""
else: "in-memory db"
defer:
if db.persistent:
tmpDir.flushDbDir(sample.name)
suite &"SyncSnap: {sample.name} accounts and proofs for {info}":
var
desc: AccountsDbSessionRef
accounts: seq[SnapAccount]
test &"Verifying {testItemLst.len} snap items for state root ..{root.pp}":
test &"Snap-proofing {testItemLst.len} items for state root ..{root.pp}":
let dbBase = if persistent: AccountsDbRef.init(db.cdb[0])
else: AccountsDbRef.init(newMemoryDB())
if not dbBase.dbBackendRocksDb():
skip()
else:
for n,w in testItemLst:
check dbBase.importAccounts(
peer, root, w.base, w.data, storeData = persistent) == OkAccDb
noisy.say "***", "import stats=", dbBase.dbImportStats.pp
for n,w in testItemLst:
check dbBase.importAccounts(
peer, root, w.base, w.data, storeData = persistent) == OkHexDb
noisy.say "***", "import stats=", dbBase.dbImportStats.pp
test &"Merging {testItemLst.len} proofs for state root ..{root.pp}":
let
dbBase = if persistent: AccountsDbRef.init(db.cdb[1])
else: AccountsDbRef.init(newMemoryDB())
desc = AccountsDbSessionRef.init(dbBase, root, peer)
let dbBase = if persistent: AccountsDbRef.init(db.cdb[1])
else: AccountsDbRef.init(newMemoryDB())
desc = AccountsDbSessionRef.init(dbBase, root, peer)
for w in testItemLst:
check desc.merge(w.data.proof) == OkAccDb
let
base = testItemLst.mapIt(it.base).sortMerge
accounts = testItemLst.mapIt(it.data.accounts).sortMerge
check desc.merge(base, accounts) == OkAccDb
check desc.merge(w.data.proof) == OkHexDb
let base = testItemLst.mapIt(it.base).sortMerge
accounts = testItemLst.mapIt(it.data.accounts).sortMerge
check desc.merge(base, accounts) == OkHexDb
desc.assignPrettyKeys() # for debugging (if any)
check desc.interpolate() == OkAccDb
check desc.interpolate() == OkHexDb
if dbBase.dbBackendRocksDb():
check desc.dbImports() == OkAccDb
noisy.say "***", "import stats=", desc.dbImportStats.pp
check desc.dbImports() == OkHexDb
noisy.say "***", "import stats=", desc.dbImportStats.pp
for acc in accounts:
let
byChainDB = desc.getChainDbAccount(acc.accHash)
byRockyBulker = desc.getRockyAccount(acc.accHash)
noisy.say "*** find",
"byChainDb=", byChainDB.pp, " inBulker=", byRockyBulker.pp
check byChainDB.isOk
check byRockyBulker.isOk
check byChainDB == byRockyBulker
test &"Revisting {accounts.len} items stored items on BaseChainDb":
for acc in accounts:
let
byChainDB = desc.getChainDbAccount(acc.accHash)
byBulker = desc.getBulkDbXAccount(acc.accHash)
noisy.say "*** find",
"byChainDb=", byChainDB.pp, " inBulker=", byBulker.pp
check byChainDB.isOk
if desc.dbBackendRocksDb():
check byBulker.isOk
check byChainDB == byBulker
#noisy.say "***", "database dump\n ", desc.dumpProofsDB.join("\n ")
@ -325,7 +325,8 @@ proc importRunner(noisy = true; persistent = true; capture = goerliCapture) =
filePath = capture.file.findFilePath(baseDir,repoDir).value
tmpDir = getTmpDir()
db = if persistent: tmpDir.testDbs(capture.name) else: testDbs()
numBlocksInfo = if capture.numBlocks == high(int): "" else: $capture.numBlocks & " "
numBlocksInfo = if capture.numBlocks == high(int): ""
else: $capture.numBlocks & " "
loadNoise = noisy
defer:
@ -753,7 +754,11 @@ proc storeRunner(noisy = true; persistent = true; cleanUp = true) =
# ------------------------------------------------------------------------------
proc syncSnapMain*(noisy = defined(debug)) =
noisy.accountsRunner()
# Caveat: running `accountsRunner(persistent=true)` twice will crash as the
# persistent database might not be fully cleared due to some stale
# locks.
noisy.accountsRunner(persistent=true)
noisy.accountsRunner(persistent=false)
noisy.importRunner() # small sample, just verify functionality
noisy.storeRunner()
@ -786,11 +791,11 @@ when isMainModule:
when false: # or true:
import ../../nimbus-eth1-blobs/replay/sync_sample1 as sample1
const
snapTest2 = AccountsProofSample(
snapTest2 = AccountsProofSample(
name: "test2",
root: sample1.snapRoot,
data: sample1.snapProofData)
snapTest3 = AccountsProofSample(
snapTest3 = AccountsProofSample(
name: "test3",
root: snapTest2.root,
data: snapTest2.data[0..0])
@ -798,22 +803,23 @@ when isMainModule:
#setTraceLevel()
setErrorLevel()
#noisy.accountsRunner(persistent=true, snapTest0)
false.accountsRunner(persistent=true, snapTest0)
false.accountsRunner(persistent=false, snapTest0)
#noisy.accountsRunner(persistent=true, snapTest1)
when defined(snapTest2):
discard
#noisy.accountsRunner(persistent=true, snapTest2)
when declared(snapTest2):
noisy.accountsRunner(persistent=false, snapTest2)
#noisy.accountsRunner(persistent=true, snapTest3)
# ---- database storage timings -------
when true: # and false:
# ---- database storage timings -------
noisy.showElapsed("importRunner()"):
noisy.importRunner(capture = bulkTest0)
noisy.showElapsed("importRunner()"):
noisy.importRunner(capture = bulkTest0)
noisy.showElapsed("storeRunner()"):
true.storeRunner(cleanUp = false)
true.storeRunner()
noisy.showElapsed("storeRunner()"):
true.storeRunner(cleanUp = false)
true.storeRunner()
# ------------------------------------------------------------------------------
# End