nimbus-eth1/tests/test_sync_snap/test_accounts.nim

172 lines
6.6 KiB
Nim
Raw Normal View History

# Nimbus - Types, data structures and shared utilities used in network sync
#
# Copyright (c) 2018-2021 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or
# distributed except according to those terms.
## Snap sync components tester and TDD environment
##
## This module provides test bodies for storing chain chain data directly
## rather than derive them by executing the EVM. Here, only accounts are
## considered.
##
## The `snap/1` protocol allows to fetch data for a certain account range. The
## following boundary conditions apply to the received data:
##
## * `State root`: All data are relaive to the same state root.
##
## * `Accounts`: There is an accounts interval sorted in strictly increasing
## order. The accounts are required consecutive, i.e. without holes in
## between although this cannot be verified immediately.
##
## * `Lower bound`: There is a start value which might be lower than the first
## account hash. There must be no other account between this start value and
## the first account (not verifyable yet.) For all practicat purposes, this
## value is mostly ignored but carried through.
##
## * `Proof`: There is a list of hexary nodes which allow to build a partial
## Patricia-Merkle trie starting at the state root with all the account
## leaves. There are enough nodes that show that there is no account before
## the least account (which is currently ignored.)
##
## There are test data samples on the sub-directory `test_sync_snap`. These
## are complete replies for some (admittedly snap) test requests from a `kiln#`
## session.
##
## There are three tests:
##
## 1. Run the `test_accountsImport()` function which is the all-in-one
## production function processoing the data described above. The test
## applies it sequentially to all argument data sets.
##
## 2. With `test_accountsMergeProofs()` individual items are tested which are
## hidden in test 1. while merging the sample data.
## * Load/accumulate `proofs` data from several samples
## * Load/accumulate accounts (needs some unique sorting)
## * Build/complete hexary trie for accounts
## * Save/bulk-store hexary trie on disk. If rocksdb is available, data
## are bulk stored via sst.
##
## 3. The function `test_accountsRevisitStoredItems()` traverses trie nodes
## stored earlier. The accounts from test 2 are re-visted using the account
## hash as access path.
##
import
std/algorithm,
eth/[common, p2p],
unittest2,
../../nimbus/db/select_backend,
../../nimbus/sync/protocol,
../../nimbus/sync/snap/range_desc,
../../nimbus/sync/snap/worker/db/[snapdb_accounts, snapdb_desc],
../replay/[pp, undump_accounts],
./test_helpers
# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------
proc flatten(list: openArray[seq[SnapProof]]): seq[SnapProof] =
for w in list:
result.add w
# ------------------------------------------------------------------------------
# Public test function
# ------------------------------------------------------------------------------
proc test_accountsImport*(
inList: seq[UndumpAccounts];
desc: SnapDbAccountsRef;
persistent: bool;
) =
## Import accounts
for n,w in inList:
check desc.importAccounts(w.base, w.data, persistent).isImportOk
proc test_accountsMergeProofs*(
inList: seq[UndumpAccounts];
desc: SnapDbAccountsRef;
accKeys: var seq[NodeKey];
) =
## Merge account proofs
# Load/accumulate data from several samples (needs some particular sort)
let baseTag = inList.mapIt(it.base).sortMerge
let packed = PackedAccountRange(
accounts: inList.mapIt(it.data.accounts).sortMerge,
proof: inList.mapIt(it.data.proof).flatten)
# Merging intervals will produce gaps, so the result is expected OK but
# different from `.isImportOk`
check desc.importAccounts(baseTag, packed, true).isOk
# check desc.merge(lowerBound, accounts) == OkHexDb
desc.assignPrettyKeys() # for debugging, make sure that state root ~ "$0"
# Update list of accounts. There might be additional accounts in the set
# of proof nodes, typically before the `lowerBound` of each block. As
# there is a list of account ranges (that were merged for testing), one
# need to check for additional records only on either end of a range.
var keySet = packed.accounts.mapIt(it.accKey).toHashSet
for w in inList:
var key = desc.prevAccountsChainDbKey(w.data.accounts[0].accKey)
while key.isOk and key.value notin keySet:
keySet.incl key.value
let newKey = desc.prevAccountsChainDbKey(key.value)
check newKey != key
key = newKey
key = desc.nextAccountsChainDbKey(w.data.accounts[^1].accKey)
while key.isOk and key.value notin keySet:
keySet.incl key.value
let newKey = desc.nextAccountsChainDbKey(key.value)
check newKey != key
key = newKey
accKeys = toSeq(keySet).mapIt(it.to(NodeTag)).sorted(cmp)
.mapIt(it.to(NodeKey))
check packed.accounts.len <= accKeys.len
proc test_accountsRevisitStoredItems*(
accKeys: seq[NodeKey];
desc: SnapDbAccountsRef;
noisy = false;
) =
## Revisit stored items on ChainDBRef
var
nextAccount = accKeys[0]
prevAccount: NodeKey
count = 0
for accKey in accKeys:
count.inc
let
pfx = $count & "#"
byChainDB = desc.getAccountsChainDb(accKey)
byNextKey = desc.nextAccountsChainDbKey(accKey)
byPrevKey = desc.prevAccountsChainDbKey(accKey)
noisy.say "*** find",
"<", count, "> byChainDb=", byChainDB.pp
check byChainDB.isOk
# Check `next` traversal funcionality. If `byNextKey.isOk` fails, the
# `nextAccount` value is still the old one and will be different from
# the account in the next for-loop cycle (if any.)
check pfx & accKey.pp(false) == pfx & nextAccount.pp(false)
if byNextKey.isOk:
nextAccount = byNextKey.get(otherwise = NodeKey.default)
# Check `prev` traversal funcionality
if prevAccount != NodeKey.default:
check byPrevKey.isOk
if byPrevKey.isOk:
check pfx & byPrevKey.value.pp(false) == pfx & prevAccount.pp(false)
prevAccount = accKey
# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------