mirror of
https://github.com/status-im/nimbus-eth1.git
synced 2025-01-13 22:04:52 +00:00
b793f0de8d
* Redefine `seq[Blob]` => `seq[SnapProof]` for `snap/1` protocol why: Proof nodes are traded as `Blob` type items rather than Nim objects. So the RLP transcoder must not extra wrap proofs which are of type seq[Blob]. Without custom encoding one would produce a `list(blob(item1), blob(item2) ..)` instead of `list(item1, item2 ..)`. * Limit leaf extractor by RLP size rather than number of items why: To be used serving `snap/1` requests, the result of function `hexaryRangeLeafsProof()` is limited by the maximal space needed to serialise the result which will be part of the `snap/1` repsonse. * Let the range extractor `hexaryRangeLeafsProof()` return RLP list sizes why: When collecting accounts, the size oft the accounts list when encoded as RLP is continually updated. So the summed up value is available anyway. For the proof nodes list, there are not many (~ 10) so summing up is not expensive here.
172 lines
6.6 KiB
Nim
172 lines
6.6 KiB
Nim
# Nimbus - Types, data structures and shared utilities used in network sync
|
|
#
|
|
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or
|
|
# distributed except according to those terms.
|
|
|
|
## Snap sync components tester and TDD environment
|
|
##
|
|
## This module provides test bodies for storing chain chain data directly
|
|
## rather than derive them by executing the EVM. Here, only accounts are
|
|
## considered.
|
|
##
|
|
## The `snap/1` protocol allows to fetch data for a certain account range. The
|
|
## following boundary conditions apply to the received data:
|
|
##
|
|
## * `State root`: All data are relaive to the same state root.
|
|
##
|
|
## * `Accounts`: There is an accounts interval sorted in strictly increasing
|
|
## order. The accounts are required consecutive, i.e. without holes in
|
|
## between although this cannot be verified immediately.
|
|
##
|
|
## * `Lower bound`: There is a start value which might be lower than the first
|
|
## account hash. There must be no other account between this start value and
|
|
## the first account (not verifyable yet.) For all practicat purposes, this
|
|
## value is mostly ignored but carried through.
|
|
##
|
|
## * `Proof`: There is a list of hexary nodes which allow to build a partial
|
|
## Patricia-Merkle trie starting at the state root with all the account
|
|
## leaves. There are enough nodes that show that there is no account before
|
|
## the least account (which is currently ignored.)
|
|
##
|
|
## There are test data samples on the sub-directory `test_sync_snap`. These
|
|
## are complete replies for some (admittedly snap) test requests from a `kiln#`
|
|
## session.
|
|
##
|
|
## There are three tests:
|
|
##
|
|
## 1. Run the `test_accountsImport()` function which is the all-in-one
|
|
## production function processoing the data described above. The test
|
|
## applies it sequentially to all argument data sets.
|
|
##
|
|
## 2. With `test_accountsMergeProofs()` individual items are tested which are
|
|
## hidden in test 1. while merging the sample data.
|
|
## * Load/accumulate `proofs` data from several samples
|
|
## * Load/accumulate accounts (needs some unique sorting)
|
|
## * Build/complete hexary trie for accounts
|
|
## * Save/bulk-store hexary trie on disk. If rocksdb is available, data
|
|
## are bulk stored via sst.
|
|
##
|
|
## 3. The function `test_accountsRevisitStoredItems()` traverses trie nodes
|
|
## stored earlier. The accounts from test 2 are re-visted using the account
|
|
## hash as access path.
|
|
##
|
|
|
|
import
|
|
std/algorithm,
|
|
eth/[common, p2p],
|
|
unittest2,
|
|
../../nimbus/db/select_backend,
|
|
../../nimbus/sync/protocol,
|
|
../../nimbus/sync/snap/range_desc,
|
|
../../nimbus/sync/snap/worker/db/[snapdb_accounts, snapdb_desc],
|
|
../replay/[pp, undump_accounts],
|
|
./test_helpers
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc flatten(list: openArray[seq[SnapProof]]): seq[SnapProof] =
|
|
for w in list:
|
|
result.add w
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public test function
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc test_accountsImport*(
|
|
inList: seq[UndumpAccounts];
|
|
desc: SnapDbAccountsRef;
|
|
persistent: bool;
|
|
) =
|
|
## Import accounts
|
|
for n,w in inList:
|
|
check desc.importAccounts(w.base, w.data, persistent).isImportOk
|
|
|
|
|
|
proc test_accountsMergeProofs*(
|
|
inList: seq[UndumpAccounts];
|
|
desc: SnapDbAccountsRef;
|
|
accKeys: var seq[NodeKey];
|
|
) =
|
|
## Merge account proofs
|
|
# Load/accumulate data from several samples (needs some particular sort)
|
|
let baseTag = inList.mapIt(it.base).sortMerge
|
|
let packed = PackedAccountRange(
|
|
accounts: inList.mapIt(it.data.accounts).sortMerge,
|
|
proof: inList.mapIt(it.data.proof).flatten)
|
|
# Merging intervals will produce gaps, so the result is expected OK but
|
|
# different from `.isImportOk`
|
|
check desc.importAccounts(baseTag, packed, true).isOk
|
|
|
|
# check desc.merge(lowerBound, accounts) == OkHexDb
|
|
desc.assignPrettyKeys() # for debugging, make sure that state root ~ "$0"
|
|
|
|
# Update list of accounts. There might be additional accounts in the set
|
|
# of proof nodes, typically before the `lowerBound` of each block. As
|
|
# there is a list of account ranges (that were merged for testing), one
|
|
# need to check for additional records only on either end of a range.
|
|
var keySet = packed.accounts.mapIt(it.accKey).toHashSet
|
|
for w in inList:
|
|
var key = desc.prevAccountsChainDbKey(w.data.accounts[0].accKey)
|
|
while key.isOk and key.value notin keySet:
|
|
keySet.incl key.value
|
|
let newKey = desc.prevAccountsChainDbKey(key.value)
|
|
check newKey != key
|
|
key = newKey
|
|
key = desc.nextAccountsChainDbKey(w.data.accounts[^1].accKey)
|
|
while key.isOk and key.value notin keySet:
|
|
keySet.incl key.value
|
|
let newKey = desc.nextAccountsChainDbKey(key.value)
|
|
check newKey != key
|
|
key = newKey
|
|
accKeys = toSeq(keySet).mapIt(it.to(NodeTag)).sorted(cmp)
|
|
.mapIt(it.to(NodeKey))
|
|
check packed.accounts.len <= accKeys.len
|
|
|
|
|
|
proc test_accountsRevisitStoredItems*(
|
|
accKeys: seq[NodeKey];
|
|
desc: SnapDbAccountsRef;
|
|
noisy = false;
|
|
) =
|
|
## Revisit stored items on ChainDBRef
|
|
var
|
|
nextAccount = accKeys[0]
|
|
prevAccount: NodeKey
|
|
count = 0
|
|
for accKey in accKeys:
|
|
count.inc
|
|
let
|
|
pfx = $count & "#"
|
|
byChainDB = desc.getAccountsChainDb(accKey)
|
|
byNextKey = desc.nextAccountsChainDbKey(accKey)
|
|
byPrevKey = desc.prevAccountsChainDbKey(accKey)
|
|
noisy.say "*** find",
|
|
"<", count, "> byChainDb=", byChainDB.pp
|
|
check byChainDB.isOk
|
|
|
|
# Check `next` traversal funcionality. If `byNextKey.isOk` fails, the
|
|
# `nextAccount` value is still the old one and will be different from
|
|
# the account in the next for-loop cycle (if any.)
|
|
check pfx & accKey.pp(false) == pfx & nextAccount.pp(false)
|
|
if byNextKey.isOk:
|
|
nextAccount = byNextKey.get(otherwise = NodeKey.default)
|
|
|
|
# Check `prev` traversal funcionality
|
|
if prevAccount != NodeKey.default:
|
|
check byPrevKey.isOk
|
|
if byPrevKey.isOk:
|
|
check pfx & byPrevKey.value.pp(false) == pfx & prevAccount.pp(false)
|
|
prevAccount = accKey
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|