Extract finding of missing nodes for healing into separate module (#1398)
why: Duplicate implementation of same functionality
This commit is contained in:
Jordan Hrycaj
GitHub
parent
88b315bb41
commit
5134bb5e04
|
|
@ -0,0 +1,133 @@
|
|||
# Nimbus
|
||||
# Copyright (c) 2021 Status Research & Development GmbH
|
||||
# Licensed under either of
|
||||
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0)
|
||||
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
||||
# http://opensource.org/licenses/MIT)
|
||||
# at your option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
## Find missing nodes for healing
|
||||
## ==============================
|
||||
##
|
||||
## This module searches for missing nodes in the database (which means that
|
||||
## nodes which link to missing ones must exist.)
|
||||
##
|
||||
## Algorithm
|
||||
## ---------
|
||||
##
|
||||
## * Find dangling node links in the current account trie by trying *plan A*,
|
||||
## and continuing with *plan B* only if *plan A* fails.
|
||||
##
|
||||
## A. Try to find nodes with envelopes that have no account in common with
|
||||
## any range interval of the `processed` set of the hexary trie. This
|
||||
## action will
|
||||
##
|
||||
## + either determine that there are no such envelopes implying that the
|
||||
## accounts trie is complete (then stop here)
|
||||
##
|
||||
## + or result in envelopes related to nodes that are all allocated on the
|
||||
## accounts trie (fail, use *plan B* below)
|
||||
##
|
||||
## + or result in some envelopes related to dangling nodes.
|
||||
##
|
||||
## B. Employ the `hexaryInspect()` trie perusal function in a limited mode
|
||||
## for finding dangling (i.e. missing) sub-nodes below the allocated nodes.
|
||||
##
|
||||
## Discussion
|
||||
## ----------
|
||||
##
|
||||
## For *plan A*, the complement of ranges in the `processed` is determined
|
||||
## and expressed as a list of node envelopes. As a consequence, the gaps
|
||||
## beween the envelopes are either blind ranges that have no leaf nodes in
|
||||
## the databse, or they are contained in the `processed` range. These gaps
|
||||
## will be silently merged into the `processed` set of ranges.
|
||||
##
|
||||
## For *plan B*, a worst case scenario of a failing *plan B* must be solved
|
||||
## by fetching and storing more nodes with other means before using this
|
||||
## algorithm to find more missing nodes.
|
||||
##
|
||||
## Due to the potentially poor performance using `hexaryInspect()`.there is
|
||||
## no general solution for *plan B* by recursively searching the whole hexary
|
||||
## trie database for more dangling nodes.
|
||||
##
|
||||
import
|
||||
std/sequtils,
|
||||
eth/common,
|
||||
stew/interval_set,
|
||||
"../../.."/[sync_desc, types],
|
||||
"../.."/[constants, range_desc, worker_desc],
|
||||
../db/[hexary_desc, hexary_envelope, hexary_inspect]
|
||||
|
||||
{.push raises: [Defect].}
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Private helpers
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
template noExceptionOops(info: static[string]; code: untyped) =
|
||||
try:
|
||||
code
|
||||
except Defect as e:
|
||||
raise e
|
||||
except Exception as e:
|
||||
raiseAssert "Ooops " & info & ": name=" & $e.name & " msg=" & e.msg
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Public functions
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
proc findMissingNodes*(
|
||||
ranges: SnapRangeBatchRef;
|
||||
rootKey: NodeKey;
|
||||
getFn: HexaryGetFn;
|
||||
planBLevelMax: uint8;
|
||||
): (seq[NodeSpecs],uint8,uint64) =
|
||||
## Find some missing nodes in the hexary trie database.
|
||||
var nodes: seq[NodeSpecs]
|
||||
|
||||
# Plan A, try complement of `processed`
|
||||
noExceptionOops("compileMissingNodesList"):
|
||||
if not ranges.processed.isEmpty:
|
||||
# Get unallocated nodes to be fetched
|
||||
let rc = ranges.processed.hexaryEnvelopeDecompose(rootKey, getFn)
|
||||
if rc.isOk:
|
||||
nodes = rc.value
|
||||
|
||||
# The gaps between resuling envelopes are either ranges that have
|
||||
# no leaf nodes, or they are contained in the `processed` range. So
|
||||
# these gaps are be merged back into the `processed` set of ranges.
|
||||
let gaps = NodeTagRangeSet.init()
|
||||
discard gaps.merge(low(NodeTag),high(NodeTag)) # All range
|
||||
for w in nodes: discard gaps.reduce w.hexaryEnvelope # Remove envelopes
|
||||
|
||||
# Merge gaps into `processed` range and update `unprocessed` ranges
|
||||
for iv in gaps.increasing:
|
||||
discard ranges.processed.merge iv
|
||||
ranges.unprocessed.reduce iv
|
||||
|
||||
# Check whether the hexary trie is complete
|
||||
if ranges.processed.isFull:
|
||||
return
|
||||
|
||||
# Remove allocated nodes
|
||||
let missing = nodes.filterIt(it.nodeKey.ByteArray32.getFn().len == 0)
|
||||
if 0 < missing.len:
|
||||
return (missing, 0u8, 0u64)
|
||||
|
||||
# Plan B, carefully employ `hexaryInspect()`
|
||||
if 0 < nodes.len:
|
||||
try:
|
||||
let
|
||||
paths = nodes.mapIt it.partialPath
|
||||
stats = getFn.hexaryInspectTrie(rootKey, paths,
|
||||
stopAtLevel = planBLevelMax,
|
||||
maxDangling = fetchRequestTrieNodesMax)
|
||||
result = (stats.dangling, stats.level, stats.count)
|
||||
except:
|
||||
discard
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# End
|
||||
# ------------------------------------------------------------------------------
|
||||
|
|
@ -12,9 +12,9 @@
|
|||
## ================
|
||||
##
|
||||
## This module is a variation of the `swap-in` module in the sense that it
|
||||
## searches for missing nodes in the database (which means that links to
|
||||
## that nodes must exist for knowing this fact) and then fetches the nodes
|
||||
## from the network.
|
||||
## searches for missing nodes in the database (which means that nodes which
|
||||
## link to missing ones must exist), and then fetches the nodes from the
|
||||
## network.
|
||||
##
|
||||
## Algorithm
|
||||
## ---------
|
||||
|
|
@ -22,20 +22,7 @@
|
|||
## * Run `swapInAccounts()` so that inheritable sub-tries are imported from
|
||||
## previous pivots.
|
||||
##
|
||||
## * Find dangling nodes in the current account trie by trying plan A, and
|
||||
## continuing with plan B only if A fails.
|
||||
##
|
||||
## A. Try to find nodes with envelopes that have no account in common with
|
||||
## any range interval of the `processed` set of the accounts trie. This
|
||||
## action will
|
||||
## + either determine that there are no such envelopes implying that the
|
||||
## accounts trie is complete (then stop here)
|
||||
## + or result in envelopes related to nodes that are all allocated on the
|
||||
## accounts trie (fail, use *plan B* below)
|
||||
## + or result in some envelopes related to dangling nodes.
|
||||
##
|
||||
## B. Employ the `hexaryInspect()` trie perusal function in a limited mode
|
||||
## for finding dangling (i.e. missing) sub-nodes below the allocated nodes.
|
||||
## * Find dangling nodes in the current account trie via `findMissingNodes()`.
|
||||
##
|
||||
## * Install that nodes from the network.
|
||||
##
|
||||
|
|
@ -44,26 +31,22 @@
|
|||
## Discussion
|
||||
## ----------
|
||||
##
|
||||
## A worst case scenario of a failing *plan B* must be solved by fetching and
|
||||
## storing more accounts and running this healing algorithm again.
|
||||
##
|
||||
## Due to the potentially poor performance using `hexaryInspect()`.there is no
|
||||
## general solution for *plan B* by recursively searching the whole accounts
|
||||
## hexary trie database for more dangling nodes.
|
||||
## A worst case scenario of a portentally failing `findMissingNodes()` call
|
||||
## must be solved by fetching and storing more accounts and running this
|
||||
## healing algorithm again.
|
||||
##
|
||||
import
|
||||
std/[math, sequtils, tables],
|
||||
std/[math, sequtils],
|
||||
chronicles,
|
||||
chronos,
|
||||
eth/[common, p2p, trie/nibbles, trie/trie_defs, rlp],
|
||||
stew/[byteutils, interval_set, keyed_queue],
|
||||
stew/[byteutils, interval_set],
|
||||
../../../../utils/prettify,
|
||||
"../../.."/[sync_desc, types],
|
||||
"../.."/[constants, range_desc, worker_desc],
|
||||
../com/[com_error, get_trie_nodes],
|
||||
../db/[hexary_desc, hexary_envelope, hexary_error, hexary_inspect,
|
||||
snapdb_accounts],
|
||||
"."/[storage_queue_helper, swap_in]
|
||||
../db/[hexary_desc, hexary_envelope, hexary_error, snapdb_accounts],
|
||||
"."/[find_missing_nodes, storage_queue_helper, swap_in]
|
||||
|
||||
{.push raises: [Defect].}
|
||||
|
||||
|
|
@ -131,7 +114,7 @@ proc compileMissingNodesList(
|
|||
buddy: SnapBuddyRef;
|
||||
env: SnapPivotRef;
|
||||
): seq[NodeSpecs] =
|
||||
## Find some missing glue nodes in accounts database to be fetched.
|
||||
## Find some missing glue nodes in accounts database.
|
||||
let
|
||||
ctx = buddy.ctx
|
||||
peer = buddy.peer
|
||||
|
|
@ -140,48 +123,20 @@ proc compileMissingNodesList(
|
|||
fa = env.fetchAccounts
|
||||
|
||||
# Import from earlier run
|
||||
while buddy.ctx.swapInAccounts(env) != 0:
|
||||
if buddy.ctrl.stopped:
|
||||
return
|
||||
if ctx.swapInAccounts(env) != 0:
|
||||
discard ctx.swapInAccounts(env)
|
||||
|
||||
var nodes: seq[NodeSpecs]
|
||||
noExceptionOops("compileMissingNodesList"):
|
||||
# Get unallocated nodes to be fetched
|
||||
let rc = fa.processed.hexaryEnvelopeDecompose(rootKey, getFn)
|
||||
if rc.isOk:
|
||||
nodes = rc.value
|
||||
|
||||
# Check whether the hexary trie is complete
|
||||
if nodes.len == 0:
|
||||
# Fill gaps
|
||||
discard fa.processed.merge(low(NodeTag),high(NodeTag))
|
||||
fa.unprocessed.clear()
|
||||
return
|
||||
|
||||
# Remove allocated nodes
|
||||
let missing = nodes.filterIt(it.nodeKey.ByteArray32.getFn().len == 0)
|
||||
if 0 < missing.len:
|
||||
when extraTraceMessages:
|
||||
trace logTxt "missing nodes", peer, ctx=buddy.healingCtx(env),
|
||||
nResult=missing.len, result=missing.toPC
|
||||
return missing
|
||||
|
||||
# Plan B, carefully employ `hexaryInspect()`
|
||||
if 0 < nodes.len:
|
||||
try:
|
||||
let
|
||||
paths = nodes.mapIt it.partialPath
|
||||
stats = getFn.hexaryInspectTrie(rootKey, paths,
|
||||
stopAtLevel = healAccountsInspectionPlanBLevel,
|
||||
maxDangling = fetchRequestTrieNodesMax)
|
||||
result = stats.dangling
|
||||
if not fa.processed.isFull:
|
||||
noExceptionOops("compileMissingNodesList"):
|
||||
let (missing, nLevel, nVisited) = fa.findMissingNodes(
|
||||
rootKey, getFn, healAccountsInspectionPlanBLevel)
|
||||
|
||||
when extraTraceMessages:
|
||||
trace logTxt "missing nodes (plan B)", peer, ctx=buddy.healingCtx(env),
|
||||
nLevel=stats.level, nVisited=stats.count,
|
||||
nResult=stats.dangling.len, result=stats.dangling.toPC
|
||||
except:
|
||||
discard
|
||||
trace logTxt "missing nodes", peer,
|
||||
ctx=buddy.healingCtx(env), nLevel, nVisited,
|
||||
nResult=missing.len, result=missing.toPC
|
||||
|
||||
result = missing
|
||||
|
||||
|
||||
proc fetchMissingNodes(
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
## storage slots hexary trie. These per-account trie work items are stored in
|
||||
## the queue `env.fetchStoragePart`.
|
||||
##
|
||||
## Theere is another such queue `env.fetchStorageFull` which is not used here.
|
||||
## There is another such queue `env.fetchStorageFull` which is not used here.
|
||||
##
|
||||
## In order to be able to checkpoint the current list of storage accounts (by
|
||||
## a parallel running process), unfinished storage accounts are temporarily
|
||||
|
|
@ -24,20 +24,7 @@
|
|||
## Algorithm applied to each entry of `env.fetchStoragePart`
|
||||
## --------------------------------------------------------
|
||||
##
|
||||
## * Find dangling nodes in the current slot trie by trying plan A, and
|
||||
## continuing with plan B only if A fails.
|
||||
##
|
||||
## A. Try to find nodes with envelopes that have no slot in common with
|
||||
## any range interval of the `processed` set of the current slot trie. This
|
||||
## action will
|
||||
## + either determine that there are no such envelopes implying that the
|
||||
## current slot trie is complete (then stop here)
|
||||
## + or result in envelopes related to nodes that are all allocated on the
|
||||
## current slot trie (fail, use *plan B* below)
|
||||
## + or result in some envelopes related to dangling nodes.
|
||||
##
|
||||
## B. Employ the `hexaryInspect()` trie perusal function in a limited mode
|
||||
## for finding dangling (i.e. missing) sub-nodes below the allocated nodes.
|
||||
## * Find dangling nodes in the current slot trie via `findMissingNodes()`.
|
||||
##
|
||||
## * Install that nodes from the network.
|
||||
##
|
||||
|
|
@ -46,25 +33,22 @@
|
|||
## Discussion
|
||||
## ----------
|
||||
##
|
||||
## A worst case scenario of a failing *plan B* must be solved by fetching
|
||||
## and storing more slots and running this healing algorithm again.
|
||||
##
|
||||
## Due to the potentially poor performance using `hexaryInspect()`.there is
|
||||
## no general solution for *plan B* by recursively searching the whole slot
|
||||
## hexary trie database for more dangling nodes.
|
||||
## A worst case scenario of a portentally failing `findMissingNodes()` call
|
||||
## must be solved by fetching and storing more storage slots and running this
|
||||
## healing algorithm again.
|
||||
##
|
||||
import
|
||||
std/[math, sequtils, tables],
|
||||
std/[math, sequtils],
|
||||
chronicles,
|
||||
chronos,
|
||||
eth/[common, p2p, trie/nibbles, trie/trie_defs, rlp],
|
||||
eth/[common, p2p, trie/nibbles],
|
||||
stew/[byteutils, interval_set, keyed_queue],
|
||||
../../../../utils/prettify,
|
||||
"../../.."/[sync_desc, types],
|
||||
"../.."/[constants, range_desc, worker_desc],
|
||||
../com/[com_error, get_trie_nodes],
|
||||
../db/[hexary_desc, hexary_envelope, hexary_inspect, snapdb_storage_slots],
|
||||
./storage_queue_helper
|
||||
../db/[hexary_desc, hexary_envelope, snapdb_storage_slots],
|
||||
"."/[find_missing_nodes, storage_queue_helper]
|
||||
|
||||
{.push raises: [Defect].}
|
||||
|
||||
|
|
@ -144,7 +128,7 @@ proc compileMissingNodesList(
|
|||
kvp: SnapSlotsQueuePair;
|
||||
env: SnapPivotRef;
|
||||
): seq[NodeSpecs] =
|
||||
## Find some missing glue nodes in storage slots database to be fetched.
|
||||
## Find some missing glue nodes in storage slots database.
|
||||
let
|
||||
ctx = buddy.ctx
|
||||
peer = buddy.peer
|
||||
|
|
@ -152,45 +136,17 @@ proc compileMissingNodesList(
|
|||
rootKey = kvp.key.to(NodeKey)
|
||||
getFn = ctx.data.snapDb.getStorageSlotsFn(kvp.data.accKey)
|
||||
|
||||
var nodes: seq[NodeSpecs]
|
||||
noExceptionOops("compileMissingNodesList"):
|
||||
if not slots.processed.isEmpty:
|
||||
# Get unallocated nodes to be fetched
|
||||
let rc = slots.processed.hexaryEnvelopeDecompose(rootKey, getFn)
|
||||
if rc.isOk:
|
||||
nodes = rc.value
|
||||
|
||||
# Check whether the hexary trie is complete
|
||||
if nodes.len == 0:
|
||||
# Fill gaps
|
||||
discard slots.processed.merge(low(NodeTag),high(NodeTag))
|
||||
slots.unprocessed.clear()
|
||||
return
|
||||
|
||||
# Remove allocated nodes
|
||||
let missing = nodes.filterIt(it.nodeKey.ByteArray32.getFn().len == 0)
|
||||
if 0 < missing.len:
|
||||
when extraTraceMessages:
|
||||
trace logTxt "missing nodes", peer, ctx=buddy.healingCtx(kvp,env),
|
||||
nResult=missing.len, result=missing.toPC
|
||||
return missing
|
||||
|
||||
# Plan B, carefully employ `hexaryInspect()`
|
||||
if 0 < nodes.len:
|
||||
try:
|
||||
let
|
||||
paths = nodes.mapIt it.partialPath
|
||||
stats = getFn.hexaryInspectTrie(rootKey, paths,
|
||||
stopAtLevel = healStorageSlotsInspectionPlanBLevel,
|
||||
maxDangling = fetchRequestTrieNodesMax)
|
||||
result = stats.dangling
|
||||
if not slots.processed.isFull:
|
||||
noExceptionOops("compileMissingNodesList"):
|
||||
let (missing, nLevel, nVisited) = slots.findMissingNodes(
|
||||
rootKey, getFn, healStorageSlotsInspectionPlanBLevel)
|
||||
|
||||
when extraTraceMessages:
|
||||
trace logTxt "missing nodes (plan B)", peer,
|
||||
ctx=buddy.healingCtx(kvp,env), nLevel=stats.level,
|
||||
nVisited=stats.count, nResult=stats.dangling.len
|
||||
except:
|
||||
discard
|
||||
trace logTxt "missing nodes", peer,
|
||||
ctx=buddy.healingCtx(env), nLevel, nVisited,
|
||||
nResult=missing.len, result=missing.toPC
|
||||
|
||||
result = missing
|
||||
|
||||
|
||||
proc getNodesFromNetwork(
|
||||
|
|
|
|||
Loading…
Reference in New Issue