1659 lines
51 KiB
Nim
1659 lines
51 KiB
Nim
#
|
|
# Copyright (c) 2021 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or distributed
|
|
# except according to those terms.
|
|
|
|
import
|
|
std/[algorithm, hashes, options, sequtils, sets, strutils, strformat,
|
|
tables, times],
|
|
chronos,
|
|
eth/[common/eth_types, p2p, rlp],
|
|
eth/trie/[db, nibbles, trie_defs],
|
|
nimcrypto/keccak,
|
|
stew/byteutils,
|
|
stint,
|
|
rocksdb,
|
|
../../../constants,
|
|
../../../db/[kvstore_rocksdb, select_backend, storage_types],
|
|
"../.."/[protocol, types],
|
|
../range_desc,
|
|
./rocky_bulk_load
|
|
|
|
{.push raises: [Defect].}
|
|
|
|
logScope:
|
|
topics = "snap-proof"
|
|
|
|
const
|
|
BasicChainTrieDebugging = false
|
|
RepairTreeDebugging = false
|
|
|
|
type
|
|
AccountsDbError* = enum
|
|
NothingSerious = 0
|
|
AccountSmallerThanBase
|
|
AccountsNotSrictlyIncreasing
|
|
AccountRangesOverlap
|
|
AccountRepairBlocked
|
|
BoundaryProofFailed
|
|
Rlp2Or17ListEntries
|
|
RlpBlobExpected
|
|
RlpBranchLinkExpected
|
|
RlpEncoding
|
|
RlpExtPathEncoding
|
|
RlpNonEmptyBlobExpected
|
|
|
|
UnresolvedRepairNode
|
|
NoRocksDbBackend
|
|
CannotOpenRocksDbBulkSession
|
|
AddBulkItemFailed
|
|
CommitBulkItemsFailed
|
|
AccountNotFound
|
|
|
|
HexaryGetFn = proc(key: Blob): Blob {.gcsafe.}
|
|
## For testing/debugging
|
|
|
|
AccountsDbXKeyKind = enum
|
|
## Extends `storage_types.DbDBKeyKind` for testing/debugging
|
|
ChainDbStateRootPfx = 200 # <state-root> <hash-key> on trie db layer
|
|
RockyBulkStateRootPfx # <state-root> <hash-key> for rocksdb bulk load
|
|
RockyBulkHexary # <hash-key> for rocksdb bulk load
|
|
|
|
AccountsDbXKey = object
|
|
## Extends `storage_types.DbDBKey` for testing/debugging
|
|
data*: array[65, byte]
|
|
dataEndPos*: uint8 # the last populated position in the data
|
|
|
|
AccountLoadStats* = object
|
|
dura*: array[4,times.Duration] ## Accumulated time statistics
|
|
size*: array[2,uint64] ## Accumulated size statistics
|
|
|
|
ByteArray32* =
|
|
array[32,byte]
|
|
|
|
ByteArray33* =
|
|
array[33,byte]
|
|
|
|
NodeKey = ## Internal DB record reference type
|
|
distinct ByteArray32
|
|
|
|
RepairKey = ## Internal DB record, `byte & NodeKey`
|
|
distinct ByteArray33
|
|
|
|
RNodeKind = enum
|
|
Branch
|
|
Extension
|
|
Leaf
|
|
|
|
RNodeState = enum
|
|
Static = 0 ## Inserted as proof record
|
|
Locked ## Like `Static`, only added on-the-fly
|
|
Mutable ## Open for modification
|
|
|
|
RNodeRef = ref object
|
|
## For building a temporary repair tree
|
|
state: RNodeState ## `Static` if added as proof data
|
|
case kind: RNodeKind
|
|
of Leaf:
|
|
lPfx: NibblesSeq ## Portion of path segment
|
|
lData: Blob
|
|
of Extension:
|
|
ePfx: NibblesSeq ## Portion of path segment
|
|
eLink: RepairKey ## Single down link
|
|
of Branch:
|
|
bLink: array[16,RepairKey] ## Down links
|
|
bData: Blob
|
|
|
|
RPathStep = object
|
|
## For constructing tree traversal `seq[RPathStep]` path
|
|
key: RepairKey ## Tree label, node hash
|
|
node: RNodeRef ## Referes to data record
|
|
nibble: int8 ## Branch node selector (if any)
|
|
|
|
RPathXStep = object
|
|
## Extended `RPathStep` needed for `NodeKey` assignmant
|
|
pos: int ## Some position into `seq[RPathStep]`
|
|
step: RPathStep ## Modified copy of an `RPathStep`
|
|
canLock: bool ## Can set `Locked` state
|
|
|
|
RPath = object
|
|
path: seq[RPathStep]
|
|
tail: NibblesSeq ## Stands for non completed leaf path
|
|
|
|
RAccount = object
|
|
## Temporarily stashed account data. Proper account records have non-empty
|
|
## payload. Records with empty payload are lower boundary records.
|
|
pathTag: NodeTag ## Equivalent to account hash
|
|
nodeKey: RepairKey ## Leaf hash into hexary repair table
|
|
payload: Blob ## Data payload
|
|
|
|
RepairTreeDB = object
|
|
tab: Table[RepairKey,RNodeRef] ## Repair table
|
|
acc: seq[RAccount] ## Accounts to appprove of
|
|
repairKeyGen: uint64 ## Unique tmp key generator
|
|
|
|
AccountsDbRef* = ref object
|
|
db: TrieDatabaseRef ## General database
|
|
rocky: RocksStoreRef ## Set if rocksdb is available
|
|
aStats: AccountLoadStats ## Accumulated time and statistics
|
|
|
|
AccountsDbSessionRef* = ref object
|
|
keyMap: Table[RepairKey,uint] ## For debugging only (will go away)
|
|
base: AccountsDbRef ## Back reference to common parameters
|
|
rootKey: NodeKey ## Current root node
|
|
peer: Peer ## For log messages
|
|
rpDB: RepairTreeDB ## Repair database
|
|
dStats: AccountLoadStats ## Time and size statistics
|
|
|
|
const
|
|
EmptyBlob = seq[byte].default
|
|
EmptyNibbleRange = EmptyBlob.initNibbleRange
|
|
|
|
RockyBulkCache = "accounts.sst"
|
|
|
|
static:
|
|
# Not that there is no doubt about this ...
|
|
doAssert NodeKey.default.ByteArray32.initNibbleRange.len == 64
|
|
|
|
# Make sure that `DBKeyKind` extension does not overlap
|
|
doAssert high(DBKeyKind).int < low(AccountsDbXKeyKind).int
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc to(tag: NodeTag; T: type NodeKey): T =
|
|
tag.UInt256.toBytesBE.T
|
|
|
|
proc to(key: NodeKey; T: type UInt256): T =
|
|
T.fromBytesBE(key.ByteArray32)
|
|
|
|
proc to(key: NodeKey; T: type NodeTag): T =
|
|
key.to(UInt256).T
|
|
|
|
proc to(h: Hash256; T: type NodeKey): T =
|
|
h.data.T
|
|
|
|
proc to(key: NodeKey; T: type Hash256): T =
|
|
result.Hash256.data = key.ByteArray32
|
|
|
|
proc to(key: NodeKey; T: type NibblesSeq): T =
|
|
key.ByteArray32.initNibbleRange
|
|
|
|
proc to(key: NodeKey; T: type RepairKey): T =
|
|
(addr result.ByteArray33[1]).copyMem(unsafeAddr key.ByteArray32[0], 32)
|
|
|
|
proc to(tag: NodeTag; T: type RepairKey): T =
|
|
tag.to(NodeKey).to(RepairKey)
|
|
|
|
proc isZero[T: NodeTag|NodeKey|RepairKey](a: T): bool =
|
|
a == T.default
|
|
|
|
proc `==`(a, b: NodeKey): bool =
|
|
a.ByteArray32 == b.ByteArray32
|
|
|
|
proc `==`(a, b: RepairKey): bool =
|
|
a.ByteArray33 == b.ByteArray33
|
|
|
|
proc hash(a: NodeKey): Hash =
|
|
a.ByteArray32.hash
|
|
|
|
proc hash(a: RepairKey): Hash =
|
|
a.ByteArray33.hash
|
|
|
|
proc digestTo(data: Blob; T: type NodeKey): T =
|
|
keccak256.digest(data).data.T
|
|
|
|
proc isNodeKey(a: RepairKey): bool =
|
|
a.ByteArray33[0] == 0
|
|
|
|
proc newRepairKey(ps: AccountsDbSessionRef): RepairKey =
|
|
ps.rpDB.repairKeyGen.inc
|
|
var src = ps.rpDB.repairKeyGen.toBytesBE
|
|
(addr result.ByteArray33[25]).copyMem(addr src[0], 8)
|
|
result.ByteArray33[0] = 1
|
|
|
|
proc init(key: var NodeKey; data: openArray[byte]): bool =
|
|
key.reset
|
|
if data.len <= 32:
|
|
if 0 < data.len:
|
|
let trg = addr key.ByteArray32[32 - data.len]
|
|
trg.copyMem(unsafeAddr data[0], data.len)
|
|
return true
|
|
|
|
proc dup(node: RNodeRef): RNodeRef =
|
|
new result
|
|
result[] = node[]
|
|
|
|
proc convertTo(data: openArray[byte]; T: type NodeKey): T =
|
|
discard result.init(data)
|
|
|
|
proc convertTo(key: RepairKey; T: type NodeKey): T =
|
|
if key.isNodeKey:
|
|
discard result.init(key.ByteArray33[1 .. 32])
|
|
|
|
proc convertTo(key: RepairKey; T: type NodeTag): T =
|
|
if key.isNodeKey:
|
|
result = UInt256.fromBytesBE(key.ByteArray33[1 .. 32]).T
|
|
|
|
proc convertTo(node: RNodeRef; T: type Blob): T =
|
|
var writer = initRlpWriter()
|
|
|
|
proc appendOk(writer: var RlpWriter; key: RepairKey): bool =
|
|
if key.isZero:
|
|
writer.append(EmptyBlob)
|
|
elif key.isNodeKey:
|
|
var hash: Hash256
|
|
(addr hash.data[0]).copyMem(unsafeAddr key.ByteArray33[1], 32)
|
|
writer.append(hash)
|
|
else:
|
|
return false
|
|
true
|
|
|
|
case node.kind:
|
|
of Branch:
|
|
writer.startList(17)
|
|
for n in 0 ..< 16:
|
|
if not writer.appendOk(node.bLink[n]):
|
|
return # empty `Blob`
|
|
writer.append(node.bData)
|
|
of Extension:
|
|
writer.startList(2)
|
|
writer.append(node.ePfx.hexPrefixEncode(isleaf = false))
|
|
if not writer.appendOk(node.eLink):
|
|
return # empty `Blob`
|
|
of Leaf:
|
|
writer.startList(2)
|
|
writer.append(node.lPfx.hexPrefixEncode(isleaf = true))
|
|
writer.append(node.lData)
|
|
|
|
writer.finish()
|
|
|
|
template noKeyError(info: static[string]; code: untyped) =
|
|
try:
|
|
code
|
|
except KeyError as e:
|
|
raiseAssert "Not possible (" & info & "): " & e.msg
|
|
|
|
template elapsed(duration: times.Duration; code: untyped) =
|
|
block:
|
|
let start = getTime()
|
|
block:
|
|
code
|
|
duration = getTime() - start
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private getters & setters
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc xPfx(node: RNodeRef): NibblesSeq =
|
|
case node.kind:
|
|
of Leaf:
|
|
return node.lPfx
|
|
of Extension:
|
|
return node.ePfx
|
|
of Branch:
|
|
doAssert node.kind != Branch # Ooops
|
|
|
|
proc `xPfx=`(node: RNodeRef, val: NibblesSeq) =
|
|
case node.kind:
|
|
of Leaf:
|
|
node.lPfx = val
|
|
of Extension:
|
|
node.ePfx = val
|
|
of Branch:
|
|
doAssert node.kind != Branch # Ooops
|
|
|
|
proc xData(node: RNodeRef): Blob =
|
|
case node.kind:
|
|
of Branch:
|
|
return node.bData
|
|
of Leaf:
|
|
return node.lData
|
|
of Extension:
|
|
doAssert node.kind != Extension # Ooops
|
|
|
|
proc `xData=`(node: RNodeRef; val: Blob) =
|
|
case node.kind:
|
|
of Branch:
|
|
node.bData = val
|
|
of Leaf:
|
|
node.lData = val
|
|
of Extension:
|
|
doAssert node.kind != Extension # Ooops
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private debugging helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
template noPpError(info: static[string]; code: untyped) =
|
|
try:
|
|
code
|
|
except ValueError as e:
|
|
raiseAssert "Inconveivable (" & info & "): " & e.msg
|
|
except KeyError as e:
|
|
raiseAssert "Not possible (" & info & "): " & e.msg
|
|
except OsError as e:
|
|
raiseAssert "Ooops (" & info & "): " & e.msg
|
|
|
|
proc pp(s: string; hex = false): string =
|
|
if hex:
|
|
let n = (s.len + 1) div 2
|
|
(if s.len < 20: s else: s[0 .. 5] & ".." & s[s.len-8 .. s.len-1]) &
|
|
"[" & (if 0 < n: "#" & $n else: "") & "]"
|
|
elif s.len <= 30:
|
|
s
|
|
else:
|
|
(if (s.len and 1) == 0: s[0 ..< 8] else: "0" & s[0 ..< 7]) &
|
|
"..(" & $s.len & ").." & s[s.len-16 ..< s.len]
|
|
|
|
proc pp(a: Hash256; collapse = true): string =
|
|
if not collapse:
|
|
a.data.mapIt(it.toHex(2)).join.toLowerAscii
|
|
elif a == emptyRlpHash:
|
|
"emptyRlpHash"
|
|
elif a == blankStringHash:
|
|
"blankStringHash"
|
|
else:
|
|
a.data.mapIt(it.toHex(2)).join[56 .. 63].toLowerAscii
|
|
|
|
proc pp(a: NodeKey; collapse = true): string =
|
|
Hash256(data: a.ByteArray32).pp(collapse)
|
|
|
|
# ---------
|
|
|
|
proc toKey(a: RepairKey; ps: AccountsDbSessionRef): uint =
|
|
if not a.isZero:
|
|
noPpError("pp(RepairKey)"):
|
|
if not ps.keyMap.hasKey(a):
|
|
ps.keyMap[a] = ps.keyMap.len.uint + 1
|
|
result = ps.keyMap[a]
|
|
|
|
proc toKey(a: NodeKey; ps: AccountsDbSessionRef): uint =
|
|
a.to(RepairKey).toKey(ps)
|
|
|
|
proc toKey(a: NodeTag; ps: AccountsDbSessionRef): uint =
|
|
a.to(NodeKey).toKey(ps)
|
|
|
|
|
|
proc pp(a: NodeKey; ps: AccountsDbSessionRef): string =
|
|
if a.isZero: "ø" else:"$" & $a.toKey(ps)
|
|
|
|
proc pp(a: RepairKey; ps: AccountsDbSessionRef): string =
|
|
if a.isZero: "ø" elif a.isNodeKey: "$" & $a.toKey(ps) else: "¶" & $a.toKey(ps)
|
|
|
|
proc pp(a: NodeTag; ps: AccountsDbSessionRef): string =
|
|
a.to(NodeKey).pp(ps)
|
|
|
|
# ---------
|
|
|
|
proc pp(q: openArray[byte]; noHash = false): string =
|
|
if q.len == 32 and not noHash:
|
|
var a: array[32,byte]
|
|
for n in 0..31: a[n] = q[n]
|
|
($Hash256(data: a)).pp
|
|
else:
|
|
q.toSeq.mapIt(it.toHex(2)).join.toLowerAscii.pp(hex = true)
|
|
|
|
proc pp(blob: Blob): string =
|
|
blob.mapIt(it.toHex(2)).join
|
|
|
|
proc pp(a: Account): string =
|
|
noPpError("pp(Account)"):
|
|
result = &"({a.nonce},{a.balance},{a.storageRoot},{a.codeHash})"
|
|
|
|
proc pp(sa: SnapAccount): string =
|
|
"(" & $sa.accHash & "," & sa.accBody.pp & ")"
|
|
|
|
proc pp(al: seq[SnapAccount]): string =
|
|
result = " @["
|
|
noPpError("pp(seq[SnapAccount])"):
|
|
for n,rec in al:
|
|
result &= &"| # <{n}>| {rec.pp},"
|
|
if 10 < result.len:
|
|
result[^1] = ']'
|
|
else:
|
|
result &= "]"
|
|
|
|
proc pp(blobs: seq[Blob]): string =
|
|
result = " @["
|
|
noPpError("pp(seq[Blob])"):
|
|
for n,rec in blobs:
|
|
result &= "| # <" & $n & ">| \"" & rec.pp & "\".hexToSeqByte,"
|
|
if 10 < result.len:
|
|
result[^1] = ']'
|
|
else:
|
|
result &= "]"
|
|
|
|
proc pp(branch: array[17,Blob]; ps: AccountsDbSessionRef): string =
|
|
result = "["
|
|
noPpError("pp(array[17,Blob])"):
|
|
for a in 0 .. 15:
|
|
result &= branch[a].convertTo(NodeKey).pp(ps) & ","
|
|
result &= branch[16].pp & "]"
|
|
|
|
proc pp(branch: array[16,RepairKey]; ps: AccountsDbSessionRef): string =
|
|
result = "["
|
|
noPpError("pp(array[17,Blob])"):
|
|
for a in 0 .. 15:
|
|
result &= branch[a].pp(ps) & ","
|
|
result[^1] = ']'
|
|
|
|
proc pp(hs: seq[NodeKey]; ps: AccountsDbSessionRef): string =
|
|
"<" & hs.mapIt(it.pp(ps)).join(",") & ">"
|
|
|
|
proc pp(hs: HashSet[NodeKey]; ps: AccountsDbSessionRef): string =
|
|
"{" &
|
|
toSeq(hs.items).mapIt(it.toKey(ps)).sorted.mapIt("$" & $it).join(",") & "}"
|
|
|
|
proc pp(w: NibblesSeq): string =
|
|
$w
|
|
|
|
proc pp(n: RNodeRef; ps: AccountsDbSessionRef): string =
|
|
proc ppStr(blob: Blob): string =
|
|
if blob.len == 0: "" else: blob.pp.pp(hex = true)
|
|
noPpError("pp(RNodeRef)"):
|
|
let so = n.state.ord
|
|
case n.kind:
|
|
of Leaf:
|
|
result = ["l","ł","L"][so] & &"({n.lPfx.pp},{n.lData.ppStr})"
|
|
of Extension:
|
|
result = ["e","€","E"][so] & &"({n.ePfx.pp},{n.eLink.pp(ps)})"
|
|
of Branch:
|
|
result = ["b","þ","B"][so] & &"({n.bLink.pp(ps)},{n.bData.ppStr})"
|
|
|
|
proc pp(w: RPathStep; ps: AccountsDbSessionRef): string =
|
|
noPpError("pp(seq[(NodeKey,RNodeRef)])"):
|
|
let nibble = if 0 <= w.nibble: &"{w.nibble:x}" else: "ø"
|
|
result = &"({w.key.pp(ps)},{nibble},{w.node.pp(ps)})"
|
|
|
|
proc pp(w: openArray[RPathStep]; ps: AccountsDbSessionRef; indent = 4): string =
|
|
let pfx = "\n" & " ".repeat(indent)
|
|
noPpError("pp(seq[(NodeKey,RNodeRef)])"):
|
|
result = w.toSeq.mapIt(it.pp(ps)).join(pfx)
|
|
|
|
proc pp(w: RPath; ps: AccountsDbSessionRef; indent = 4): string =
|
|
let pfx = "\n" & " ".repeat(indent)
|
|
noPpError("pp(RPath)"):
|
|
result = w.path.pp(ps,indent) & &"{pfx}({w.tail.pp})"
|
|
|
|
proc pp(w: RPathXStep; ps: AccountsDbSessionRef): string =
|
|
noPpError("pp(seq[(int,RPathStep)])"):
|
|
let y = if w.canLock: "lockOk" else: "noLock"
|
|
result = &"({w.pos},{y},{w.step.pp(ps)})"
|
|
|
|
proc pp(w: seq[RPathXStep]; ps: AccountsDbSessionRef; indent = 4): string =
|
|
let pfx = "\n" & " ".repeat(indent)
|
|
noPpError("pp(seq[RPathXStep])"):
|
|
result = w.mapIt(it.pp(ps)).join(pfx)
|
|
|
|
proc pp(a: DbKey|AccountsDbXKey): string =
|
|
a.data.toSeq.mapIt(it.toHex(2)).join
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private helpers for bulk load testing
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc chainDbHexaryKey*(a: RepairKey): ByteArray32 =
|
|
a.convertTo(NodeKey).ByteArray32
|
|
|
|
proc chainDbStateRootPfxKey*(a: NodeKey; b: NodeTag): AccountsDbXKey =
|
|
result.data[0] = byte ord(ChainDbStateRootPfx)
|
|
result.data[1 .. 32] = a.ByteArray32
|
|
result.data[33 .. 64] = b.to(NodeKey).ByteArray32
|
|
result.dataEndPos = uint8 64
|
|
|
|
proc rockyBulkHexaryKey(a: NodeTag): DbKey =
|
|
result.data[0] = byte ord(RockyBulkHexary)
|
|
result.data[1 .. 32] = a.to(NodeKey).ByteArray32
|
|
result.dataEndPos = uint8 32
|
|
|
|
proc rockyBulkStateRootPfxKey*(a: NodeKey; b: NodeTag): AccountsDbXKey =
|
|
result.data[0] = byte ord(RockyBulkStateRootPfx)
|
|
result.data[1 .. 32] = a.ByteArray32
|
|
result.data[33 .. 64] = b.to(NodeKey).ByteArray32
|
|
result.dataEndPos = uint8 64
|
|
|
|
template toOpenArray*(k: AccountsDbXKey): openArray[byte] =
|
|
k.data.toOpenArray(0, int(k.dataEndPos))
|
|
|
|
template toOpenArray*(k: ByteArray32): openArray[byte] =
|
|
k.toOpenArray(0, 31)
|
|
|
|
|
|
proc storeAccountPathsOnChainDb(
|
|
ps: AccountsDbSessionRef
|
|
): Result[void,AccountsDbError] =
|
|
let dbTx = ps.base.db.beginTransaction
|
|
defer: dbTx.commit
|
|
|
|
for a in ps.rpDB.acc:
|
|
if a.payload.len != 0:
|
|
ps.base.db.put(
|
|
ps.rootKey.chainDbStateRootPfxKey(a.pathTag).toOpenArray, a.payload)
|
|
ok()
|
|
|
|
proc storeHexaryNodesOnChainDb(
|
|
ps: AccountsDbSessionRef
|
|
): Result[void,AccountsDbError] =
|
|
let dbTx = ps.base.db.beginTransaction
|
|
defer: dbTx.commit
|
|
|
|
for (key,value) in ps.rpDB.tab.pairs:
|
|
if not key.isNodeKey:
|
|
let error = UnresolvedRepairNode
|
|
trace "Unresolved node in repair table", error
|
|
return err(error)
|
|
ps.base.db.put(key.chainDbHexaryKey.toOpenArray, value.convertTo(Blob))
|
|
ok()
|
|
|
|
proc storeAccountPathsOnRockyDb(
|
|
ps: AccountsDbSessionRef
|
|
): Result[void,AccountsDbError]
|
|
{.gcsafe, raises: [Defect,OSError,ValueError].} =
|
|
if ps.base.rocky.isNil:
|
|
return err(NoRocksDbBackend)
|
|
let bulker = RockyBulkLoadRef.init(ps.base.rocky)
|
|
defer: bulker.destroy()
|
|
if not bulker.begin(RockyBulkCache):
|
|
let error = CannotOpenRocksDbBulkSession
|
|
trace "Rocky accounts session initiation failed",
|
|
error, info=bulker.lastError()
|
|
return err(error)
|
|
|
|
for n,a in ps.rpDB.acc:
|
|
if a.payload.len != 0:
|
|
let key = ps.rootKey.rockyBulkStateRootPfxKey(a.pathTag)
|
|
if not bulker.add(key.toOpenArray, a.payload):
|
|
let error = AddBulkItemFailed
|
|
trace "Rocky accounts bulk load failure",
|
|
n, len=ps.rpDB.acc.len, error, info=bulker.lastError()
|
|
return err(error)
|
|
|
|
if bulker.finish().isErr:
|
|
let error = CommitBulkItemsFailed
|
|
trace "Rocky accounts commit failure",
|
|
len=ps.rpDB.acc.len, error, info=bulker.lastError()
|
|
return err(error)
|
|
ok()
|
|
|
|
|
|
proc storeHexaryNodesOnRockyDb(
|
|
ps: AccountsDbSessionRef
|
|
): Result[void,AccountsDbError]
|
|
{.gcsafe, raises: [Defect,OSError,KeyError,ValueError].} =
|
|
if ps.base.rocky.isNil:
|
|
return err(NoRocksDbBackend)
|
|
let bulker = RockyBulkLoadRef.init(ps.base.rocky)
|
|
defer: bulker.destroy()
|
|
if not bulker.begin(RockyBulkCache):
|
|
let error = CannotOpenRocksDbBulkSession
|
|
trace "Rocky hexary session initiation failed",
|
|
error, info=bulker.lastError()
|
|
return err(error)
|
|
|
|
#let keyList = toSeq(ps.rpDB.tab.keys)
|
|
# .filterIt(it.isNodeKey)
|
|
# .mapIt(it.convertTo(NodeTag))
|
|
# .sorted(cmp)
|
|
var
|
|
keyList = newSeq[NodeTag](ps.rpDB.tab.len)
|
|
inx = 0
|
|
for repairKey in ps.rpDB.tab.keys:
|
|
if repairKey.isNodeKey:
|
|
keyList[inx] = repairKey.convertTo(NodeTag)
|
|
inx.inc
|
|
if inx < ps.rpDB.tab.len:
|
|
return err(UnresolvedRepairNode)
|
|
keyList.sort(cmp)
|
|
|
|
for n,nodeTag in keyList:
|
|
let
|
|
key = nodeTag.rockyBulkHexaryKey()
|
|
data = ps.rpDB.tab[nodeTag.to(RepairKey)].convertTo(Blob)
|
|
if not bulker.add(key.toOpenArray, data):
|
|
let error = AddBulkItemFailed
|
|
trace "Rocky hexary bulk load failure",
|
|
n, len=ps.rpDB.tab.len, error, info=bulker.lastError()
|
|
return err(error)
|
|
|
|
if bulker.finish().isErr:
|
|
let error = CommitBulkItemsFailed
|
|
trace "Rocky hexary commit failure",
|
|
len=ps.rpDB.acc.len, error, info=bulker.lastError()
|
|
return err(error)
|
|
ok()
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private functions
|
|
# ------------------------------------------------------------------------------
|
|
|
|
# Example trie from https://eth.wiki/en/fundamentals/patricia-tree
|
|
#
|
|
# lookup data:
|
|
# "do": "verb"
|
|
# "dog": "puppy"
|
|
# "dodge": "coin"
|
|
# "horse": "stallion"
|
|
#
|
|
# trie DB:
|
|
# root: [16 A]
|
|
# A: [* * * * B * * * [20+"orse" "stallion"] * * * * * * * *]
|
|
# B: [00+"o" D]
|
|
# D: [* * * * * * E * * * * * * * * * "verb"]
|
|
# E: [17 [* * * * * * [35 "coin"] * * * * * * * * * "puppy"]]
|
|
#
|
|
# with first nibble of two-column rows:
|
|
# hex bits | node type length
|
|
# ---------+------------------
|
|
# 0 0000 | extension even
|
|
# 1 0001 | extension odd
|
|
# 2 0010 | leaf even
|
|
# 3 0011 | leaf odd
|
|
#
|
|
# and key path:
|
|
# "do": 6 4 6 f
|
|
# "dog": 6 4 6 f 6 7
|
|
# "dodge": 6 4 6 f 6 7 6 5
|
|
# "horse": 6 8 6 f 7 2 7 3 6 5
|
|
#
|
|
|
|
proc hexaryImport(
|
|
ps: AccountsDbSessionRef;
|
|
recData: Blob
|
|
): Result[void,AccountsDbError]
|
|
{.gcsafe, raises: [Defect, RlpError].} =
|
|
## Decode a single trie item for adding to the table and add it to the
|
|
## database. Branch and exrension record links are collected.
|
|
let
|
|
nodeKey = recData.digestTo(NodeKey)
|
|
repairKey = nodeKey.to(RepairKey) # for repair table
|
|
var
|
|
rlp = recData.rlpFromBytes
|
|
blobs = newSeq[Blob](2) # temporary, cache
|
|
links: array[16,RepairKey] # reconstruct branch node
|
|
blob16: Blob # reconstruct branch node
|
|
top = 0 # count entries
|
|
rNode: RNodeRef # repair tree node
|
|
|
|
# Collect lists of either 2 or 17 blob entries.
|
|
for w in rlp.items:
|
|
case top
|
|
of 0, 1:
|
|
if not w.isBlob:
|
|
return err(RlpBlobExpected)
|
|
blobs[top] = rlp.read(Blob)
|
|
of 2 .. 15:
|
|
var key: NodeKey
|
|
if not key.init(rlp.read(Blob)):
|
|
return err(RlpBranchLinkExpected)
|
|
# Update ref pool
|
|
links[top] = key.to(RepairKey)
|
|
of 16:
|
|
if not w.isBlob:
|
|
return err(RlpBlobExpected)
|
|
blob16 = rlp.read(Blob)
|
|
else:
|
|
return err(Rlp2Or17ListEntries)
|
|
top.inc
|
|
|
|
# Verify extension data
|
|
case top
|
|
of 2:
|
|
if blobs[0].len == 0:
|
|
return err(RlpNonEmptyBlobExpected)
|
|
let (isLeaf, pathSegment) = hexPrefixDecode blobs[0]
|
|
if isLeaf:
|
|
rNode = RNodeRef(
|
|
kind: Leaf,
|
|
lPfx: pathSegment,
|
|
lData: blobs[1])
|
|
else:
|
|
var key: NodeKey
|
|
if not key.init(blobs[1]):
|
|
return err(RlpExtPathEncoding)
|
|
# Update ref pool
|
|
rNode = RNodeRef(
|
|
kind: Extension,
|
|
ePfx: pathSegment,
|
|
eLink: key.to(RepairKey))
|
|
of 17:
|
|
for n in [0,1]:
|
|
var key: NodeKey
|
|
if not key.init(blobs[n]):
|
|
return err(RlpBranchLinkExpected)
|
|
# Update ref pool
|
|
links[n] = key.to(RepairKey)
|
|
rNode = RNodeRef(
|
|
kind: Branch,
|
|
bLink: links,
|
|
bData: blob16)
|
|
else:
|
|
discard
|
|
|
|
# Add to repair database
|
|
ps.rpDB.tab[repairKey] = rNode
|
|
|
|
# Add to hexary trie database -- disabled, using bulk import later
|
|
#ps.base.db.put(nodeKey.ByteArray32, recData)
|
|
|
|
when RepairTreeDebugging:
|
|
# Rebuild blob from repair record
|
|
let nodeBlob = rNode.convertTo(Blob)
|
|
if nodeBlob != recData:
|
|
echo "*** hexaryImport oops:",
|
|
" kind=", rNode.kind,
|
|
" key=", repairKey.pp(ps),
|
|
" nodeBlob=", nodeBlob.pp,
|
|
" recData=", recData.pp
|
|
doAssert nodeBlob == recData
|
|
|
|
ok()
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private functions, repair tree action helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc rTreeExtendLeaf(
|
|
ps: AccountsDbSessionRef;
|
|
rPath: RPath;
|
|
key: RepairKey
|
|
): RPath =
|
|
## Append a `Leaf` node to a `Branch` node (see `rTreeExtend()`.)
|
|
if 0 < rPath.tail.len:
|
|
let
|
|
nibble = rPath.path[^1].nibble
|
|
leaf = RNodeRef(
|
|
state: Mutable,
|
|
kind: Leaf,
|
|
lPfx: rPath.tail)
|
|
ps.rpDB.tab[key] = leaf
|
|
if not key.isNodeKey:
|
|
rPath.path[^1].node.bLink[nibble] = key
|
|
return RPath(
|
|
path: rPath.path & RPathStep(key: key, node: leaf, nibble: -1),
|
|
tail: EmptyNibbleRange)
|
|
|
|
proc rTreeExtendLeaf(
|
|
ps: AccountsDbSessionRef;
|
|
rPath: RPath;
|
|
key: RepairKey;
|
|
node: RNodeRef
|
|
): RPath =
|
|
## Register `node` and append/link a `Leaf` node to a `Branch` node (see
|
|
## `rTreeExtend()`.)
|
|
if 1 < rPath.tail.len and node.state == Mutable:
|
|
let
|
|
nibble = rPath.tail[0].int8
|
|
xStep = RPathStep(key: key, node: node, nibble: nibble)
|
|
xPath = RPath(path: rPath.path & xStep, tail: rPath.tail.slice(1))
|
|
return ps.rTreeExtendLeaf(xPath, ps.newRepairKey())
|
|
|
|
|
|
proc rTreeSplitNode(
|
|
ps: AccountsDbSessionRef;
|
|
rPath: RPath;
|
|
key: RepairKey;
|
|
node: RNodeRef
|
|
): RPath =
|
|
## Replace `Leaf` or `Extension` node in tuple `(key,node)` by parts (see
|
|
## `rTreeExtend()`):
|
|
##
|
|
## left(Extension) -> middle(Branch) -> right(Extension or Leaf)
|
|
## ^ ^
|
|
## | |
|
|
## added-to-path added-to-path
|
|
##
|
|
## where either `left()` or `right()` extensions might be missing.
|
|
##
|
|
let
|
|
nibbles = node.xPfx
|
|
lLen = rPath.tail.sharedPrefixLen(nibbles)
|
|
if nibbles.len == 0 or rPath.tail.len <= lLen:
|
|
return # Ooops (^^^^^ otherwise `rPath` was not the longest)
|
|
var
|
|
mKey = key
|
|
let
|
|
mNibble = nibbles[lLen] # exists as `lLen < tail.len`
|
|
rPfx = nibbles.slice(lLen + 1) # might be empty OK
|
|
|
|
result = rPath
|
|
|
|
# Insert node (if any): left(Extension)
|
|
if 0 < lLen:
|
|
let lNode = RNodeRef(
|
|
state: Mutable,
|
|
kind: Extension,
|
|
ePfx: result.tail.slice(0,lLen),
|
|
eLink: ps.newRepairKey())
|
|
ps.rpDB.tab[key] = lNode
|
|
result.path.add RPathStep(key: key, node: lNode, nibble: -1)
|
|
result.tail = result.tail.slice(lLen)
|
|
mKey = lNode.eLink
|
|
|
|
# Insert node: middle(Branch)
|
|
let mNode = RNodeRef(
|
|
state: Mutable,
|
|
kind: Branch)
|
|
ps.rpDB.tab[mKey] = mNode
|
|
result.path.add RPathStep(key: mKey, node: mNode, nibble: -1) # no nibble yet
|
|
|
|
# Insert node (if any): right(Extension) -- not to be registered in `rPath`
|
|
if 0 < rPfx.len:
|
|
let rKey = ps.newRepairKey()
|
|
# Re-use argument node
|
|
mNode.bLink[mNibble] = rKey
|
|
ps.rpDB.tab[rKey] = node
|
|
node.xPfx = rPfx
|
|
# Otherwise merge argument node
|
|
elif node.kind == Extension:
|
|
mNode.bLink[mNibble] = node.eLink
|
|
else:
|
|
# Oops, does it make sense, at all?
|
|
mNode.bData = node.lData
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private functions, repair tree actions
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc rTreeFollow(nodeKey: NodeKey; ps: AccountsDbSessionRef): RPath =
|
|
## Compute logest possible path matching the `nodeKey` nibbles.
|
|
result.tail = nodeKey.to(NibblesSeq)
|
|
noKeyError("rTreeFollow"):
|
|
var key = ps.rootKey.to(RepairKey)
|
|
while ps.rpDB.tab.hasKey(key) and 0 < result.tail.len:
|
|
let node = ps.rpDB.tab[key]
|
|
case node.kind:
|
|
of Leaf:
|
|
if result.tail.len == result.tail.sharedPrefixLen(node.lPfx):
|
|
# Bingo, got full path
|
|
result.path.add RPathStep(key: key, node: node, nibble: -1)
|
|
result.tail = EmptyNibbleRange
|
|
return
|
|
of Branch:
|
|
let nibble = result.tail[0].int8
|
|
if node.bLink[nibble].isZero:
|
|
return
|
|
result.path.add RPathStep(key: key, node: node, nibble: nibble)
|
|
result.tail = result.tail.slice(1)
|
|
key = node.bLink[nibble]
|
|
of Extension:
|
|
if node.ePfx.len != result.tail.sharedPrefixLen(node.ePfx):
|
|
return
|
|
result.path.add RPathStep(key: key, node: node, nibble: -1)
|
|
result.tail = result.tail.slice(node.ePfx.len)
|
|
key = node.eLink
|
|
|
|
proc rTreeFollow(nodeTag: NodeTag; ps: AccountsDbSessionRef): RPath =
|
|
## Variant of `rTreeFollow()`
|
|
nodeTag.to(NodeKey).rTreeFollow(ps)
|
|
|
|
|
|
proc rTreeInterpolate(rPath: RPath; ps: AccountsDbSessionRef): RPath =
|
|
## Extend path, add missing nodes to tree. The last node added will be
|
|
## a `Leaf` node if this function succeeds.
|
|
##
|
|
## The function assumed that the `RPath` argument is the longest possible
|
|
## as just constructed by `rTreeFollow()`
|
|
if 0 < rPath.path.len and 0 < rPath.tail.len:
|
|
noKeyError("rTreeExtend"):
|
|
let step = rPath.path[^1]
|
|
case step.node.kind:
|
|
of Branch:
|
|
# Now, the slot must not be empty. An empty slot would lead to a
|
|
# rejection of this record as last valid step, contrary to the
|
|
# assumption `path` is the longest one.
|
|
if step.nibble < 0:
|
|
return # sanitary check failed
|
|
let key = step.node.bLink[step.nibble]
|
|
if key.isZero:
|
|
return # sanitary check failed
|
|
|
|
# Case: unused slot => add leaf record
|
|
if not ps.rpDB.tab.hasKey(key):
|
|
return ps.rTreeExtendLeaf(rPath, key)
|
|
|
|
# So a `child` node exits but it is something that could not be used to
|
|
# extend the argument `path` which is assumed the longest possible one.
|
|
let child = ps.rpDB.tab[key]
|
|
case child.kind:
|
|
of Branch:
|
|
# So a `Leaf` node can be linked into the `child` branch
|
|
return ps.rTreeExtendLeaf(rPath, key, child)
|
|
|
|
# Need to split the right `grandChild` in `child -> grandChild`
|
|
# into parts:
|
|
#
|
|
# left(Extension) -> middle(Branch)
|
|
# | |
|
|
# | +-----> right(Extension or Leaf) ...
|
|
# +---------> new Leaf record
|
|
#
|
|
# where either `left()` or `right()` extensions might be missing
|
|
of Extension, Leaf:
|
|
var xPath = ps.rTreeSplitNode(rPath, key, child)
|
|
if 0 < xPath.path.len:
|
|
# Append `Leaf` node
|
|
xPath.path[^1].nibble = xPath.tail[0].int8
|
|
xPath.tail = xPath.tail.slice(1)
|
|
return ps.rTreeExtendLeaf(xPath, ps.newRepairKey())
|
|
of Leaf:
|
|
return # Oops
|
|
of Extension:
|
|
let key = step.node.eLink
|
|
|
|
var child: RNodeRef
|
|
if ps.rpDB.tab.hasKey(key):
|
|
child = ps.rpDB.tab[key]
|
|
# `Extension` can only be followed by a `Branch` node
|
|
if child.kind != Branch:
|
|
return
|
|
else:
|
|
# Case: unused slot => add `Branch` and `Leaf` record
|
|
child = RNodeRef(
|
|
state: Mutable,
|
|
kind: Branch)
|
|
ps.rpDB.tab[key] = child
|
|
|
|
# So a `Leaf` node can be linked into the `child` branch
|
|
return ps.rTreeExtendLeaf(rPath, key, child)
|
|
|
|
|
|
proc rTreeInterpolate(
|
|
rPath: RPath;
|
|
ps: AccountsDbSessionRef;
|
|
payload: Blob
|
|
): RPath =
|
|
## Variant of `rTreeExtend()` which completes a `Leaf` record.
|
|
result = rPath.rTreeInterpolate(ps)
|
|
if 0 < result.path.len and result.tail.len == 0:
|
|
let node = result.path[^1].node
|
|
if node.kind != Extension and node.state == Mutable:
|
|
node.xData = payload
|
|
|
|
|
|
proc rTreeUpdateKeys(rPath: RPath; ps: AccountsDbSessionRef): Result[void,int] =
|
|
## The argument `rPath` is assumed to organise database nodes as
|
|
##
|
|
## root -> ... -> () -> () -> ... -> () -> () ...
|
|
## |-------------| |------------| |------
|
|
## static nodes locked nodes mutable nodes
|
|
##
|
|
## Where
|
|
## * Static nodes are read-only nodes provided by the proof database
|
|
## * Locked nodes are added read-only nodes that satisfy the proof condition
|
|
## * Mutable nodes are incomplete nodes
|
|
##
|
|
## Then update nodes from the right end and set all the mutable nodes
|
|
## locked if possible.
|
|
var
|
|
rTop = rPath.path.len
|
|
stack: seq[RPathXStep]
|
|
|
|
if 0 < rTop and
|
|
rPath.path[^1].node.state == Mutable and
|
|
rPath.path[0].node.state != Mutable:
|
|
|
|
# Set `Leaf` entry
|
|
let leafNode = rPath.path[^1].node.dup
|
|
stack.add RPathXStep(
|
|
pos: rTop - 1,
|
|
canLock: true,
|
|
step: RPathStep(
|
|
node: leafNode,
|
|
key: leafNode.convertTo(Blob).digestTo(NodeKey).to(RepairKey),
|
|
nibble: -1))
|
|
|
|
while true:
|
|
rTop.dec
|
|
|
|
# Update parent node (note that `2 <= rPath.path.len`)
|
|
let
|
|
thisKey = stack[^1].step.key
|
|
preStep = rPath.path[rTop-1]
|
|
preNibble = preStep.nibble
|
|
|
|
# End reached
|
|
if preStep.node.state != Mutable:
|
|
|
|
# Verify the tail matches
|
|
var key = RepairKey.default
|
|
case preStep.node.kind:
|
|
of Branch:
|
|
key = preStep.node.bLink[preNibble]
|
|
of Extension:
|
|
key = preStep.node.eLink
|
|
of Leaf:
|
|
discard
|
|
if key != thisKey:
|
|
return err(rTop-1)
|
|
|
|
when RepairTreeDebugging:
|
|
echo "*** rTreeUpdateKeys",
|
|
" rPath\n ", rPath.pp(ps),
|
|
"\n stack\n ", stack.pp(ps)
|
|
|
|
# Ok, replace database records by stack entries
|
|
var lockOk = true
|
|
for n in countDown(stack.len-1,0):
|
|
let item = stack[n]
|
|
ps.rpDB.tab.del(rPath.path[item.pos].key)
|
|
ps.rpDB.tab[item.step.key] = item.step.node
|
|
if lockOk:
|
|
if item.canLock:
|
|
item.step.node.state = Locked
|
|
else:
|
|
lockOk = false
|
|
if not lockOk:
|
|
return err(rTop-1) # repeat
|
|
break # Done ok()
|
|
|
|
stack.add RPathXStep(
|
|
pos: rTop - 1,
|
|
step: RPathStep(
|
|
node: preStep.node.dup, # (!)
|
|
nibble: preNibble,
|
|
key: preStep.key))
|
|
|
|
case stack[^1].step.node.kind:
|
|
of Branch:
|
|
stack[^1].step.node.bLink[preNibble] = thisKey
|
|
# Check whether all keys are proper, non-temporary keys
|
|
stack[^1].canLock = true
|
|
for n in 0 ..< 16:
|
|
if not stack[^1].step.node.bLink[n].isNodeKey:
|
|
stack[^1].canLock = false
|
|
break
|
|
of Extension:
|
|
stack[^1].step.node.eLink = thisKey
|
|
stack[^1].canLock = thisKey.isNodeKey
|
|
of Leaf:
|
|
return err(rTop-1)
|
|
|
|
# Must not overwrite a non-temprary key
|
|
if stack[^1].canLock:
|
|
stack[^1].step.key =
|
|
stack[^1].step.node.convertTo(Blob).digestTo(NodeKey).to(RepairKey)
|
|
|
|
ok()
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private walk along hexary trie records
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc hexaryFollow(
|
|
ps: AccountsDbSessionRef;
|
|
root: NodeKey;
|
|
path: NibblesSeq;
|
|
getFn: HexaryGetFn
|
|
): (int, bool, Blob)
|
|
{.gcsafe, raises: [Defect,RlpError]} =
|
|
## Returns the number of matching digits/nibbles from the argument `path`
|
|
## found in the proofs trie.
|
|
let
|
|
nNibbles = path.len
|
|
var
|
|
inPath = path
|
|
recKey = root.ByteArray32.toSeq
|
|
leafBlob: Blob
|
|
emptyRef = false
|
|
|
|
when BasicChainTrieDebugging:
|
|
trace "follow", rootKey=root.pp(ps), path
|
|
|
|
while true:
|
|
let value = getFn(recKey)
|
|
if value.len == 0:
|
|
break
|
|
|
|
var nodeRlp = rlpFromBytes value
|
|
case nodeRlp.listLen:
|
|
of 2:
|
|
let
|
|
(isLeaf, pathSegment) = hexPrefixDecode nodeRlp.listElem(0).toBytes
|
|
sharedNibbles = inPath.sharedPrefixLen(pathSegment)
|
|
fullPath = sharedNibbles == pathSegment.len
|
|
inPathLen = inPath.len
|
|
inPath = inPath.slice(sharedNibbles)
|
|
|
|
# Leaf node
|
|
if isLeaf:
|
|
let leafMode = sharedNibbles == inPathLen
|
|
if fullPath and leafMode:
|
|
leafBlob = nodeRlp.listElem(1).toBytes
|
|
when BasicChainTrieDebugging:
|
|
let nibblesLeft = inPathLen - sharedNibbles
|
|
trace "follow leaf",
|
|
fullPath, leafMode, sharedNibbles, nibblesLeft,
|
|
pathSegment, newPath=inPath
|
|
break
|
|
|
|
# Extension node
|
|
if fullPath:
|
|
let branch = nodeRlp.listElem(1)
|
|
if branch.isEmpty:
|
|
when BasicChainTrieDebugging:
|
|
trace "follow extension", newKey="n/a"
|
|
emptyRef = true
|
|
break
|
|
recKey = branch.toBytes
|
|
when BasicChainTrieDebugging:
|
|
trace "follow extension",
|
|
newKey=recKey.convertTo(NodeKey).pp(ps), newPath=inPath
|
|
else:
|
|
when BasicChainTrieDebugging:
|
|
trace "follow extension",
|
|
fullPath, sharedNibbles, pathSegment,
|
|
inPathLen, newPath=inPath
|
|
break
|
|
|
|
of 17:
|
|
# Branch node
|
|
if inPath.len == 0:
|
|
leafBlob = nodeRlp.listElem(1).toBytes
|
|
break
|
|
let
|
|
inx = inPath[0].int
|
|
branch = nodeRlp.listElem(inx)
|
|
if branch.isEmpty:
|
|
when BasicChainTrieDebugging:
|
|
trace "follow branch", newKey="n/a"
|
|
emptyRef = true
|
|
break
|
|
inPath = inPath.slice(1)
|
|
recKey = branch.toBytes
|
|
when BasicChainTrieDebugging:
|
|
trace "follow branch",
|
|
newKey=recKey.convertTo(NodeKey).pp(ps), inx, newPath=inPath
|
|
|
|
else:
|
|
when BasicChainTrieDebugging:
|
|
trace "follow oops",
|
|
nColumns = nodeRlp.listLen
|
|
break
|
|
|
|
# end while
|
|
|
|
let pathLen = nNibbles - inPath.len
|
|
|
|
when BasicChainTrieDebugging:
|
|
trace "follow done",
|
|
recKey, emptyRef, pathLen, leafSize=leafBlob.len
|
|
|
|
(pathLen, emptyRef, leafBlob)
|
|
|
|
|
|
proc hexaryFollow(
|
|
ps: AccountsDbSessionRef;
|
|
root: NodeKey;
|
|
path: NodeKey;
|
|
getFn: HexaryGetFn
|
|
): (int, bool, Blob)
|
|
{.gcsafe, raises: [Defect,RlpError]} =
|
|
## Variant of `hexaryFollow()`
|
|
ps.hexaryFollow(root, path.to(NibblesSeq), getFn)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public constructor
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc init*(
|
|
T: type AccountsDbRef;
|
|
db: TrieDatabaseRef
|
|
): T =
|
|
## Main object constructor
|
|
T(db: db)
|
|
|
|
proc init*(
|
|
T: type AccountsDbRef;
|
|
db: ChainDb
|
|
): T =
|
|
## Variant of `init()`
|
|
result = T(db: db.trieDB, rocky: db.rocksStoreRef)
|
|
if not result.rocky.isNil:
|
|
# A cache file might hang about from a previous crash
|
|
try:
|
|
discard result.rocky.clearCacheFile(RockyBulkCache)
|
|
except OSError as e:
|
|
result.rocky = nil
|
|
error "Cannot clear rocksdb cache, bulkload disabled",
|
|
exception=($e.name), msg=e.msg
|
|
|
|
proc init*(
|
|
T: type AccountsDbSessionRef;
|
|
pv: AccountsDbRef;
|
|
root: Hash256;
|
|
peer: Peer = nil
|
|
): T =
|
|
## Start a new session, do some actions an then discard the session
|
|
## descriptor (probably after commiting data.)
|
|
AccountsDbSessionRef(base: pv, peer: peer, rootKey: root.to(NodeKey))
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public functions, session related
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc merge*(
|
|
ps: AccountsDbSessionRef;
|
|
proof: SnapAccountProof
|
|
): Result[void,AccountsDbError]
|
|
{.gcsafe, raises: [Defect, RlpError].} =
|
|
## Import account proof records (as received with the snap message
|
|
## `AccountRange`) into the hexary trie of the repair database. These hexary
|
|
## trie records can be extended to a full trie at a later stage and used for
|
|
## validating account data.
|
|
for n,rlpRec in proof:
|
|
let rc = ps.hexaryImport(rlpRec)
|
|
if rc.isErr:
|
|
trace "merge(SnapAccountProof)", peer=ps.peer,
|
|
proofs=ps.rpDB.tab.len, accounts=ps.rpDB.acc.len, error=rc.error
|
|
return err(rc.error)
|
|
|
|
ok()
|
|
|
|
|
|
proc merge*(
|
|
ps: AccountsDbSessionRef;
|
|
base: NodeTag;
|
|
acc: seq[SnapAccount];
|
|
): Result[void,AccountsDbError]
|
|
{.gcsafe, raises: [Defect, RlpError].} =
|
|
## Import account records (as received with the snap message `AccountRange`)
|
|
## into the accounts list of the repair database. The accounts, together
|
|
## with some hexary trie records for proof can be used for validating
|
|
## the argument account data.
|
|
##
|
|
if acc.len != 0:
|
|
let
|
|
pathTag0 = acc[0].accHash.to(NodeTag)
|
|
pathTagTop = acc[^1].accHash.to(NodeTag)
|
|
saveLen = ps.rpDB.acc.len
|
|
|
|
# For error logging
|
|
(peer, proofs, accounts) = (ps.peer, ps.rpDB.tab.len, ps.rpDB.acc.len)
|
|
|
|
var
|
|
error = NothingSerious
|
|
saveQ: seq[RAccount]
|
|
prependOk = false
|
|
if 0 < ps.rpDB.acc.len:
|
|
if pathTagTop <= ps.rpDB.acc[0].pathTag:
|
|
# Prepend `acc` argument before `ps.rpDB.acc`
|
|
saveQ = ps.rpDB.acc
|
|
prependOk = true
|
|
|
|
# Append, verify that there is no overlap
|
|
elif pathTag0 <= ps.rpDB.acc[^1].pathTag:
|
|
return err(AccountRangesOverlap)
|
|
|
|
block collectAccounts:
|
|
# Verify lower bound
|
|
if pathTag0 < base:
|
|
error = AccountSmallerThanBase
|
|
trace "merge(seq[SnapAccount])", peer, proofs, base, accounts, error
|
|
break collectAccounts
|
|
|
|
# Add base for the records (no payload). Note that the assumption
|
|
# holds: `ps.rpDB.acc[^1].tag <= base`
|
|
if base < pathTag0:
|
|
ps.rpDB.acc.add RAccount(pathTag: base)
|
|
|
|
# Check for the case that accounts are appended
|
|
elif 0 < ps.rpDB.acc.len and pathTag0 <= ps.rpDB.acc[^1].pathTag:
|
|
error = AccountsNotSrictlyIncreasing
|
|
trace "merge(seq[SnapAccount])", peer, proofs, base, accounts, error
|
|
break collectAccounts
|
|
|
|
# Add first account
|
|
ps.rpDB.acc.add RAccount(
|
|
pathTag: pathTag0, payload: acc[0].accBody.encode)
|
|
|
|
# Veify & add other accounts
|
|
for n in 1 ..< acc.len:
|
|
let nodeTag = acc[n].accHash.to(NodeTag)
|
|
|
|
if nodeTag <= ps.rpDB.acc[^1].pathTag:
|
|
# Recover accounts list and return error
|
|
ps.rpDB.acc.setLen(saveLen)
|
|
|
|
error = AccountsNotSrictlyIncreasing
|
|
trace "merge(seq[SnapAccount])", peer, proofs, base, accounts, error
|
|
break collectAccounts
|
|
|
|
ps.rpDB.acc.add RAccount(
|
|
pathTag: nodeTag, payload: acc[n].accBody.encode)
|
|
|
|
# End block `collectAccounts`
|
|
|
|
if prependOk:
|
|
if error == NothingSerious:
|
|
ps.rpDB.acc = ps.rpDB.acc & saveQ
|
|
else:
|
|
ps.rpDB.acc = saveQ
|
|
|
|
if error != NothingSerious:
|
|
return err(error)
|
|
|
|
ok()
|
|
|
|
|
|
proc interpolate*(ps: AccountsDbSessionRef): Result[void,AccountsDbError] =
|
|
## Verifiy accounts by interpolating the collected accounts on the hexary
|
|
## trie of the repair database. If all accounts can be represented in the
|
|
## hexary trie, they are vonsidered validated.
|
|
##
|
|
## Note:
|
|
## This function temporary and proof-of-concept. for production purposes,
|
|
## it must be replaced by the new facility of the upcoming re-factored
|
|
## database layer.
|
|
##
|
|
# Walk top down and insert/complete missing account access nodes
|
|
for n in countDown(ps.rpDB.acc.len-1,0):
|
|
let acc = ps.rpDB.acc[n]
|
|
if acc.payload.len != 0:
|
|
let rPath = acc.pathTag.rTreeFollow(ps)
|
|
var repairKey = acc.nodeKey
|
|
if repairKey.isZero and 0 < rPath.path.len and rPath.tail.len == 0:
|
|
repairKey = rPath.path[^1].key
|
|
ps.rpDB.acc[n].nodeKey = repairKey
|
|
if repairKey.isZero:
|
|
let
|
|
update = rPath.rTreeInterpolate(ps, acc.payload)
|
|
final = acc.pathTag.rTreeFollow(ps)
|
|
if update != final:
|
|
return err(AccountRepairBlocked)
|
|
ps.rpDB.acc[n].nodeKey = rPath.path[^1].key
|
|
|
|
# Replace temporary repair keys by proper hash based node keys.
|
|
var reVisit: seq[NodeTag]
|
|
for n in countDown(ps.rpDB.acc.len-1,0):
|
|
let acc = ps.rpDB.acc[n]
|
|
if not acc.nodeKey.isZero:
|
|
let rPath = acc.pathTag.rTreeFollow(ps)
|
|
if rPath.path[^1].node.state == Mutable:
|
|
let rc = rPath.rTreeUpdateKeys(ps)
|
|
if rc.isErr:
|
|
reVisit.add acc.pathTag
|
|
|
|
while 0 < reVisit.len:
|
|
var again: seq[NodeTag]
|
|
for nodeTag in reVisit:
|
|
let rc = nodeTag.rTreeFollow(ps).rTreeUpdateKeys(ps)
|
|
if rc.isErr:
|
|
again.add nodeTag
|
|
if reVisit.len <= again.len:
|
|
return err(BoundaryProofFailed)
|
|
reVisit = again
|
|
|
|
ok()
|
|
|
|
|
|
proc dbImports*(ps: AccountsDbSessionRef): Result[void,AccountsDbError] =
|
|
## Experimental: try several db-import modes and record statistics
|
|
var als: AccountLoadStats
|
|
noPpError("dbImports"):
|
|
als.dura[0].elapsed:
|
|
let rc = ps.storeAccountPathsOnChainDb()
|
|
if rc.isErr: return rc
|
|
als.dura[1].elapsed:
|
|
let rc = ps.storeHexaryNodesOnChainDb()
|
|
if rc.isErr: return rc
|
|
als.dura[2].elapsed:
|
|
let rc = ps.storeAccountPathsOnRockyDb()
|
|
if rc.isErr: return rc
|
|
als.dura[3].elapsed:
|
|
let rc = ps.storeHexaryNodesOnRockyDb()
|
|
if rc.isErr: return rc
|
|
|
|
for n in 0 ..< 4:
|
|
ps.dStats.dura[n] += als.dura[n]
|
|
ps.base.aStats.dura[n] += als.dura[n]
|
|
|
|
ps.dStats.size[0] += ps.rpDB.acc.len.uint64
|
|
ps.base.aStats.size[0] += ps.rpDB.acc.len.uint64
|
|
|
|
ps.dStats.size[1] += ps.rpDB.tab.len.uint64
|
|
ps.base.aStats.size[1] += ps.rpDB.tab.len.uint64
|
|
|
|
ok()
|
|
|
|
|
|
proc sortMerge*(base: openArray[NodeTag]): NodeTag =
|
|
## Helper for merging several `(NodeTag,seq[SnapAccount])` data sets
|
|
## so that there are no overlap which would be rejected by `merge()`.
|
|
##
|
|
## This function selects a `NodeTag` from a list.
|
|
result = high(NodeTag)
|
|
for w in base:
|
|
if w < result:
|
|
result = w
|
|
|
|
proc sortMerge*(acc: openArray[seq[SnapAccount]]): seq[SnapAccount] =
|
|
## Helper for merging several `(NodeTag,seq[SnapAccount])` data sets
|
|
## so that there are no overlap which would be rejected by `merge()`.
|
|
##
|
|
## This function flattens and sorts the argument account lists.
|
|
noPpError("sortMergeAccounts"):
|
|
var accounts: Table[NodeTag,SnapAccount]
|
|
for accList in acc:
|
|
for item in accList:
|
|
accounts[item.accHash.to(NodeTag)] = item
|
|
result = toSeq(accounts.keys).sorted(cmp).mapIt(accounts[it])
|
|
|
|
proc nHexaryRecords*(ps: AccountsDbSessionRef): int =
|
|
## Number of hexary record entries in the session database.
|
|
ps.rpDB.tab.len
|
|
|
|
proc nAccountRecords*(ps: AccountsDbSessionRef): int =
|
|
## Number of account records in the session database. This number includes
|
|
## lower bound entries (which are not accoiunts, strictly speaking.)
|
|
ps.rpDB.acc.len
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public functions
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc dbBackendRocksDb*(pv: AccountsDbRef): bool =
|
|
## Returns `true` if rocksdb features are available
|
|
not pv.rocky.isNil
|
|
|
|
proc importAccounts*(
|
|
pv: AccountsDbRef;
|
|
peer: Peer, ## for log messages
|
|
root: Hash256; ## state root
|
|
base: NodeTag; ## before or at first account entry in `data`
|
|
data: SnapAccountRange; ## `snap/1 ` reply data
|
|
storeData = false
|
|
): Result[void,AccountsDbError] =
|
|
## Validate and accounts and proofs (as received with the snap message
|
|
## `AccountRange`). This function combines the functionality of the `merge()`
|
|
## and the `interpolate()` functions.
|
|
##
|
|
## At a later stage, that function also will bulk-import the accounts into
|
|
## the block chain database
|
|
##
|
|
## Note that the `peer` argument is for log messages, only.
|
|
let ps = AccountsDbSessionRef.init(pv, root, peer)
|
|
try:
|
|
block:
|
|
let rc = ps.merge(data.proof)
|
|
if rc.isErr:
|
|
return err(rc.error)
|
|
block:
|
|
let rc = ps.merge(base, data.accounts)
|
|
if rc.isErr:
|
|
return err(rc.error)
|
|
except RlpError:
|
|
return err(RlpEncoding)
|
|
|
|
block:
|
|
# Note:
|
|
# `interpolate()` is a temporary proof-of-concept function. For
|
|
# production purposes, it must be replaced by the new facility of
|
|
# the upcoming re-factored database layer.
|
|
let rc = ps.interpolate()
|
|
if rc.isErr:
|
|
return err(rc.error)
|
|
|
|
if storeData:
|
|
# Experimental
|
|
let rc = ps.dbImports()
|
|
if rc.isErr:
|
|
return err(rc.error)
|
|
|
|
trace "Accounts and proofs ok", peer, root=root.data.toHex,
|
|
proof=data.proof.len, base, accounts=data.accounts.len
|
|
ok()
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Debugging
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc getChainDbAccount*(
|
|
ps: AccountsDbSessionRef;
|
|
accHash: Hash256
|
|
): Result[Account,AccountsDbError] =
|
|
## Fetch account via `BaseChainDB`
|
|
try:
|
|
let
|
|
getFn: HexaryGetFn = proc(key: Blob): Blob = ps.base.db.get(key)
|
|
(_, _, leafBlob) = ps.hexaryFollow(ps.rootKey, accHash.to(NodeKey), getFn)
|
|
if 0 < leafBlob.len:
|
|
let acc = rlp.decode(leafBlob,Account)
|
|
return ok(acc)
|
|
except RlpError:
|
|
return err(RlpEncoding)
|
|
except Defect as e:
|
|
raise e
|
|
except Exception as e:
|
|
raiseAssert "Ooops getChainDbAccount(): name=" & $e.name & " msg=" & e.msg
|
|
|
|
err(AccountNotFound)
|
|
|
|
proc getRockyAccount*(
|
|
ps: AccountsDbSessionRef;
|
|
accHash: Hash256
|
|
): Result[Account,AccountsDbError] =
|
|
## Fetch account via rocksdb table (paraellel to `BaseChainDB`)
|
|
try:
|
|
let
|
|
getFn: HexaryGetFn = proc(key: Blob): Blob =
|
|
var tag: NodeTag
|
|
discard tag.init(key)
|
|
ps.base.db.get(tag.rockyBulkHexaryKey().toOpenArray)
|
|
(_, _, leafBlob) = ps.hexaryFollow(ps.rootKey, accHash.to(NodeKey), getFn)
|
|
if 0 < leafBlob.len:
|
|
let acc = rlp.decode(leafBlob,Account)
|
|
return ok(acc)
|
|
except RlpError:
|
|
return err(RlpEncoding)
|
|
except Defect as e:
|
|
raise e
|
|
except Exception as e:
|
|
raiseAssert "Ooops getChainDbAccount(): name=" & $e.name & " msg=" & e.msg
|
|
|
|
err(AccountNotFound)
|
|
|
|
|
|
proc dbImportStats*(ps: AccountsDbSessionRef): AccountLoadStats =
|
|
## Session data load statistics
|
|
ps.dStats
|
|
|
|
proc dbImportStats*(pv: AccountsDbRef): AccountLoadStats =
|
|
## Accumulated data load statistics
|
|
pv.aStats
|
|
|
|
proc assignPrettyKeys*(ps: AccountsDbSessionRef) =
|
|
## Prepare foe pretty pringing/debugging. Run early enough this function
|
|
## sets the root key to `"$"`, for instance.
|
|
noPpError("validate(1)"):
|
|
# Make keys assigned in pretty order for printing
|
|
var keysList = toSeq(ps.rpDB.tab.keys)
|
|
let rootKey = ps.rootKey.to(RepairKey)
|
|
discard rootKey.toKey(ps)
|
|
if ps.rpDB.tab.hasKey(rootKey):
|
|
keysList = @[rootKey] & keysList
|
|
for key in keysList:
|
|
let node = ps.rpDB.tab[key]
|
|
discard key.toKey(ps)
|
|
case node.kind:
|
|
of Branch: (for w in node.bLink: discard w.toKey(ps))
|
|
of Extension: discard node.eLink.toKey(ps)
|
|
of Leaf: discard
|
|
|
|
proc dumpPath*(ps: AccountsDbSessionRef; key: NodeTag): seq[string] =
|
|
## Pretty print helper compiling the path into the repair tree for the
|
|
## argument `key`.
|
|
let rPath = key.rTreeFollow(ps)
|
|
rPath.path.mapIt(it.pp(ps)) & @["(" & rPath.tail.pp & ")"]
|
|
|
|
proc dumpProofsDB*(ps: AccountsDbSessionRef): seq[string] =
|
|
## Dump the entries from the repair tree.
|
|
var accu = @[(0u, "($0" & "," & ps.rootKey.pp(ps) & ")")]
|
|
for key,node in ps.rpDB.tab.pairs:
|
|
accu.add (key.toKey(ps), "(" & key.pp(ps) & "," & node.pp(ps) & ")")
|
|
proc cmpIt(x, y: (uint,string)): int =
|
|
cmp(x[0],y[0])
|
|
result = accu.sorted(cmpIt).mapIt(it[1])
|
|
|
|
# ---------
|
|
|
|
proc dumpRoot*(root: Hash256; name = "snapRoot*"): string =
|
|
noPpError("dumpRoot"):
|
|
result = "import\n"
|
|
result &= " eth/common/eth_types,\n"
|
|
result &= " nimcrypto/hash,\n"
|
|
result &= " stew/byteutils\n\n"
|
|
result &= "const\n"
|
|
result &= &" {name} =\n"
|
|
result &= &" \"{root.pp(false)}\".toDigest\n"
|
|
|
|
proc dumpSnapAccountRange*(
|
|
base: NodeTag;
|
|
data: SnapAccountRange;
|
|
name = "snapData*"
|
|
): string =
|
|
noPpError("dumpSnapAccountRange"):
|
|
result = &" {name} = ("
|
|
result &= &"\n \"{base.to(Hash256).pp(false)}\".toDigest,"
|
|
result &= "\n @["
|
|
let accPfx = "\n "
|
|
for n in 0 ..< data.accounts.len:
|
|
let
|
|
hash = data.accounts[n].accHash
|
|
body = data.accounts[n].accBody
|
|
if 0 < n:
|
|
result &= accPfx
|
|
result &= &"# <{n}>"
|
|
result &= &"{accPfx}(\"{hash.pp(false)}\".toDigest,"
|
|
result &= &"{accPfx} {body.nonce}u64,"
|
|
result &= &"{accPfx} \"{body.balance}\".parse(Uint256),"
|
|
result &= &"{accPfx} \"{body.storageRoot.pp(false)}\".toDigest,"
|
|
result &= &"{accPfx} \"{body.codehash.pp(false)}\".toDigest),"
|
|
if result[^1] == ',':
|
|
result[^1] = ']'
|
|
else:
|
|
result &= "]"
|
|
result &= ",\n @["
|
|
let blobPfx = "\n "
|
|
for n in 0 ..< data.proof.len:
|
|
let blob = data.proof[n]
|
|
if 0 < n:
|
|
result &= blobPfx
|
|
result &= &"# <{n}>"
|
|
result &= &"{blobPfx}\"{blob.pp}\".hexToSeqByte,"
|
|
if result[^1] == ',':
|
|
result[^1] = ']'
|
|
else:
|
|
result &= "]"
|
|
result &= ")\n"
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|