status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix multikeys algorithm with multiple nibbles common prefix

This commit is contained in:
andri lim 2020-05-07 21:36:51 +07:00
parent 57355c98fd
commit ba4360ba34
No known key found for this signature in database
GPG Key ID: 31702AE10541E6B9
3 changed files with 168 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
stateless/multi_keys.nim
View File

@ -101,41 +101,52 @@ func groups*(m: MultikeysRef, parentGroup: Group, depth: int): BranchGroup =
setBranchMaskBit(result.mask, nibble.int)
result.groups[nibble.int] = g
iterator groups*(m: MultikeysRef, depth: int, n: NibblesSeq, parentGroup: Group): MatchGroup =
func groups*(m: MultikeysRef, depth: int, n: NibblesSeq, parentGroup: Group): MatchGroup =
# using common-prefix comparison, this iterator
# will produce groups, usually only one match group
# the rest will be not match
# in case of wrong path, there will be no match at all
var g = Group(first: parentGroup.first, last: parentGroup.first)
var match = compareNibbles(m.keys[g.first].hash, depth, n)
let last = parentGroup.last
var haveGroup = false
var groupResult: Group
var matchResult: bool
for i in parentGroup.first..parentGroup.last:
if compareNibbles(m.keys[i].hash, depth, n) != match:
g.last = i - 1
haveGroup = true
matchResult = match
groupResult = g
match = not match
g = Group(first: g.last, last: g.last)
if i == last:
haveGroup = true
g.last = last
groupResult = g
matchResult = match
if haveGroup:
haveGroup = false
yield (matchResult, groupResult)
# will produce one match group or no match at all
var g = Group(first: parentGroup.first)
iterator keyDatas*(m: MultikeysRef, g: Group): var KeyData =
for i in g.first..g.last:
yield m.keys[i]
if compareNibbles(m.keys[g.first].hash, depth, n):
var i = g.first + 1
while i <= parentGroup.last:
if not compareNibbles(m.keys[i].hash, depth, n):
g.last = i - 1
# condition 1: match and no match
return (true, g)
inc i
iterator storageKeys*(m :MultikeysRef): MultikeysRef =
for x in m.keys:
yield x.storageKeys
# condition 2: all is a match group
g.last = parentGroup.last
return (true, g)
func match*(kd: KeyData, n: NibblesSeq, depth: int): bool {.inline.} =
compareNibbles(kd.hash, depth, n)
# no match came first, skip no match
# we only interested in a match group
var i = g.first + 1
while i <= parentGroup.last:
if compareNibbles(m.keys[i].hash, depth, n):
g.first = i
break
inc i
if i <= parentGroup.last:
while i <= parentGroup.last:
if not compareNibbles(m.keys[i].hash, depth, n):
# condition 3: no match, match, and no match
g.last = i - 1
return (true, g)
inc i
# condition 4: no match and match
g.last = parentGroup.last
return (true, g)
# condition 5: no match at all
result = (false, g)
func isValidMatch(mg: MatchGroup): bool =
result = mg.match and mg.group.first == mg.group.last
proc visitMatch*(m: var MultikeysRef, mg: MatchGroup, depth: int, k: NibblesSeq): KeyData =
doAssert(mg.isValidMatch)
m.keys[mg.group.first].visited = true
result = m.keys[mg.group.first]

97
stateless/test_witness_keys.nim
View File

@ -1,6 +1,6 @@
import
randutils, random, unittest,
eth/[common, rlp], eth/trie/[hexary, db, trie_defs],
randutils, random, unittest, stew/byteutils,
eth/[common, rlp], eth/trie/[hexary, db, trie_defs, nibbles],
faststreams/input_stream, nimcrypto/sysrand,
../stateless/[witness_from_tree, tree_from_witness],
../nimbus/db/storage_types, ./witness_types, ./multi_keys
@ -109,13 +109,21 @@ proc runTest(numPairs: int, testStatusIMPL: var TestStatus, addInvalidKeys: stat
when addInvalidKeys:
for kd in mkeys.keys:
if kd.address == invalidAddress:
check kd.visited == false
check kd.visited == false
else:
check kd.visited == true
else:
for kd in mkeys.keys:
check kd.visited == true
proc initMultiKeys(keys: openArray[string]): MultikeysRef =
result.new
for x in keys:
result.keys.add KeyData(
storageMode: false,
hash: hexToByteArray[32](x)
)
proc witnessKeysMain*() =
suite "random keys block witness roundtrip test":
randomize()
@ -132,5 +140,88 @@ proc witnessKeysMain*() =
test "invalid address ignored":
runTest(rand(1..30), testStatusIMPL, addInvalidKeys = true)
test "case 1: all keys is a match":
let keys = [
"0abc7124bce7762869be690036144c12c256bdb06ee9073ad5ecca18a47c3254",
"0abccc5b491732f964182ce4bde5e2468318692ed446e008f621b26f8ff56606",
"0abca163140158288775c8912aed274fb9d6a3a260e9e95e03e70ba8df30f6bb"
]
let m = initMultiKeys(keys)
let pg = m.initGroup()
let n = initNibbleRange(hexToByteArray[2]("0abc"))
let mg = m.groups(0, n, pg)
check:
mg.match == true
mg.group.first == 0
mg.group.last == 2
test "case 2: all keys is not a match":
let keys = [
"01237124bce7762869be690036144c12c256bdb06ee9073ad5ecca18a47c3254",
"0890cc5b491732f964182ce4bde5e2468318692ed446e008f621b26f8ff56606",
"0456a163140158288775c8912aed274fb9d6a3a260e9e95e03e70ba8df30f6bb"
]
let m = initMultiKeys(keys)
let pg = m.initGroup()
let n = initNibbleRange(hexToByteArray[2]("0abc"))
let mg = m.groups(0, n, pg)
check:
mg.match == false
test "case 3: not match and match":
let keys = [
"01237124bce7762869be690036144c12c256bdb06ee9073ad5ecca18a47c3254",
"0890cc5b491732f964182ce4bde5e2468318692ed446e008f621b26f8ff56606",
"0abc6a163140158288775c8912aed274fb9d6a3a260e9e95e03e70ba8df30f6b",
"0abc7a163140158288775c8912aed274fb9d6a3a260e9e95e03e70ba8df30f6b"
]
let m = initMultiKeys(keys)
let pg = m.initGroup()
let n = initNibbleRange(hexToByteArray[2]("0abc"))
let mg = m.groups(0, n, pg)
check:
mg.match == true
mg.group.first == 2
mg.group.last == 3
test "case 4: match and not match":
let keys = [
"0abc6a163140158288775c8912aed274fb9d6a3a260e9e95e03e70ba8df30f6b",
"0abc7a163140158288775c8912aed274fb9d6a3a260e9e95e03e70ba8df30f6b",
"01237124bce7762869be690036144c12c256bdb06ee9073ad5ecca18a47c3254",
"0890cc5b491732f964182ce4bde5e2468318692ed446e008f621b26f8ff56606"
]
let m = initMultiKeys(keys)
let pg = m.initGroup()
let n = initNibbleRange(hexToByteArray[2]("0abc"))
let mg = m.groups(0, n, pg)
check:
mg.match == true
mg.group.first == 0
mg.group.last == 1
test "case 5: not match, match and not match":
let keys = [
"01237124bce7762869be690036144c12c256bdb06ee9073ad5ecca18a47c3254",
"0890cc5b491732f964182ce4bde5e2468318692ed446e008f621b26f8ff56606",
"0abc6a163140158288775c8912aed274fb9d6a3a260e9e95e03e70ba8df30f6b",
"0abc7a163140158288775c8912aed274fb9d6a3a260e9e95e03e70ba8df30f6b",
"01237124bce7762869be690036144c12c256bdb06ee9073ad5ecca18a47c3254",
"0890cc5b491732f964182ce4bde5e2468318692ed446e008f621b26f8ff56606"
]
let m = initMultiKeys(keys)
let pg = m.initGroup()
let n = initNibbleRange(hexToByteArray[2]("0abc"))
let mg = m.groups(0, n, pg)
check:
mg.match == true
mg.group.first == 2
mg.group.last == 3
when isMainModule:
witnessKeysMain()

64
stateless/witness_from_tree.nim
View File

@ -219,42 +219,36 @@ proc getBranchRecurse(wb: var WitnessBuilder, z: var StackElem) =
case nodeRlp.listLen
of 2:
let (isLeaf, k) = nodeRlp.extensionNodeKey
var match = false
# only zero or one group can match the path
# but if there is a match, it can be in any position
# 1st, 2nd, or max 3rd position
# recursion will go deeper depend on the common-prefix length nibbles
for mg in groups(z.keys, z.depth, k, z.parentGroup):
if not mg.match: continue
doAssert(match == false) # should be only one match
match = true
let value = nodeRlp.listElem(1)
if not isLeaf:
# ExtensionNodeType
writeExtensionNode(wb, k, z.depth, z.node)
var zz = StackElem(
node: value.getNode,
parentGroup: mg.group,
keys: z.keys,
depth: z.depth + k.len,
storageMode: z.storageMode
)
getBranchRecurse(wb, zz)
else:
# this should be only one match
# if there is more than one match
# it means we encounter an invalid address
for kd in keyDatas(z.keys, mg.group):
if not match(kd, k, z.depth): continue # skip the invalid address
kd.visited = true
if z.storageMode:
doAssert(kd.storageMode)
writeAccountStorageLeafNode(wb, kd.storageSlot, value.toBytes.decode(UInt256), k, z.node, z.depth)
else:
doAssert(not kd.storageMode)
writeAccountNode(wb, kd, value.toBytes.decode(Account), k, z.node, z.depth)
if not match:
let mg = groups(z.keys, z.depth, k, z.parentGroup)
if not mg.match:
# return immediately if there is no match
writeHashNode(wb, keccak(z.node).data)
return
let value = nodeRlp.listElem(1)
if not isLeaf:
# recursion will go deeper depend on the common-prefix length nibbles
writeExtensionNode(wb, k, z.depth, z.node)
var zz = StackElem(
node: value.getNode,
parentGroup: mg.group,
keys: z.keys,
depth: z.depth + k.len,
storageMode: z.storageMode
)
getBranchRecurse(wb, zz)
return
# there should be only one match
let kd = z.keys.visitMatch(mg, z.depth, k)
if z.storageMode:
doAssert(kd.storageMode)
writeAccountStorageLeafNode(wb, kd.storageSlot, value.toBytes.decode(UInt256), k, z.node, z.depth)
else:
doAssert(not kd.storageMode)
writeAccountNode(wb, kd, value.toBytes.decode(Account), k, z.node, z.depth)
of 17:
let branchMask = rlpListToBitmask(nodeRlp)
writeBranchNode(wb, branchMask, z.depth, z.node)