status-im
/
nimbus-eth1
mirror of https://github.com/status-im/nimbus-eth1.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

implement account storage encoding and decoding

This commit is contained in:
andri lim 2020-04-28 21:06:37 +07:00
parent 4de8eaa81d
commit d95ded217b
No known key found for this signature in database
GPG Key ID: 31702AE10541E6B9
4 changed files with 84 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
stateless/test_witness_keys.nim
View File

@ -24,14 +24,24 @@ proc randCode(db: DB): Hash256 =
result = hexary.keccak(code)
db.put(result.data, code)
proc randHash(): Hash256 =
discard randomBytes(result.data[0].addr, sizeof(result))
proc randStorage(db: DB): Hash256 =
if rand(0..1) == 0:
result = emptyRlpHash
else:
var trie = initSecureHexaryTrie(db)
let numPairs = rand(1..5)
for i in 0..<numPairs:
# we bypass u256 key to slot conversion
# discard randomBytes(key.addr, sizeof(key))
trie.put(i.u256.toByteArrayBE, rlp.encode(randU256()))
result = trie.rootHash
proc randAccount(db: DB): Account =
result.nonce = randNonce()
result.balance = randU256()
result.codeHash = randCode(db)
result.storageRoot = randHash()
result.storageRoot = randStorage(db)
proc runTest(numPairs: int) =
var memDB = newMemoryDB()
@ -46,7 +56,7 @@ proc runTest(numPairs: int) =
let rootHash = trie.rootHash
var wb = initWitnessBuilder(memDB, rootHash)
var witness = wb.getBranchRecurse(addrs[0])
var witness = wb.getBranchRecurse(hexary.keccak(addrs[0]).data)
var db = newMemoryDB()
when defined(useInputStream):
var input = memoryInput(witness)

37
stateless/tree_from_witness.nim
View File

@ -106,21 +106,21 @@ proc writeCode(t: var TreeBuilder, code: openArray[byte]): Hash256 =
result = keccak(code)
put(t.db, result.data, code)
proc branchNode(t: var TreeBuilder, depth: int): NodeKey
proc extensionNode(t: var TreeBuilder, depth: int): NodeKey
proc branchNode(t: var TreeBuilder, depth: int, storageMode: bool): NodeKey
proc extensionNode(t: var TreeBuilder, depth: int, storageMode: bool): NodeKey
proc accountNode(t: var TreeBuilder, depth: int): NodeKey
proc accountStorageLeafNode(t: var TreeBuilder, depth: int): NodeKey
proc hashNode(t: var TreeBuilder): NodeKey
proc treeNode*(t: var TreeBuilder, depth: int = 0, accountMode = false): NodeKey =
proc treeNode*(t: var TreeBuilder, depth: int = 0, storageMode = false): NodeKey =
assert(depth < 64)
let nodeType = TrieNodeType(t.readByte)
case nodeType
of BranchNodeType: result = t.branchNode(depth)
of ExtensionNodeType: result = t.extensionNode(depth)
of BranchNodeType: result = t.branchNode(depth, storageMode)
of ExtensionNodeType: result = t.extensionNode(depth, storageMode)
of AccountNodeType:
if accountMode:
if storageMode:
# parse account storage leaf node
result = t.accountStorageLeafNode(depth)
else:
@ -131,7 +131,7 @@ proc treeNode*(t: var TreeBuilder, depth: int = 0, accountMode = false): NodeKey
result.data = keccak(result.data.toOpenArray(0, result.usedBytes-1)).data
result.usedBytes = 32
proc branchNode(t: var TreeBuilder, depth: int): NodeKey =
proc branchNode(t: var TreeBuilder, depth: int, storageMode: bool): NodeKey =
assert(depth < 64)
let mask = constructBranchMask(t.readByte, t.readByte)
@ -146,7 +146,7 @@ proc branchNode(t: var TreeBuilder, depth: int): NodeKey =
for i in 0 ..< 16:
if mask.branchMaskBitIsSet(i):
r.append t.treeNode(depth+1)
r.append t.treeNode(depth+1, storageMode)
else:
r.append ""
@ -182,7 +182,7 @@ func hexPrefix(r: var RlpWriter, x: openArray[byte], nibblesLen: int, isLeaf: st
r.append toOpenArray(bytes, 0, nibblesLen div 2)
proc extensionNode(t: var TreeBuilder, depth: int): NodeKey =
proc extensionNode(t: var TreeBuilder, depth: int, storageMode: bool): NodeKey =
assert(depth < 63)
let nibblesLen = int(t.readByte)
assert(nibblesLen < 65)
@ -200,7 +200,7 @@ proc extensionNode(t: var TreeBuilder, depth: int): NodeKey =
let nodeType = TrieNodeType(t.readByte)
case nodeType
of BranchNodeType: r.append t.branchNode(depth + nibblesLen)
of BranchNodeType: r.append t.branchNode(depth + nibblesLen, storageMode)
of HashNodeType: r.append t.hashNode()
else: raise newException(ValueError, "wrong type during parsing child of extension node")
@ -227,7 +227,9 @@ proc accountNode(t: var TreeBuilder, depth: int): NodeKey =
var r = initRlpList(2)
r.hexPrefix(t.read(nibblesLen div 2 + nibblesLen mod 2), nibblesLen, true)
#let address = toAddress(t.read(20))
# TODO: parse address
# let address = toAddress(t.read(20))
var acc = Account(
balance: UInt256.fromBytesBE(t.read(32), false),
# TODO: why nonce must be 32 bytes, isn't 64 bit uint enough?
@ -245,7 +247,7 @@ proc accountNode(t: var TreeBuilder, depth: int): NodeKey =
# switch to account storage parsing mode
# and reset the depth
let storageRoot = t.treeNode(0, accountMode = true)
let storageRoot = t.treeNode(0, storageMode = true)
doAssert(storageRoot.usedBytes == 32)
acc.storageRoot.data = storageRoot.data
@ -258,9 +260,14 @@ proc accountNode(t: var TreeBuilder, depth: int): NodeKey =
proc accountStorageLeafNode(t: var TreeBuilder, depth: int): NodeKey =
assert(depth < 65)
let nibblesLen = 64 - depth
let pathNibbles = @(t.read(nibblesLen div 2 + nibblesLen mod 2))
let key = @(t.read(32))
let val = @(t.read(32))
var r = initRlpList(2)
r.hexPrefix(t.read(nibblesLen div 2 + nibblesLen mod 2), nibblesLen, true)
# TODO: parse key
# let key = @(t.read(32))
# UInt256 -> BytesBE -> keccak
let val = UInt256.fromBytesBE(t.read(32))
r.append rlp.encode(val)
result = toNodeKey(r.finish)
proc hashNode(t: var TreeBuilder): NodeKey =
result.toKeccak(t.read(32))

47
stateless/witness_from_tree.nim
View File

@ -43,6 +43,8 @@ proc writeU32(wb: var WitnessBuilder, x: uint32) =
wb.output.append(toBytesBE(x))
proc writeNibbles(wb: var WitnessBuilder; n: NibblesSeq, withLen: bool = true) =
# convert the NibblesSeq into left aligned byte seq
# perhaps we can optimize it if the NibblesSeq already left aligned
let nibblesLen = n.len
let numBytes = nibblesLen div 2 + nibblesLen mod 2
var bytes: array[32, byte]
@ -78,6 +80,7 @@ proc writeBranchNode(wb: var WitnessBuilder, mask: uint, depth: int, node: openA
doAssert mask.branchMaskBitIsSet(16) == false
wb.output.append(BranchNodeType.byte)
# write branch mask
# countOnes(branch mask) >= 2 and <= 16
wb.output.append(((mask shr 8) and 0xFF).byte)
wb.output.append((mask and 0xFF).byte)
@ -92,6 +95,8 @@ proc writeHashNode(wb: var WitnessBuilder, node: openArray[byte]) =
wb.output.append(HashNodeType.byte)
wb.output.append(node)
proc getBranchRecurseAux(wb: var WitnessBuilder, node: openArray[byte], path: NibblesSeq, depth: int, storageMode: bool)
proc writeAccountNode(wb: var WitnessBuilder, acc: Account, nibbles: NibblesSeq, node: openArray[byte], depth: int) =
# write type
wb.output.append(AccountNodeType.byte)
@ -109,7 +114,10 @@ proc writeAccountNode(wb: var WitnessBuilder, acc: Account, nibbles: NibblesSeq,
wb.output.append(accountType.byte)
wb.writeNibbles(nibbles, false)
#wb.output.append(acc.address)
# TODO: where the address come from?
# single proof is easy, but multiproof will be harder
# concat the path and then look into LUT?
# wb.output.append(acc.address)
wb.output.append(acc.balance.toBytesBE)
wb.output.append(acc.nonce.u256.toBytesBE)
@ -124,20 +132,38 @@ proc writeAccountNode(wb: var WitnessBuilder, acc: Account, nibbles: NibblesSeq,
wb.writeU32(0'u32)
if acc.storageRoot != emptyRlpHash:
wb.writeHashNode(acc.storageRoot.data)
# switch to account mode
var node = wb.db.get(acc.storageRoot.data)
var key = keccak(0.u256.toByteArrayBE)
getBranchRecurseAux(wb, node, initNibbleRange(key.data), 0, true)
else:
wb.writeHashNode(emptyRlpHash.data)
#0x00 pathnibbles:<Nibbles(64-d)> address:<Address> balance:<Bytes32> nonce:<Bytes32>
#0x01 pathnibbles:<Nibbles(64-d)> address:<Address> balance:<Bytes32> nonce:<Bytes32> bytecode:<Bytecode> storage:<Tree_Node(0,1)>
proc writeShortNode(wb: var WitnessBuilder, node: openArray[byte], depth: int) =
proc writeAccountStorageLeafNode(wb: var WitnessBuilder, val: UInt256, nibbles: NibblesSeq, node: openArray[byte], depth: int) =
wb.output.append(StorageLeafNodeType.byte)
doAssert(nibbles.len == 64 - depth)
wb.writeNibbles(nibbles, false)
# TODO: write key
# wb.output.append(key.toByteArrayBE)
wb.output.append(val.toByteArrayBE)
#<Storage_Leaf_Node(d<65)> := pathnibbles:<Nibbles(64-d))> key:<Bytes32> val:<Bytes32>
proc writeShortNode(wb: var WitnessBuilder, node: openArray[byte], depth: int, storageMode: bool) =
var nodeRlp = rlpFromBytes node
if not nodeRlp.hasData or nodeRlp.isEmpty: return
case nodeRlp.listLen
of 2:
let (isLeaf, k) = nodeRlp.extensionNodeKey
if isLeaf:
if storageMode:
let val = nodeRlp.listElem(1).toBytes.decode(UInt256)
writeAccountStorageLeafNode(wb, val, k, node, depth)
else:
let acc = nodeRlp.listElem(1).toBytes.decode(Account)
writeAccountNode(wb, acc, k, node, depth)
else:
@ -155,7 +181,7 @@ proc writeShortNode(wb: var WitnessBuilder, node: openArray[byte], depth: int) =
if branchMask.branchMaskBitIsSet(i):
var branch = nodeRlp.listElem(i)
let nextLookup = branch.getNode
writeShortNode(wb, nextLookup, depth + 1)
writeShortNode(wb, nextLookup, depth + 1, storageMode)
# contrary to yellow paper spec,
# the 17th elem never exist in reality.
@ -166,7 +192,7 @@ proc writeShortNode(wb: var WitnessBuilder, node: openArray[byte], depth: int) =
else:
raise newException(CorruptedTrieDatabase, "Bad Short Node")
proc getBranchRecurseAux(wb: var WitnessBuilder, node: openArray[byte], path: NibblesSeq, depth: int) =
proc getBranchRecurseAux(wb: var WitnessBuilder, node: openArray[byte], path: NibblesSeq, depth: int, storageMode: bool) =
var nodeRlp = rlpFromBytes node
if not nodeRlp.hasData or nodeRlp.isEmpty: return
@ -180,9 +206,12 @@ proc getBranchRecurseAux(wb: var WitnessBuilder, node: openArray[byte], path: Ni
# ExtensionNodeType
writeExtensionNode(wb, k, depth, node)
let nextLookup = value.getNode
getBranchRecurseAux(wb, nextLookup, path.slice(sharedNibbles), depth + sharedNibbles)
getBranchRecurseAux(wb, nextLookup, path.slice(sharedNibbles), depth + sharedNibbles, storageMode)
else:
# AccountNodeType
if storageMode:
writeAccountStorageLeafNode(wb, value.toBytes.decode(UInt256), k, node, depth)
else:
writeAccountNode(wb, value.toBytes.decode(Account), k, node, depth)
else:
# this is a potential branch for multiproof
@ -197,11 +226,11 @@ proc getBranchRecurseAux(wb: var WitnessBuilder, node: openArray[byte], path: Ni
var branch = nodeRlp.listElem(i)
if notLeaf and i == path[0].int:
let nextLookup = branch.getNode
getBranchRecurseAux(wb, nextLookup, path.slice(1), depth + 1)
getBranchRecurseAux(wb, nextLookup, path.slice(1), depth + 1, storageMode)
else:
if branch.isList:
let nextLookup = branch.getNode
writeShortNode(wb, nextLookup, depth + 1)
writeShortNode(wb, nextLookup, depth + 1, storageMode)
else:
# this is a potential branch for multiproof
writeHashNode(wb, branch.expectHash)
@ -214,7 +243,7 @@ proc getBranchRecurseAux(wb: var WitnessBuilder, node: openArray[byte], path: Ni
proc getBranchRecurse*(wb: var WitnessBuilder; key: openArray[byte]): seq[byte] =
var node = wb.db.get(wb.root.data)
getBranchRecurseAux(wb, node, initNibbleRange(key), 0)
getBranchRecurseAux(wb, node, initNibbleRange(key), 0, false)
result = wb.output.getOutput(seq[byte])
proc getBranchStack*(wb: WitnessBuilder; key: openArray[byte]): string =

3
stateless/witness_types.nim
View File

@ -11,6 +11,9 @@ type
SimpleAccountType
ExtendedAccountType
const
StorageLeafNodeType* = AccountNodeType
proc setBranchMaskBit*(x: var uint, i: int) {.inline.} =
assert(i >= 0 and i < 17)
x = x or (1 shl i).uint