nimbus-eth1/stateless/tree_from_witness.nim

import
  typetraits,
  faststreams/inputs, eth/[common, rlp], stint,
  eth/trie/[db, trie_defs],
  ./witness_types, stew/byteutils, ../nimbus/constants

type
  DB = TrieDatabaseRef

  NodeKey = object
    usedBytes: int
    data: array[32, byte]

  AccountAndSlots* = object
    address*: EthAddress
    codeLen*: int
    slots*: seq[StorageSlot]

  TreeBuilder = object
    when defined(useInputStream):
      input: InputStream
    else:
      input: seq[byte]
      pos: int
    db: DB
    root: KeccakHash
    flags: WitnessFlags
    keys: seq[AccountAndSlots]

# this TreeBuilder support short node parsing
# but a block witness should not contains short node
# for account trie. Short rlp node only appears in
# storage trie with depth >= 9

# the InputStream still unstable
# when using large dataset for testing
# or run longer

when defined(useInputStream):
  proc initTreeBuilder*(input: InputStream, db: DB, flags: WitnessFlags): TreeBuilder =
    result.input = input
    result.db = db
    result.root = emptyRlpHash
    result.flags = flags

  proc initTreeBuilder*(input: openArray[byte], db: DB, flags: WitnessFlags): TreeBuilder =
    result.input = memoryInput(input)
    result.db = db
    result.root = emptyRlpHash
    result.flags = flags
else:
  proc initTreeBuilder*(input: openArray[byte], db: DB, flags: WitnessFlags): TreeBuilder =
    result.input = @input
    result.db = db
    result.root = emptyRlpHash
    result.flags = flags

func rootHash*(t: TreeBuilder): KeccakHash {.inline.} =
  t.root

func getDB*(t: TreeBuilder): DB {.inline.} =
  t.db

when defined(useInputStream):
  template readByte(t: var TreeBuilder): byte =
    t.input.read

  template len(t: TreeBuilder): int =
    t.input.len

  template read(t: var TreeBuilder, len: int): auto =
    t.input.read(len)

  template readable(t: var TreeBuilder): bool =
    t.input.readable

  template readable(t: var TreeBuilder, len: int): bool =
    t.input.readable(len)

else:
  template readByte(t: var TreeBuilder): byte =
    let pos = t.pos
    inc t.pos
    t.input[pos]

  template len(t: TreeBuilder): int =
    t.input.len

  template readable(t: var TreeBuilder): bool =
    t.pos < t.input.len

  template readable(t: var TreeBuilder, length: int): bool =
    t.pos + length <= t.input.len

  template read(t: var TreeBuilder, len: int): auto =
    let pos = t.pos
    inc(t.pos, len)
    toOpenArray(t.input, pos, pos+len-1)

proc safeReadByte(t: var TreeBuilder): byte =
  if t.readable:
    result = t.readByte()
  else:
    raise newException(ParsingError, "Cannot read byte from input stream")

when defined(debugHash):
  proc safeReadU32(t: var TreeBuilder): uint32 =
    if t.readable(4):
      result = fromBytesBE(uint32, t.read(4))
    else:
      raise newException(ParsingError, "Cannot read U32 from input stream")

template safeReadEnum(t: var TreeBuilder, T: type): untyped =
  let typ = t.safeReadByte.int
  if typ < low(T).int or typ > high(T).int:
    raise newException(ParsingError, "Wrong " & T.name & " value " & $typ)
  T(typ)

template safeReadBytes(t: var TreeBuilder, length: int, body: untyped) =
  if t.readable(length):
    body
  else:
    raise newException(ParsingError, "Failed when try to read " & $length & " bytes")

proc readUVarint32(t: var TreeBuilder): uint32 =
  # LEB128 varint encoding
  var shift = 0
  while true:
    let b = t.safeReadByte()
    result = result or ((b and 0x7F).uint32 shl shift)
    if (0x80 and b) == 0:
      break
    inc(shift, 7)
    if shift > 28:
      raise newException(ParsingError, "Failed when try to parse uvarint32")

proc readUVarint256(t: var TreeBuilder): UInt256 =
  # LEB128 varint encoding
  var shift = 0
  while true:
    let b = t.safeReadByte()
    result = result or ((b and 0x7F).u256 shl shift)
    if (0x80 and b) == 0:
      break
    inc(shift, 7)
    if shift > 252:
      raise newException(ParsingError, "Failed when try to parse uvarint256")

proc toKeccak(r: var NodeKey, x: openArray[byte]) {.inline.} =
  r.data[0..31] = x[0..31]
  r.usedBytes = 32

proc toKeccak(r: var NodeKey, z: byte, x: openArray[byte]) {.inline.} =
  r.data[0] = z
  r.data[1..31] = x[0..30]
  r.usedBytes = 32

proc append(r: var RlpWriter, n: NodeKey) =
  if n.usedBytes < 32:
    r.append rlpFromBytes(n.data.toOpenArray(0, n.usedBytes-1))
  else:
    r.append n.data.toOpenArray(0, n.usedBytes-1)

proc toNodeKey(t: var TreeBuilder, z: openArray[byte]): NodeKey =
  if z.len < 32:
    result.usedBytes = z.len
    result.data[0..z.len-1] = z[0..z.len-1]
  else:
    result.data = keccakHash(z).data
    result.usedBytes = 32
    t.db.put(result.data, z)

proc toNodeKey(z: openArray[byte]): NodeKey =
  if z.len >= 32:
    raise newException(ParsingError, "Failed when try to convert short rlp to NodeKey")
  result.usedBytes = z.len
  result.data[0..z.len-1] = z[0..z.len-1]

proc forceSmallNodeKeyToHash(t: var TreeBuilder, r: NodeKey): NodeKey =
  let hash = keccakHash(r.data.toOpenArray(0, r.usedBytes-1))
  t.db.put(hash.data, r.data.toOpenArray(0, r.usedBytes-1))
  result.data = hash.data
  result.usedBytes = 32

proc writeCode(t: var TreeBuilder, code: openArray[byte]): Hash256 =
  result = keccakHash(code)
  put(t.db, result.data, code)

proc branchNode(t: var TreeBuilder, depth: int, storageMode: bool): NodeKey
proc extensionNode(t: var TreeBuilder, depth: int, storageMode: bool): NodeKey
proc accountNode(t: var TreeBuilder, depth: int): NodeKey
proc accountStorageLeafNode(t: var TreeBuilder, depth: int): NodeKey
proc hashNode(t: var TreeBuilder, depth: int, storageMode: bool): NodeKey
proc treeNode(t: var TreeBuilder, depth: int = 0, storageMode = false): NodeKey

proc buildTree*(t: var TreeBuilder): KeccakHash
  {.raises: [ContractCodeError, Defect, IOError, ParsingError, Exception].} =
  let version = t.safeReadByte().int
  if version != BlockWitnessVersion.int:
    raise newException(ParsingError, "Wrong block witness version")

  # one or more trees

  # we only parse one tree here
  let metadataType = t.safeReadByte().int
  if metadataType != MetadataNothing.int:
    raise newException(ParsingError, "This tree builder support no metadata")

  var res = treeNode(t)
  if res.usedBytes != 32:
    raise newException(ParsingError, "Buildtree should produce hash")

  result.data = res.data

# after the block witness spec mention how to split the big tree into
# chunks, modify this buildForest into chunked witness tree builder
proc buildForest*(t: var TreeBuilder): seq[KeccakHash]
  {.raises: [ContractCodeError, Defect, IOError, ParsingError, Exception].} =
  let version = t.safeReadByte().int
  if version != BlockWitnessVersion.int:
    raise newException(ParsingError, "Wrong block witness version")

  while t.readable:
    let metadataType = t.safeReadByte().int
    if metadataType != MetadataNothing.int:
      raise newException(ParsingError, "This tree builder support no metadata")

    var res = treeNode(t)
    if res.usedBytes != 32:
      raise newException(ParsingError, "Buildtree should produce hash")

    result.add KeccakHash(data: res.data)

proc treeNode(t: var TreeBuilder, depth: int, storageMode = false): NodeKey =
  if depth > 64:
    raise newException(ParsingError, "invalid trie structure")

  let nodeType = safeReadEnum(t, TrieNodeType)
  case nodeType
  of BranchNodeType: result = t.branchNode(depth, storageMode)
  of ExtensionNodeType: result = t.extensionNode(depth, storageMode)
  of AccountNodeType:
    if storageMode:
      # parse account storage leaf node
      result = t.accountStorageLeafNode(depth)
    else:
      result = t.accountNode(depth)
  of HashNodeType: result = t.hashNode(depth, storageMode)

  if depth == 0 and result.usedBytes < 32:
    result = t.forceSmallNodeKeyToHash(result)

proc branchNode(t: var TreeBuilder, depth: int, storageMode: bool): NodeKey =
  if depth >= 64:
    raise newException(ParsingError, "invalid trie structure")

  let mask = constructBranchMask(t.safeReadByte, t.safeReadByte)

  when defined(debugDepth):
    let readDepth = t.safeReadByte().int
    doAssert(readDepth == depth, "branchNode " & $readDepth & " vs. " & $depth)

  when defined(debugHash):
    var hash: NodeKey
    toKeccak(hash, t.read(32))

  var r = initRlpList(17)

  for i in 0 ..< 16:
    if mask.branchMaskBitIsSet(i):
      r.append t.treeNode(depth+1, storageMode)
    else:
      r.append ""

  if branchMaskBitIsSet(mask, 16):
    raise newException(ParsingError, "The 17th elem of branch node should empty")

  # 17th elem should always empty
  r.append ""

  result = t.toNodeKey(r.finish)

  when defined(debugHash):
    if result != hash:
      debugEcho "DEPTH: ", depth
      debugEcho "result: ", result.data.toHex, " vs. ", hash.data.toHex

func hexPrefixExtension(r: var RlpWriter, x: openArray[byte], nibblesLen: int) =
  # extension hexPrefix
  doAssert(nibblesLen >= 1 and nibblesLen <= 64)
  var bytes: array[33, byte]
  let last = nibblesLen div 2
  if (nibblesLen mod 2) == 0: # even
    bytes[0] = 0.byte
    var i = 1
    for y in x:
      bytes[i] = y
      inc i
  else: # odd
    bytes[0] = 0b0001_0000.byte or (x[0] shr 4)
    for i in 1..last:
      bytes[i] = (x[i-1] shl 4) or (x[i] shr 4)

  r.append toOpenArray(bytes, 0, last)

func hexPrefixLeaf(r: var RlpWriter, x: openArray[byte], depth: int) =
  # leaf hexPrefix
  doAssert(depth >= 0 and depth <= 64)
  let nibblesLen = 64 - depth
  var bytes: array[33, byte]
  var start = depth div 2
  if (nibblesLen mod 2) == 0: # even
    bytes[0] = 0b0010_0000.byte
  else: # odd
    bytes[0] = 0b0011_0000.byte or (x[start] and 0x0F)
    inc start

  var i = 1
  for z in start..31:
    bytes[i] = x[z]
    inc i

  r.append toOpenArray(bytes, 0, nibblesLen div 2)

proc extensionNode(t: var TreeBuilder, depth: int, storageMode: bool): NodeKey =
  if depth >= 63:
    raise newException(ParsingError, "invalid trie structure")

  let nibblesLen = t.safeReadByte().int
  if nibblesLen > 64 or nibblesLen < 1:
    raise newException(ParsingError, "nibblesLen should between 1..64")

  var r = initRlpList(2)
  let pathLen = nibblesLen div 2 + nibblesLen mod 2
  safeReadBytes(t, pathLen):
    r.hexPrefixExtension(t.read(pathLen), nibblesLen)

  when defined(debugDepth):
    let readDepth = t.safeReadByte().int
    doAssert(readDepth == depth, "extensionNode " & $readDepth & " vs. " & $depth)

  when defined(debugHash):
    var hash: NodeKey
    toKeccak(hash, t.read(32))

  if nibblesLen + depth > 64 or nibblesLen + depth < 1:
    raise newException(ParsingError, "depth should between 1..64")

  let nodeType = safeReadEnum(t, TrieNodeType)
  case nodeType
  of BranchNodeType: r.append t.branchNode(depth + nibblesLen, storageMode)
  of HashNodeType: r.append t.hashNode(depth, storageMode)
  else: raise newException(ParsingError, "wrong type during parsing child of extension node")

  result = t.toNodeKey(r.finish)

  when defined(debugHash):
    if result != hash:
      debugEcho "DEPTH: ", depth
    doAssert(result == hash, "EXT HASH DIFF " & result.data.toHex & " vs. " & hash.data.toHex)

func toAddress(x: openArray[byte]): EthAddress =
  result[0..19] = x[0..19]

proc readAddress(t: var TreeBuilder): Hash256 =
  safeReadBytes(t, 20):
    let address = toAddress(t.read(20))
    result = keccakHash(address)
    t.keys.add AccountAndSlots(address: address)

proc readCodeLen(t: var TreeBuilder): int =
  let codeLen = t.readUVarint32()
  if wfEIP170 in t.flags and codeLen > EIP170_MAX_CODE_SIZE:
    raise newException(ContractCodeError, "code len exceed EIP170 code size limit: " & $codeLen)
  t.keys[^1].codeLen = codeLen.int
  result = codeLen.int

proc readHashNode(t: var TreeBuilder, depth: int, storageMode: bool): NodeKey =
  let nodeType = safeReadEnum(t, TrieNodeType)
  if nodeType != HashNodeType:
    raise newException(ParsingError, "hash node expected but got " & $nodeType)
  result = t.hashNode(depth, storageMode)

proc readByteCode(t: var TreeBuilder, acc: var Account, depth: int) =
  let bytecodeType = safeReadEnum(t, BytecodeType)
  case bytecodeType
  of CodeTouched:
    let codeLen = t.readCodeLen()
    safeReadBytes(t, codeLen):
      acc.codeHash = t.writeCode(t.read(codeLen))
  of CodeUntouched:
    # readCodeLen already save the codeLen
    # along with recovered address
    # we could discard it here
    discard t.readCodeLen()

    let codeHash = t.readHashNode(depth, false)
    doAssert(codeHash.usedBytes == 32)
    acc.codeHash.data = codeHash.data

proc accountNode(t: var TreeBuilder, depth: int): NodeKey =
  if depth >= 65:
    raise newException(ParsingError, "invalid trie structure")

  when defined(debugHash):
    let len = t.safeReadU32().int
    let node = @(t.read(len))
    let nodeKey = t.toNodeKey(node)

  when defined(debugDepth):
    let readDepth = t.safeReadByte().int
    doAssert(readDepth == depth, "accountNode " & $readDepth & " vs. " & $depth)

  let accountType = safeReadEnum(t, AccountType)
  let addressHash = t.readAddress()

  var r = initRlpList(2)
  r.hexPrefixLeaf(addressHash.data, depth)

  var acc = Account(
    balance: t.readUVarint256(),
    nonce: t.readUVarint256().truncate(AccountNonce)
  )

  case accountType
  of SimpleAccountType:
    acc.codeHash = blankStringHash
    acc.storageRoot = emptyRlpHash
  of ExtendedAccountType:
    t.readByteCode(acc, depth)

    # switch to account storage parsing mode
    # and reset the depth
    let storageRoot = t.treeNode(0, storageMode = true)
    doAssert(storageRoot.usedBytes == 32)
    acc.storageRoot.data = storageRoot.data

  r.append rlp.encode(acc)

  let nodeRes = r.finish
  result = t.toNodeKey(nodeRes)

  when defined(debugHash):
    if result != nodeKey:
      debugEcho "Address: ", t.keys[^1].address.toHex
      debugEcho "addressHash: ", addressHash.data.toHex
      debugEcho "depth: ", depth
      debugEcho "result.usedBytes: ", result.usedBytes
      debugEcho "nodeKey.usedBytes: ", nodeKey.usedBytes
      var rlpa = rlpFromBytes(node)
      var rlpb = rlpFromBytes(nodeRes)
      debugEcho "Expected: ", inspect(rlpa)
      debugEcho "Actual: ", inspect(rlpb)
      var a = rlpa.listElem(1).toBytes.decode(Account)
      var b = rlpb.listElem(1).toBytes.decode(Account)
      debugEcho "Expected: ", a
      debugEcho "Actual: ", b

    doAssert(result == nodeKey, "account node parsing error")

func toStorageSlot(x: openArray[byte]): StorageSlot =
  result[0..31] = x[0..31]

proc readStorageSlot(t: var TreeBuilder): Hash256 =
  safeReadBytes(t, 32):
    let slot = toStorageSlot(t.read(32))
    result = keccakHash(slot)
    t.keys[^1].slots.add slot

proc accountStorageLeafNode(t: var TreeBuilder, depth: int): NodeKey =
  if depth >= 65:
    raise newException(ParsingError, "invalid trie structure")

  when defined(debugHash):
    let len = t.safeReadU32().int
    let node = @(t.read(len))
    let nodeKey = t.toNodeKey(node)

  when defined(debugDepth):
    let readDepth = t.safeReadByte().int
    doAssert(readDepth == depth, "accountNode " & $readDepth & " vs. " & $depth)

  var r = initRlpList(2)
  let slotHash = t.readStorageSlot()
  r.hexPrefixLeaf(slotHash.data, depth)

  safeReadBytes(t, 32):
    let val = UInt256.fromBytesBE(t.read(32))
    r.append rlp.encode(val)
    result = t.toNodeKey(r.finish)

  when defined(debugHash):
    doAssert(result == nodeKey, "account storage leaf node parsing error")

proc hashNode(t: var TreeBuilder, depth: int, storageMode: bool): NodeKey =
  if storageMode and depth >= 9:
    let z = t.safeReadByte()
    if z == ShortRlpPrefix:
      let rlpLen = t.safeReadByte().int
      if rlpLen == 0:
        safeReadBytes(t, 31):
          result.toKeccak(0, t.read(31))
      else:
        safeReadBytes(t, rlpLen):
          result = toNodeKey(t.read(rlpLen))
    else:
      safeReadBytes(t, 31):
        result.toKeccak(z, t.read(31))
  else:
    safeReadBytes(t, 32):
      result.toKeccak(t.read(32))