mirror of https://github.com/status-im/nim-eth.git
Merge pull request #447 from etan-status/rm-ssz
remove outdated and incorrect SSZ code
This commit is contained in:
Etan Kissling
GitHub
commit
923fc428f2
|
|
@ -66,9 +66,6 @@ task test_trie, "Run trie tests":
|
||||||
task test_db, "Run db tests":
|
task test_db, "Run db tests":
|
||||||
runTest("tests/db/all_tests")
|
runTest("tests/db/all_tests")
|
||||||
|
|
||||||
task test_ssz, "Run ssz tests":
|
|
||||||
runTest("tests/ssz/all_tests")
|
|
||||||
|
|
||||||
task test_utp, "Run utp tests":
|
task test_utp, "Run utp tests":
|
||||||
runTest("tests/utp/all_utp_tests")
|
runTest("tests/utp/all_utp_tests")
|
||||||
|
|
||||||
|
|
@ -84,7 +81,6 @@ task test, "Run all tests":
|
||||||
test_p2p_task()
|
test_p2p_task()
|
||||||
test_trie_task()
|
test_trie_task()
|
||||||
test_db_task()
|
test_db_task()
|
||||||
test_ssz_task()
|
|
||||||
test_utp_task()
|
test_utp_task()
|
||||||
|
|
||||||
task test_discv5_full, "Run discovery v5 and its dependencies tests":
|
task test_discv5_full, "Run discovery v5 and its dependencies tests":
|
||||||
|
|
|
||||||
|
|
@ -1,313 +0,0 @@
|
||||||
# nim-eth
|
|
||||||
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
|
||||||
# Licensed and distributed under either of
|
|
||||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
|
||||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
|
||||||
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
|
||||||
|
|
||||||
{.push raises: [Defect].}
|
|
||||||
|
|
||||||
import
|
|
||||||
stew/[bitops2, endians2, ptrops]
|
|
||||||
|
|
||||||
type
|
|
||||||
Bytes = seq[byte]
|
|
||||||
|
|
||||||
BitSeq* = distinct Bytes
|
|
||||||
## The current design of BitSeq tries to follow precisely
|
|
||||||
## the bitwise representation of the SSZ bitlists.
|
|
||||||
## This is a relatively compact representation, but as
|
|
||||||
## evident from the code below, many of the operations
|
|
||||||
## are not trivial.
|
|
||||||
|
|
||||||
BitArray*[bits: static int] = object
|
|
||||||
bytes*: array[(bits + 7) div 8, byte]
|
|
||||||
|
|
||||||
func bitsLen*(bytes: openArray[byte]): int =
|
|
||||||
let
|
|
||||||
bytesCount = bytes.len
|
|
||||||
lastByte = bytes[bytesCount - 1]
|
|
||||||
markerPos = log2trunc(lastByte)
|
|
||||||
|
|
||||||
bytesCount * 8 - (8 - markerPos)
|
|
||||||
|
|
||||||
template len*(s: BitSeq): int =
|
|
||||||
bitsLen(Bytes s)
|
|
||||||
|
|
||||||
template len*(a: BitArray): int =
|
|
||||||
a.bits
|
|
||||||
|
|
||||||
func add*(s: var BitSeq, value: bool) =
|
|
||||||
let
|
|
||||||
lastBytePos = s.Bytes.len - 1
|
|
||||||
lastByte = s.Bytes[lastBytePos]
|
|
||||||
|
|
||||||
if (lastByte and byte(128)) == 0:
|
|
||||||
# There is at least one leading zero, so we have enough
|
|
||||||
# room to store the new bit
|
|
||||||
let markerPos = log2trunc(lastByte)
|
|
||||||
s.Bytes[lastBytePos].changeBit markerPos, value
|
|
||||||
s.Bytes[lastBytePos].setBit markerPos + 1
|
|
||||||
else:
|
|
||||||
s.Bytes[lastBytePos].changeBit 7, value
|
|
||||||
s.Bytes.add byte(1)
|
|
||||||
|
|
||||||
func toBytesLE(x: uint): array[sizeof(x), byte] =
|
|
||||||
# stew/endians2 supports explicitly sized uints only
|
|
||||||
when sizeof(uint) == 4:
|
|
||||||
static: doAssert sizeof(uint) == sizeof(uint32)
|
|
||||||
toBytesLE(x.uint32)
|
|
||||||
elif sizeof(uint) == 8:
|
|
||||||
static: doAssert sizeof(uint) == sizeof(uint64)
|
|
||||||
toBytesLE(x.uint64)
|
|
||||||
else:
|
|
||||||
static: doAssert false, "requires a 32-bit or 64-bit platform"
|
|
||||||
|
|
||||||
func loadLEBytes(WordType: type, bytes: openArray[byte]): WordType =
|
|
||||||
# TODO: this is a temporary proc until the endians API is improved
|
|
||||||
var shift = 0
|
|
||||||
for b in bytes:
|
|
||||||
result = result or (WordType(b) shl shift)
|
|
||||||
shift += 8
|
|
||||||
|
|
||||||
func storeLEBytes(value: SomeUnsignedInt, dst: var openArray[byte]) =
|
|
||||||
doAssert dst.len <= sizeof(value)
|
|
||||||
let bytesLE = toBytesLE(value)
|
|
||||||
copyMem(addr dst[0], unsafeAddr bytesLE[0], dst.len)
|
|
||||||
|
|
||||||
template loopOverWords(lhs, rhs: BitSeq,
|
|
||||||
lhsIsVar, rhsIsVar: static bool,
|
|
||||||
WordType: type,
|
|
||||||
lhsBits, rhsBits, body: untyped) =
|
|
||||||
const hasRhs = astToStr(lhs) != astToStr(rhs)
|
|
||||||
|
|
||||||
let bytesCount = len Bytes(lhs)
|
|
||||||
when hasRhs: doAssert len(Bytes(rhs)) == bytesCount
|
|
||||||
|
|
||||||
var fullWordsCount = bytesCount div sizeof(WordType)
|
|
||||||
let lastWordSize = bytesCount mod sizeof(WordType)
|
|
||||||
|
|
||||||
block:
|
|
||||||
var lhsWord: WordType
|
|
||||||
when hasRhs:
|
|
||||||
var rhsWord: WordType
|
|
||||||
var firstByteOfLastWord, lastByteOfLastWord: int
|
|
||||||
|
|
||||||
# TODO: Returning a `var` value from an iterator is always safe due to
|
|
||||||
# the way inlining works, but currently the compiler reports an error
|
|
||||||
# when a local variable escapes. We have to cheat it with this location
|
|
||||||
# obfuscation through pointers:
|
|
||||||
template lhsBits: auto = (addr(lhsWord))[]
|
|
||||||
|
|
||||||
when hasRhs:
|
|
||||||
template rhsBits: auto = (addr(rhsWord))[]
|
|
||||||
|
|
||||||
template lastWordBytes(bitseq): auto =
|
|
||||||
Bytes(bitseq).toOpenArray(firstByteOfLastWord, lastByteOfLastWord)
|
|
||||||
|
|
||||||
template initLastWords =
|
|
||||||
lhsWord = loadLEBytes(WordType, lastWordBytes(lhs))
|
|
||||||
when hasRhs: rhsWord = loadLEBytes(WordType, lastWordBytes(rhs))
|
|
||||||
|
|
||||||
if lastWordSize == 0:
|
|
||||||
firstByteOfLastWord = bytesCount - sizeof(WordType)
|
|
||||||
lastByteOfLastWord = bytesCount - 1
|
|
||||||
dec fullWordsCount
|
|
||||||
else:
|
|
||||||
firstByteOfLastWord = bytesCount - lastWordSize
|
|
||||||
lastByteOfLastWord = bytesCount - 1
|
|
||||||
|
|
||||||
initLastWords()
|
|
||||||
let markerPos = log2trunc(lhsWord)
|
|
||||||
when hasRhs: doAssert log2trunc(rhsWord) == markerPos
|
|
||||||
|
|
||||||
lhsWord.clearBit markerPos
|
|
||||||
when hasRhs: rhsWord.clearBit markerPos
|
|
||||||
|
|
||||||
body
|
|
||||||
|
|
||||||
when lhsIsVar or rhsIsVar:
|
|
||||||
let
|
|
||||||
markerBit = uint(1 shl markerPos)
|
|
||||||
mask = markerBit - 1'u
|
|
||||||
|
|
||||||
when lhsIsVar:
|
|
||||||
let lhsEndResult = (lhsWord and mask) or markerBit
|
|
||||||
storeLEBytes(lhsEndResult, lastWordBytes(lhs))
|
|
||||||
|
|
||||||
when rhsIsVar:
|
|
||||||
let rhsEndResult = (rhsWord and mask) or markerBit
|
|
||||||
storeLEBytes(rhsEndResult, lastWordBytes(rhs))
|
|
||||||
|
|
||||||
var lhsCurrAddr = cast[ptr WordType](unsafeAddr Bytes(lhs)[0])
|
|
||||||
let lhsEndAddr = offset(lhsCurrAddr, fullWordsCount)
|
|
||||||
when hasRhs:
|
|
||||||
var rhsCurrAddr = cast[ptr WordType](unsafeAddr Bytes(rhs)[0])
|
|
||||||
|
|
||||||
while lhsCurrAddr < lhsEndAddr:
|
|
||||||
template lhsBits: auto = lhsCurrAddr[]
|
|
||||||
when hasRhs:
|
|
||||||
template rhsBits: auto = rhsCurrAddr[]
|
|
||||||
|
|
||||||
body
|
|
||||||
|
|
||||||
lhsCurrAddr = offset(lhsCurrAddr, 1)
|
|
||||||
when hasRhs: rhsCurrAddr = offset(rhsCurrAddr, 1)
|
|
||||||
|
|
||||||
iterator words*(x: var BitSeq): var uint =
|
|
||||||
loopOverWords(x, x, true, false, uint, word, wordB):
|
|
||||||
yield word
|
|
||||||
|
|
||||||
iterator words*(x: BitSeq): uint =
|
|
||||||
loopOverWords(x, x, false, false, uint, word, word):
|
|
||||||
yield word
|
|
||||||
|
|
||||||
iterator words*(a, b: BitSeq): (uint, uint) =
|
|
||||||
loopOverWords(a, b, false, false, uint, wordA, wordB):
|
|
||||||
yield (wordA, wordB)
|
|
||||||
|
|
||||||
iterator words*(a: var BitSeq, b: BitSeq): (var uint, uint) =
|
|
||||||
loopOverWords(a, b, true, false, uint, wordA, wordB):
|
|
||||||
yield (wordA, wordB)
|
|
||||||
|
|
||||||
iterator words*(a, b: var BitSeq): (var uint, var uint) =
|
|
||||||
loopOverWords(a, b, true, true, uint, wordA, wordB):
|
|
||||||
yield (wordA, wordB)
|
|
||||||
|
|
||||||
func `[]`*(s: BitSeq, pos: Natural): bool {.inline.} =
|
|
||||||
doAssert pos < s.len
|
|
||||||
s.Bytes.getBit pos
|
|
||||||
|
|
||||||
func `[]=`*(s: var BitSeq, pos: Natural, value: bool) {.inline.} =
|
|
||||||
doAssert pos < s.len
|
|
||||||
s.Bytes.changeBit pos, value
|
|
||||||
|
|
||||||
func setBit*(s: var BitSeq, pos: Natural) {.inline.} =
|
|
||||||
doAssert pos < s.len
|
|
||||||
setBit s.Bytes, pos
|
|
||||||
|
|
||||||
func clearBit*(s: var BitSeq, pos: Natural) {.inline.} =
|
|
||||||
doAssert pos < s.len
|
|
||||||
clearBit s.Bytes, pos
|
|
||||||
|
|
||||||
func init*(T: type BitSeq, len: int): T =
|
|
||||||
result = BitSeq newSeq[byte](1 + len div 8)
|
|
||||||
Bytes(result).setBit len
|
|
||||||
|
|
||||||
func init*(T: type BitArray): T =
|
|
||||||
# The default zero-initializatio is fine
|
|
||||||
discard
|
|
||||||
|
|
||||||
template `[]`*(a: BitArray, pos: Natural): bool =
|
|
||||||
getBit a.bytes, pos
|
|
||||||
|
|
||||||
template `[]=`*(a: var BitArray, pos: Natural, value: bool) =
|
|
||||||
changeBit a.bytes, pos, value
|
|
||||||
|
|
||||||
template setBit*(a: var BitArray, pos: Natural) =
|
|
||||||
setBit a.bytes, pos
|
|
||||||
|
|
||||||
template clearBit*(a: var BitArray, pos: Natural) =
|
|
||||||
clearBit a.bytes, pos
|
|
||||||
|
|
||||||
# TODO: Submit this to the standard library as `cmp`
|
|
||||||
# At the moment, it doesn't work quite well because Nim selects
|
|
||||||
# the generic cmp[T] from the system module instead of choosing
|
|
||||||
# the openArray overload
|
|
||||||
func compareArrays[T](a, b: openArray[T]): int =
|
|
||||||
result = cmp(a.len, b.len)
|
|
||||||
if result != 0: return
|
|
||||||
|
|
||||||
for i in 0 ..< a.len:
|
|
||||||
result = cmp(a[i], b[i])
|
|
||||||
if result != 0: return
|
|
||||||
|
|
||||||
template cmp*(a, b: BitSeq): int =
|
|
||||||
compareArrays(Bytes a, Bytes b)
|
|
||||||
|
|
||||||
template `==`*(a, b: BitSeq): bool =
|
|
||||||
cmp(a, b) == 0
|
|
||||||
|
|
||||||
func `$`*(a: BitSeq | BitArray): string =
|
|
||||||
let length = a.len
|
|
||||||
result = newStringOfCap(2 + length)
|
|
||||||
result.add "0b"
|
|
||||||
for i in countdown(length - 1, 0):
|
|
||||||
result.add if a[i]: '1' else: '0'
|
|
||||||
|
|
||||||
func incl*(tgt: var BitSeq, src: BitSeq) =
|
|
||||||
# Update `tgt` to include the bits of `src`, as if applying `or` to each bit
|
|
||||||
doAssert tgt.len == src.len
|
|
||||||
for tgtWord, srcWord in words(tgt, src):
|
|
||||||
tgtWord = tgtWord or srcWord
|
|
||||||
|
|
||||||
func overlaps*(a, b: BitSeq): bool =
|
|
||||||
for wa, wb in words(a, b):
|
|
||||||
if (wa and wb) != 0:
|
|
||||||
return true
|
|
||||||
|
|
||||||
func countOverlap*(a, b: BitSeq): int =
|
|
||||||
var res = 0
|
|
||||||
for wa, wb in words(a, b):
|
|
||||||
res += countOnes(wa and wb)
|
|
||||||
res
|
|
||||||
|
|
||||||
func isSubsetOf*(a, b: BitSeq): bool =
|
|
||||||
let alen = a.len
|
|
||||||
doAssert b.len == alen
|
|
||||||
for i in 0 ..< alen:
|
|
||||||
if a[i] and not b[i]:
|
|
||||||
return false
|
|
||||||
true
|
|
||||||
|
|
||||||
func isZeros*(x: BitSeq): bool =
|
|
||||||
for w in words(x):
|
|
||||||
if w != 0: return false
|
|
||||||
return true
|
|
||||||
|
|
||||||
func countOnes*(x: BitSeq): int =
|
|
||||||
# Count the number of set bits
|
|
||||||
var res = 0
|
|
||||||
for w in words(x):
|
|
||||||
res += w.countOnes()
|
|
||||||
res
|
|
||||||
|
|
||||||
func clear*(x: var BitSeq) =
|
|
||||||
for w in words(x):
|
|
||||||
w = 0
|
|
||||||
|
|
||||||
func countZeros*(x: BitSeq): int =
|
|
||||||
x.len() - x.countOnes()
|
|
||||||
|
|
||||||
template bytes*(x: BitSeq): untyped =
|
|
||||||
seq[byte](x)
|
|
||||||
|
|
||||||
iterator items*(x: BitArray): bool =
|
|
||||||
for i in 0..<x.bits:
|
|
||||||
yield x[i]
|
|
||||||
|
|
||||||
iterator pairs*(x: BitArray): (int, bool) =
|
|
||||||
for i in 0..<x.bits:
|
|
||||||
yield (i, x[i])
|
|
||||||
|
|
||||||
func incl*(a: var BitArray, b: BitArray) =
|
|
||||||
# Update `a` to include the bits of `b`, as if applying `or` to each bit
|
|
||||||
for i in 0..<a.bytes.len:
|
|
||||||
a[i] = a[i] or b[i]
|
|
||||||
|
|
||||||
func clear*(a: var BitArray) =
|
|
||||||
for b in a.bytes.mitems(): b = 0
|
|
||||||
|
|
||||||
# Set operations
|
|
||||||
func `+`*(a, b: BitArray): BitArray =
|
|
||||||
for i in 0..<a.bytes.len:
|
|
||||||
result.bytes[i] = a.bytes[i] or b.bytes[i]
|
|
||||||
|
|
||||||
func `-`*(a, b: BitArray): BitArray =
|
|
||||||
for i in 0..<a.bytes.len:
|
|
||||||
result.bytes[i] = a.bytes[i] and (not b.bytes[i])
|
|
||||||
|
|
||||||
iterator oneIndices*(a: BitArray): int =
|
|
||||||
for i in 0..<a.len:
|
|
||||||
if a[i]: yield i
|
|
||||||
|
|
||||||
|
|
@ -1,218 +0,0 @@
|
||||||
# nim-eth - Limited SSZ implementation
|
|
||||||
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
|
||||||
# Licensed and distributed under either of
|
|
||||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
|
||||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
|
||||||
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
|
||||||
|
|
||||||
{.push raises: [Defect].}
|
|
||||||
|
|
||||||
import
|
|
||||||
std/[typetraits, options],
|
|
||||||
stew/[endians2, objects],
|
|
||||||
./types
|
|
||||||
|
|
||||||
template raiseIncorrectSize*(T: type) =
|
|
||||||
const typeName = name(T)
|
|
||||||
raise newException(MalformedSszError,
|
|
||||||
"SSZ " & typeName & " input of incorrect size")
|
|
||||||
|
|
||||||
template setOutputSize[R, T](a: var array[R, T], length: int) =
|
|
||||||
if length != a.len:
|
|
||||||
raiseIncorrectSize a.type
|
|
||||||
|
|
||||||
proc setOutputSize(list: var List, length: int) {.raises: [SszError, Defect].} =
|
|
||||||
if not list.setLen length:
|
|
||||||
raise newException(MalformedSszError, "SSZ list maximum size exceeded")
|
|
||||||
|
|
||||||
# fromSszBytes copies the wire representation to a Nim variable,
|
|
||||||
# assuming there's enough data in the buffer
|
|
||||||
func fromSszBytes*(T: type UintN, data: openArray[byte]):
|
|
||||||
T {.raises: [MalformedSszError, Defect].} =
|
|
||||||
## Convert directly to bytes the size of the int. (e.g. ``uint16 = 2 bytes``)
|
|
||||||
## All integers are serialized as **little endian**.
|
|
||||||
if data.len != sizeof(result):
|
|
||||||
raiseIncorrectSize T
|
|
||||||
|
|
||||||
T.fromBytesLE(data)
|
|
||||||
|
|
||||||
func fromSszBytes*(T: type bool, data: openArray[byte]):
|
|
||||||
T {.raises: [MalformedSszError, Defect].} =
|
|
||||||
# Strict: only allow 0 or 1
|
|
||||||
if data.len != 1 or byte(data[0]) > byte(1):
|
|
||||||
raise newException(MalformedSszError, "invalid boolean value")
|
|
||||||
data[0] == 1
|
|
||||||
|
|
||||||
template fromSszBytes*(T: type BitSeq, bytes: openArray[byte]): auto =
|
|
||||||
BitSeq @bytes
|
|
||||||
|
|
||||||
proc `[]`[T, U, V](s: openArray[T], x: HSlice[U, V]) {.error:
|
|
||||||
"Please don't use openArray's [] as it allocates a result sequence".}
|
|
||||||
|
|
||||||
template checkForForbiddenBits(ResulType: type,
|
|
||||||
input: openArray[byte],
|
|
||||||
expectedBits: static int64) =
|
|
||||||
## This checks if the input contains any bits set above the maximum
|
|
||||||
## sized allowed. We only need to check the last byte to verify this:
|
|
||||||
const bitsInLastByte = (expectedBits mod 8)
|
|
||||||
when bitsInLastByte != 0:
|
|
||||||
# As an example, if there are 3 bits expected in the last byte,
|
|
||||||
# we calculate a bitmask equal to 11111000. If the input has any
|
|
||||||
# raised bits in range of the bitmask, this would be a violation
|
|
||||||
# of the size of the BitArray:
|
|
||||||
const forbiddenBitsMask = byte(byte(0xff) shl bitsInLastByte)
|
|
||||||
|
|
||||||
if (input[^1] and forbiddenBitsMask) != 0:
|
|
||||||
raiseIncorrectSize ResulType
|
|
||||||
|
|
||||||
func readSszValue*[T](input: openArray[byte], val: var T)
|
|
||||||
{.raises: [SszError, Defect].} =
|
|
||||||
mixin fromSszBytes, toSszType
|
|
||||||
|
|
||||||
template readOffsetUnchecked(n: int): uint32 {.used.}=
|
|
||||||
fromSszBytes(uint32, input.toOpenArray(n, n + offsetSize - 1))
|
|
||||||
|
|
||||||
template readOffset(n: int): int {.used.} =
|
|
||||||
let offset = readOffsetUnchecked(n)
|
|
||||||
if offset > input.len.uint32:
|
|
||||||
raise newException(MalformedSszError, "SSZ list element offset points past the end of the input")
|
|
||||||
int(offset)
|
|
||||||
|
|
||||||
when val is BitList:
|
|
||||||
if input.len == 0:
|
|
||||||
raise newException(MalformedSszError, "Invalid empty SSZ BitList value")
|
|
||||||
|
|
||||||
# Since our BitLists have an in-memory representation that precisely
|
|
||||||
# matches their SSZ encoding, we can deserialize them as regular Lists:
|
|
||||||
const maxExpectedSize = (val.maxLen div 8) + 1
|
|
||||||
type MatchingListType = List[byte, maxExpectedSize]
|
|
||||||
|
|
||||||
when false:
|
|
||||||
# TODO: Nim doesn't like this simple type coercion,
|
|
||||||
# we'll rely on `cast` for now (see below)
|
|
||||||
readSszValue(input, MatchingListType val)
|
|
||||||
else:
|
|
||||||
static:
|
|
||||||
# As a sanity check, we verify that the coercion is accepted by the compiler:
|
|
||||||
doAssert MatchingListType(val) is MatchingListType
|
|
||||||
readSszValue(input, cast[ptr MatchingListType](addr val)[])
|
|
||||||
|
|
||||||
let resultBytesCount = len bytes(val)
|
|
||||||
|
|
||||||
if bytes(val)[resultBytesCount - 1] == 0:
|
|
||||||
raise newException(MalformedSszError, "SSZ BitList is not properly terminated")
|
|
||||||
|
|
||||||
if resultBytesCount == maxExpectedSize:
|
|
||||||
checkForForbiddenBits(T, input, val.maxLen + 1)
|
|
||||||
|
|
||||||
elif val is List|array:
|
|
||||||
type E = type val[0]
|
|
||||||
|
|
||||||
when E is byte:
|
|
||||||
val.setOutputSize input.len
|
|
||||||
if input.len > 0:
|
|
||||||
copyMem(addr val[0], unsafeAddr input[0], input.len)
|
|
||||||
|
|
||||||
elif isFixedSize(E):
|
|
||||||
const elemSize = fixedPortionSize(E)
|
|
||||||
if input.len mod elemSize != 0:
|
|
||||||
var ex = new SszSizeMismatchError
|
|
||||||
ex.deserializedType = cstring typetraits.name(T)
|
|
||||||
ex.actualSszSize = input.len
|
|
||||||
ex.elementSize = elemSize
|
|
||||||
raise ex
|
|
||||||
val.setOutputSize input.len div elemSize
|
|
||||||
for i in 0 ..< val.len:
|
|
||||||
let offset = i * elemSize
|
|
||||||
readSszValue(input.toOpenArray(offset, offset + elemSize - 1), val[i])
|
|
||||||
|
|
||||||
else:
|
|
||||||
if input.len == 0:
|
|
||||||
# This is an empty list.
|
|
||||||
# The default initialization of the return value is fine.
|
|
||||||
val.setOutputSize 0
|
|
||||||
return
|
|
||||||
elif input.len < offsetSize:
|
|
||||||
raise newException(MalformedSszError, "SSZ input of insufficient size")
|
|
||||||
|
|
||||||
var offset = readOffset 0
|
|
||||||
let resultLen = offset div offsetSize
|
|
||||||
|
|
||||||
if resultLen == 0:
|
|
||||||
# If there are too many elements, other constraints detect problems
|
|
||||||
# (not monotonically increasing, past end of input, or last element
|
|
||||||
# not matching up with its nextOffset properly)
|
|
||||||
raise newException(MalformedSszError, "SSZ list incorrectly encoded of zero length")
|
|
||||||
|
|
||||||
val.setOutputSize resultLen
|
|
||||||
for i in 1 ..< resultLen:
|
|
||||||
let nextOffset = readOffset(i * offsetSize)
|
|
||||||
if nextOffset <= offset:
|
|
||||||
raise newException(MalformedSszError, "SSZ list element offsets are not monotonically increasing")
|
|
||||||
else:
|
|
||||||
readSszValue(input.toOpenArray(offset, nextOffset - 1), val[i - 1])
|
|
||||||
offset = nextOffset
|
|
||||||
|
|
||||||
readSszValue(input.toOpenArray(offset, input.len - 1), val[resultLen - 1])
|
|
||||||
|
|
||||||
elif val is UintN|bool:
|
|
||||||
val = fromSszBytes(T, input)
|
|
||||||
|
|
||||||
elif val is BitArray:
|
|
||||||
if sizeof(val) != input.len:
|
|
||||||
raiseIncorrectSize(T)
|
|
||||||
checkForForbiddenBits(T, input, val.bits)
|
|
||||||
copyMem(addr val.bytes[0], unsafeAddr input[0], input.len)
|
|
||||||
|
|
||||||
elif val is object|tuple:
|
|
||||||
let inputLen = uint32 input.len
|
|
||||||
const minimallyExpectedSize = uint32 fixedPortionSize(T)
|
|
||||||
|
|
||||||
if inputLen < minimallyExpectedSize:
|
|
||||||
raise newException(MalformedSszError, "SSZ input of insufficient size")
|
|
||||||
|
|
||||||
enumInstanceSerializedFields(val, fieldName, field):
|
|
||||||
const boundingOffsets = getFieldBoundingOffsets(T, fieldName)
|
|
||||||
|
|
||||||
# type FieldType = type field # buggy
|
|
||||||
# For some reason, Nim gets confused about the alias here. This could be a
|
|
||||||
# generics caching issue caused by the use of distinct types. Such an
|
|
||||||
# issue is very scary in general.
|
|
||||||
# The bug can be seen with the two List[uint64, N] types that exist in
|
|
||||||
# the spec, with different N.
|
|
||||||
|
|
||||||
type SszType = type toSszType(declval type(field))
|
|
||||||
|
|
||||||
when isFixedSize(SszType):
|
|
||||||
const
|
|
||||||
startOffset = boundingOffsets[0]
|
|
||||||
endOffset = boundingOffsets[1]
|
|
||||||
else:
|
|
||||||
let
|
|
||||||
startOffset = readOffsetUnchecked(boundingOffsets[0])
|
|
||||||
endOffset = if boundingOffsets[1] == -1: inputLen
|
|
||||||
else: readOffsetUnchecked(boundingOffsets[1])
|
|
||||||
|
|
||||||
when boundingOffsets.isFirstOffset:
|
|
||||||
if startOffset != minimallyExpectedSize:
|
|
||||||
raise newException(MalformedSszError, "SSZ object dynamic portion starts at invalid offset")
|
|
||||||
|
|
||||||
if startOffset > endOffset:
|
|
||||||
raise newException(MalformedSszError, "SSZ field offsets are not monotonically increasing")
|
|
||||||
elif endOffset > inputLen:
|
|
||||||
raise newException(MalformedSszError, "SSZ field offset points past the end of the input")
|
|
||||||
elif startOffset < minimallyExpectedSize:
|
|
||||||
raise newException(MalformedSszError, "SSZ field offset points outside bounding offsets")
|
|
||||||
|
|
||||||
# TODO The extra type escaping here is a work-around for a Nim issue:
|
|
||||||
when type(field) is type(SszType):
|
|
||||||
readSszValue(
|
|
||||||
input.toOpenArray(int(startOffset), int(endOffset - 1)),
|
|
||||||
field)
|
|
||||||
else:
|
|
||||||
field = fromSszBytes(
|
|
||||||
type(field),
|
|
||||||
input.toOpenArray(int(startOffset), int(endOffset - 1)))
|
|
||||||
|
|
||||||
else:
|
|
||||||
unsupported T
|
|
||||||
|
|
@ -1,111 +0,0 @@
|
||||||
{.push raises: [Defect].}
|
|
||||||
|
|
||||||
import
|
|
||||||
math, sequtils, ssz_serialization, options, algorithm,
|
|
||||||
nimcrypto/hash,
|
|
||||||
../common/eth_types, ./types, ./merkleization
|
|
||||||
|
|
||||||
const maxTreeDepth: uint64 = 32
|
|
||||||
const empty: seq[Digest] = @[]
|
|
||||||
|
|
||||||
type
|
|
||||||
MerkleNodeType = enum
|
|
||||||
LeafType,
|
|
||||||
NodeType,
|
|
||||||
ZeroType
|
|
||||||
|
|
||||||
MerkleNode = ref object
|
|
||||||
case kind: MerkleNodeType
|
|
||||||
of LeafType:
|
|
||||||
digest: Digest
|
|
||||||
of NodeType:
|
|
||||||
innerDigest: Digest
|
|
||||||
left: MerkleNode
|
|
||||||
right: MerkleNode
|
|
||||||
of ZeroType:
|
|
||||||
depth: uint64
|
|
||||||
|
|
||||||
func zeroNodes(): seq[MerkleNode] =
|
|
||||||
var nodes = newSeq[MerkleNode]()
|
|
||||||
for i in 0..maxTreeDepth:
|
|
||||||
nodes.add(MerkleNode(kind: ZeroType, depth: i))
|
|
||||||
return nodes
|
|
||||||
|
|
||||||
let zNodes = zeroNodes()
|
|
||||||
|
|
||||||
# This look like something that should be in standard lib.
|
|
||||||
func splitAt[T](s: openArray[T], idx: uint64): (seq[T], seq[T]) =
|
|
||||||
var lSeq = newSeq[T]()
|
|
||||||
var rSeq = newSeq[T]()
|
|
||||||
for i, e in s:
|
|
||||||
if (uint64(i) < idx):
|
|
||||||
lSeq.add(e)
|
|
||||||
else:
|
|
||||||
rSeq.add(e)
|
|
||||||
(lSeq, rSeq)
|
|
||||||
|
|
||||||
func splitLeaves(l: openArray[Digest], cap: uint64): (seq[Digest], seq[Digest]) =
|
|
||||||
if (uint64(len(l)) <= cap):
|
|
||||||
(l.toSeq(), empty)
|
|
||||||
else:
|
|
||||||
splitAt(l, cap)
|
|
||||||
|
|
||||||
proc getSubTrees(node: MerkleNode): Option[(MerkleNode, MerkleNode)] =
|
|
||||||
case node.kind
|
|
||||||
of LeafType:
|
|
||||||
return none[(MerkleNode, MerkleNode)]()
|
|
||||||
of NodeType:
|
|
||||||
return some((node.left, node.right))
|
|
||||||
of ZeroType:
|
|
||||||
if node.depth == 0:
|
|
||||||
return none[(MerkleNode, MerkleNode)]()
|
|
||||||
else:
|
|
||||||
return some((zNodes[node.depth - 1], zNodes[node.depth - 1]))
|
|
||||||
|
|
||||||
func hash*(node: MerkleNode): Digest =
|
|
||||||
case node.kind
|
|
||||||
of LeafType:
|
|
||||||
node.digest
|
|
||||||
of NodeType:
|
|
||||||
node.innerDigest
|
|
||||||
of ZeroType:
|
|
||||||
zeroHashes[node.depth]
|
|
||||||
|
|
||||||
func getCapacityAtDepth(depth: uint64): uint64 =
|
|
||||||
uint64 math.pow(2'f64, float64 depth)
|
|
||||||
|
|
||||||
func createTree*(leaves: openArray[Digest], depth: uint64): MerkleNode =
|
|
||||||
if len(leaves) == 0:
|
|
||||||
return MerkleNode(kind: ZeroType, depth: depth)
|
|
||||||
elif depth == 0:
|
|
||||||
return MerkleNode(kind: LeafType, digest: leaves[0])
|
|
||||||
else:
|
|
||||||
let nexLevelDepth = depth - 1
|
|
||||||
let subCap = getCapacityAtDepth(nexLevelDepth)
|
|
||||||
let (left, right) = splitLeaves(leaves, subCap)
|
|
||||||
let leftTree = createTree(left, nexLevelDepth)
|
|
||||||
let rightTree = createTree(right, nexLevelDepth)
|
|
||||||
let finalHash = mergeBranches(leftTree.hash(), rightTree.hash())
|
|
||||||
return MerkleNode(kind: NodeType, innerDigest: finalHash, left: leftTree, right: rightTree)
|
|
||||||
|
|
||||||
proc genProof*(tree: MerkleNode, idx: uint64, treeDepth: uint64): seq[Digest] =
|
|
||||||
var proof = newSeq[Digest]()
|
|
||||||
var currNode = tree
|
|
||||||
var currDepth = treeDepth
|
|
||||||
while currDepth > 0:
|
|
||||||
let ithBit = (idx shr (currDepth - 1)) and 1
|
|
||||||
# should be safe to call unsafeGet() as leaves are on lowest level, and depth is
|
|
||||||
# always larger than 0
|
|
||||||
let (left, right) = getSubTrees(currNode).unsafeGet()
|
|
||||||
if ithBit == 1:
|
|
||||||
proof.add(left.hash())
|
|
||||||
currNode = right
|
|
||||||
else:
|
|
||||||
proof.add(right.hash())
|
|
||||||
currNode = left
|
|
||||||
currDepth = currDepth - 1
|
|
||||||
|
|
||||||
proof.reverse()
|
|
||||||
proof
|
|
||||||
|
|
||||||
# TODO add method to add leaf to the exisiting tree
|
|
||||||
|
|
@ -1,660 +0,0 @@
|
||||||
# ssz_serialization
|
|
||||||
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
|
||||||
# Licensed and distributed under either of
|
|
||||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
|
||||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
|
||||||
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
|
||||||
|
|
||||||
# This module contains the parts necessary to create a merkle hash from the core
|
|
||||||
# SSZ types outlined in the spec:
|
|
||||||
# https://github.com/ethereum/consensus-specs/blob/v1.0.1/ssz/simple-serialize.md#merkleization
|
|
||||||
|
|
||||||
{.push raises: [Defect].}
|
|
||||||
|
|
||||||
import
|
|
||||||
math, sequtils,
|
|
||||||
stew/[bitops2, endians2, ptrops],
|
|
||||||
stew/ranges/ptr_arith, nimcrypto/[hash, sha2],
|
|
||||||
serialization/testing/tracing,
|
|
||||||
"."/[bitseqs, types]
|
|
||||||
|
|
||||||
export
|
|
||||||
types
|
|
||||||
|
|
||||||
when hasSerializationTracing:
|
|
||||||
import stew/byteutils, typetraits
|
|
||||||
|
|
||||||
const
|
|
||||||
zero64 = default array[64, byte]
|
|
||||||
bitsPerChunk = bytesPerChunk * 8
|
|
||||||
|
|
||||||
func binaryTreeHeight*(totalElements: Limit): int =
|
|
||||||
bitWidth nextPow2(uint64 totalElements)
|
|
||||||
|
|
||||||
type
|
|
||||||
SszMerkleizerImpl = object
|
|
||||||
combinedChunks: ptr UncheckedArray[Digest]
|
|
||||||
totalChunks: uint64
|
|
||||||
topIndex: int
|
|
||||||
|
|
||||||
SszMerkleizer*[limit: static[Limit]] = object
|
|
||||||
combinedChunks: ref array[binaryTreeHeight limit, Digest]
|
|
||||||
impl: SszMerkleizerImpl
|
|
||||||
|
|
||||||
template chunks*(m: SszMerkleizerImpl): openArray[Digest] =
|
|
||||||
m.combinedChunks.toOpenArray(0, m.topIndex)
|
|
||||||
|
|
||||||
template getChunkCount*(m: SszMerkleizer): uint64 =
|
|
||||||
m.impl.totalChunks
|
|
||||||
|
|
||||||
template getCombinedChunks*(m: SszMerkleizer): openArray[Digest] =
|
|
||||||
toOpenArray(m.impl.combinedChunks, 0, m.impl.topIndex)
|
|
||||||
|
|
||||||
type DigestCtx* = sha2.sha256
|
|
||||||
|
|
||||||
template computeDigest*(body: untyped): Digest =
|
|
||||||
## This little helper will init the hash function and return the sliced
|
|
||||||
## hash:
|
|
||||||
## let hashOfData = withHash: h.update(data)
|
|
||||||
when nimvm:
|
|
||||||
# In SSZ, computeZeroHashes require compile-time SHA256
|
|
||||||
block:
|
|
||||||
var h {.inject.}: sha256
|
|
||||||
init(h)
|
|
||||||
body
|
|
||||||
finish(h)
|
|
||||||
else:
|
|
||||||
block:
|
|
||||||
var h {.inject, noInit.}: DigestCtx
|
|
||||||
init(h)
|
|
||||||
body
|
|
||||||
finish(h)
|
|
||||||
|
|
||||||
func digest(a: openArray[byte]): Digest =
|
|
||||||
result = computeDigest:
|
|
||||||
h.update(a)
|
|
||||||
|
|
||||||
func digest(a, b: openArray[byte]): Digest =
|
|
||||||
result = computeDigest:
|
|
||||||
trs "DIGESTING ARRAYS ", toHex(a), " ", toHex(b)
|
|
||||||
trs toHex(a)
|
|
||||||
trs toHex(b)
|
|
||||||
|
|
||||||
h.update a
|
|
||||||
h.update b
|
|
||||||
trs "HASH RESULT ", result
|
|
||||||
|
|
||||||
func digest(a, b, c: openArray[byte]): Digest =
|
|
||||||
result = computeDigest:
|
|
||||||
trs "DIGESTING ARRAYS ", toHex(a), " ", toHex(b), " ", toHex(c)
|
|
||||||
|
|
||||||
h.update a
|
|
||||||
h.update b
|
|
||||||
h.update c
|
|
||||||
trs "HASH RESULT ", result
|
|
||||||
|
|
||||||
func mergeBranches(existing: Digest, newData: openArray[byte]): Digest =
|
|
||||||
trs "MERGING BRANCHES OPEN ARRAY"
|
|
||||||
|
|
||||||
let paddingBytes = bytesPerChunk - newData.len
|
|
||||||
digest(existing.data, newData, zero64.toOpenArray(0, paddingBytes - 1))
|
|
||||||
|
|
||||||
template mergeBranches(existing: Digest, newData: array[32, byte]): Digest =
|
|
||||||
trs "MERGING BRANCHES ARRAY"
|
|
||||||
digest(existing.data, newData)
|
|
||||||
|
|
||||||
template mergeBranches*(a, b: Digest): Digest =
|
|
||||||
trs "MERGING BRANCHES DIGEST"
|
|
||||||
digest(a.data, b.data)
|
|
||||||
|
|
||||||
func computeZeroHashes: array[sizeof(Limit) * 8, Digest] =
|
|
||||||
result[0] = Digest()
|
|
||||||
for i in 1 .. result.high:
|
|
||||||
result[i] = mergeBranches(result[i - 1], result[i - 1])
|
|
||||||
|
|
||||||
const zeroHashes* = computeZeroHashes()
|
|
||||||
|
|
||||||
func addChunk*(merkleizer: var SszMerkleizerImpl, data: openArray[byte]) =
|
|
||||||
doAssert data.len > 0 and data.len <= bytesPerChunk
|
|
||||||
|
|
||||||
if getBitLE(merkleizer.totalChunks, 0):
|
|
||||||
var hash = mergeBranches(merkleizer.combinedChunks[0], data)
|
|
||||||
|
|
||||||
for i in 1 .. merkleizer.topIndex:
|
|
||||||
trs "ITERATING"
|
|
||||||
if getBitLE(merkleizer.totalChunks, i):
|
|
||||||
trs "CALLING MERGE BRANCHES"
|
|
||||||
hash = mergeBranches(merkleizer.combinedChunks[i], hash)
|
|
||||||
else:
|
|
||||||
trs "WRITING FRESH CHUNK AT ", i, " = ", hash
|
|
||||||
merkleizer.combinedChunks[i] = hash
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
let paddingBytes = bytesPerChunk - data.len
|
|
||||||
|
|
||||||
merkleizer.combinedChunks[0].data[0..<data.len] = data
|
|
||||||
merkleizer.combinedChunks[0].data[data.len..<bytesPerChunk] =
|
|
||||||
zero64.toOpenArray(0, paddingBytes - 1)
|
|
||||||
|
|
||||||
trs "WROTE BASE CHUNK ",
|
|
||||||
toHex(merkleizer.combinedChunks[0].data), " ", data.len
|
|
||||||
|
|
||||||
inc merkleizer.totalChunks
|
|
||||||
|
|
||||||
template isOdd(x: SomeNumber): bool =
|
|
||||||
(x and 1) != 0
|
|
||||||
|
|
||||||
func addChunkAndGenMerkleProof*(merkleizer: var SszMerkleizerImpl,
|
|
||||||
hash: Digest,
|
|
||||||
outProof: var openArray[Digest]) =
|
|
||||||
var
|
|
||||||
hashWrittenToMerkleizer = false
|
|
||||||
hash = hash
|
|
||||||
|
|
||||||
doAssert merkleizer.topIndex < outProof.len
|
|
||||||
|
|
||||||
for level in 0 .. merkleizer.topIndex:
|
|
||||||
if getBitLE(merkleizer.totalChunks, level):
|
|
||||||
outProof[level] = merkleizer.combinedChunks[level]
|
|
||||||
hash = mergeBranches(merkleizer.combinedChunks[level], hash)
|
|
||||||
else:
|
|
||||||
if not hashWrittenToMerkleizer:
|
|
||||||
merkleizer.combinedChunks[level] = hash
|
|
||||||
hashWrittenToMerkleizer = true
|
|
||||||
outProof[level] = zeroHashes[level]
|
|
||||||
hash = mergeBranches(hash, zeroHashes[level])
|
|
||||||
|
|
||||||
merkleizer.totalChunks += 1
|
|
||||||
|
|
||||||
func completeStartedChunk(merkleizer: var SszMerkleizerImpl,
|
|
||||||
hash: Digest, atLevel: int) =
|
|
||||||
when false:
|
|
||||||
let
|
|
||||||
insertedChunksCount = 1'u64 shl (atLevel - 1)
|
|
||||||
chunksStateMask = (insertedChunksCount shl 1) - 1
|
|
||||||
doAssert (merkleizer.totalChunks and chunksStateMask) == insertedChunksCount
|
|
||||||
|
|
||||||
var hash = hash
|
|
||||||
for i in atLevel .. merkleizer.topIndex:
|
|
||||||
if getBitLE(merkleizer.totalChunks, i):
|
|
||||||
hash = mergeBranches(merkleizer.combinedChunks[i], hash)
|
|
||||||
else:
|
|
||||||
merkleizer.combinedChunks[i] = hash
|
|
||||||
break
|
|
||||||
|
|
||||||
func addChunksAndGenMerkleProofs*(merkleizer: var SszMerkleizerImpl,
|
|
||||||
chunks: openArray[Digest]): seq[Digest] =
|
|
||||||
doAssert chunks.len > 0 and merkleizer.topIndex > 0
|
|
||||||
|
|
||||||
let proofHeight = merkleizer.topIndex + 1
|
|
||||||
result = newSeq[Digest](chunks.len * proofHeight)
|
|
||||||
|
|
||||||
if chunks.len == 1:
|
|
||||||
merkleizer.addChunkAndGenMerkleProof(chunks[0], result)
|
|
||||||
return
|
|
||||||
|
|
||||||
let newTotalChunks = merkleizer.totalChunks + chunks.len.uint64
|
|
||||||
|
|
||||||
var
|
|
||||||
# A perfect binary tree will take either `chunks.len * 2` values if the
|
|
||||||
# number of elements in the base layer is odd and `chunks.len * 2 - 1`
|
|
||||||
# otherwise. Each row may also need a single extra element at most if
|
|
||||||
# it must be combined with the existing values in the Merkleizer:
|
|
||||||
merkleTree = newSeqOfCap[Digest](chunks.len + merkleizer.topIndex)
|
|
||||||
inRowIdx = merkleizer.totalChunks
|
|
||||||
postUpdateInRowIdx = newTotalChunks
|
|
||||||
zeroMixed = false
|
|
||||||
|
|
||||||
template writeResult(chunkIdx, level: int, chunk: Digest) =
|
|
||||||
result[chunkIdx * proofHeight + level] = chunk
|
|
||||||
|
|
||||||
# We'll start by generating the first row of the merkle tree.
|
|
||||||
var currPairEnd = if inRowIdx.isOdd:
|
|
||||||
# an odd chunk number means that we must combine the
|
|
||||||
# hash with the existing pending sibling hash in the
|
|
||||||
# merkleizer.
|
|
||||||
writeResult(0, 0, merkleizer.combinedChunks[0])
|
|
||||||
merkleTree.add mergeBranches(merkleizer.combinedChunks[0], chunks[0])
|
|
||||||
|
|
||||||
# TODO: can we immediately write this out?
|
|
||||||
merkleizer.completeStartedChunk(merkleTree[^1], 1)
|
|
||||||
2
|
|
||||||
else:
|
|
||||||
1
|
|
||||||
|
|
||||||
if postUpdateInRowIdx.isOdd:
|
|
||||||
merkleizer.combinedChunks[0] = chunks[^1]
|
|
||||||
|
|
||||||
while currPairEnd < chunks.len:
|
|
||||||
writeResult(currPairEnd - 1, 0, chunks[currPairEnd])
|
|
||||||
writeResult(currPairEnd, 0, chunks[currPairEnd - 1])
|
|
||||||
merkleTree.add mergeBranches(chunks[currPairEnd - 1],
|
|
||||||
chunks[currPairEnd])
|
|
||||||
currPairEnd += 2
|
|
||||||
|
|
||||||
if currPairEnd - 1 < chunks.len:
|
|
||||||
zeroMixed = true
|
|
||||||
writeResult(currPairEnd - 1, 0, zeroHashes[0])
|
|
||||||
merkleTree.add mergeBranches(chunks[currPairEnd - 1],
|
|
||||||
zeroHashes[0])
|
|
||||||
var
|
|
||||||
level = 0
|
|
||||||
baseChunksPerElement = 1
|
|
||||||
treeRowStart = 0
|
|
||||||
rowLen = merkleTree.len
|
|
||||||
|
|
||||||
template writeProofs(rowChunkIdx: int, hash: Digest) =
|
|
||||||
let
|
|
||||||
startAbsIdx = (inRowIdx.int + rowChunkIdx) * baseChunksPerElement
|
|
||||||
endAbsIdx = startAbsIdx + baseChunksPerElement
|
|
||||||
startResIdx = max(startAbsIdx - merkleizer.totalChunks.int, 0)
|
|
||||||
endResIdx = min(endAbsIdx - merkleizer.totalChunks.int, chunks.len)
|
|
||||||
|
|
||||||
for resultPos in startResIdx ..< endResIdx:
|
|
||||||
writeResult(resultPos, level, hash)
|
|
||||||
|
|
||||||
if rowLen > 1:
|
|
||||||
while level < merkleizer.topIndex:
|
|
||||||
inc level
|
|
||||||
baseChunksPerElement *= 2
|
|
||||||
inRowIdx = inRowIdx div 2
|
|
||||||
postUpdateInRowIdx = postUpdateInRowIdx div 2
|
|
||||||
|
|
||||||
var currPairEnd = if inRowIdx.isOdd:
|
|
||||||
# an odd chunk number means that we must combine the
|
|
||||||
# hash with the existing pending sibling hash in the
|
|
||||||
# merkleizer.
|
|
||||||
writeProofs(0, merkleizer.combinedChunks[level])
|
|
||||||
merkleTree.add mergeBranches(merkleizer.combinedChunks[level],
|
|
||||||
merkleTree[treeRowStart])
|
|
||||||
|
|
||||||
# TODO: can we immediately write this out?
|
|
||||||
merkleizer.completeStartedChunk(merkleTree[^1], level + 1)
|
|
||||||
2
|
|
||||||
else:
|
|
||||||
1
|
|
||||||
|
|
||||||
if postUpdateInRowIdx.isOdd:
|
|
||||||
merkleizer.combinedChunks[level] = merkleTree[treeRowStart + rowLen -
|
|
||||||
ord(zeroMixed) - 1]
|
|
||||||
while currPairEnd < rowLen:
|
|
||||||
writeProofs(currPairEnd - 1, merkleTree[treeRowStart + currPairEnd])
|
|
||||||
writeProofs(currPairEnd, merkleTree[treeRowStart + currPairEnd - 1])
|
|
||||||
merkleTree.add mergeBranches(merkleTree[treeRowStart + currPairEnd - 1],
|
|
||||||
merkleTree[treeRowStart + currPairEnd])
|
|
||||||
currPairEnd += 2
|
|
||||||
|
|
||||||
if currPairEnd - 1 < rowLen:
|
|
||||||
zeroMixed = true
|
|
||||||
writeProofs(currPairEnd - 1, zeroHashes[level])
|
|
||||||
merkleTree.add mergeBranches(merkleTree[treeRowStart + currPairEnd - 1],
|
|
||||||
zeroHashes[level])
|
|
||||||
|
|
||||||
treeRowStart += rowLen
|
|
||||||
rowLen = merkleTree.len - treeRowStart
|
|
||||||
|
|
||||||
if rowLen == 1:
|
|
||||||
break
|
|
||||||
|
|
||||||
doAssert rowLen == 1
|
|
||||||
|
|
||||||
if (inRowIdx and 2) != 0:
|
|
||||||
merkleizer.completeStartedChunk(
|
|
||||||
mergeBranches(merkleizer.combinedChunks[level + 1], merkleTree[^1]),
|
|
||||||
level + 2)
|
|
||||||
|
|
||||||
if (not zeroMixed) and (postUpdateInRowIdx and 2) != 0:
|
|
||||||
merkleizer.combinedChunks[level + 1] = merkleTree[^1]
|
|
||||||
|
|
||||||
while level < merkleizer.topIndex:
|
|
||||||
inc level
|
|
||||||
baseChunksPerElement *= 2
|
|
||||||
inRowIdx = inRowIdx div 2
|
|
||||||
|
|
||||||
let hash = if getBitLE(merkleizer.totalChunks, level):
|
|
||||||
merkleizer.combinedChunks[level]
|
|
||||||
else:
|
|
||||||
zeroHashes[level]
|
|
||||||
|
|
||||||
writeProofs(0, hash)
|
|
||||||
|
|
||||||
merkleizer.totalChunks = newTotalChunks
|
|
||||||
|
|
||||||
proc init*(S: type SszMerkleizer): S =
|
|
||||||
new result.combinedChunks
|
|
||||||
result.impl = SszMerkleizerImpl(
|
|
||||||
combinedChunks: cast[ptr UncheckedArray[Digest]](
|
|
||||||
addr result.combinedChunks[][0]),
|
|
||||||
topIndex: binaryTreeHeight(result.limit) - 1,
|
|
||||||
totalChunks: 0)
|
|
||||||
|
|
||||||
proc init*(S: type SszMerkleizer,
|
|
||||||
combinedChunks: openArray[Digest],
|
|
||||||
totalChunks: uint64): S =
|
|
||||||
new result.combinedChunks
|
|
||||||
result.combinedChunks[][0 ..< combinedChunks.len] = combinedChunks
|
|
||||||
result.impl = SszMerkleizerImpl(
|
|
||||||
combinedChunks: cast[ptr UncheckedArray[Digest]](
|
|
||||||
addr result.combinedChunks[][0]),
|
|
||||||
topIndex: binaryTreeHeight(result.limit) - 1,
|
|
||||||
totalChunks: totalChunks)
|
|
||||||
|
|
||||||
proc copy*[L: static[Limit]](cloned: SszMerkleizer[L]): SszMerkleizer[L] =
|
|
||||||
new result.combinedChunks
|
|
||||||
result.combinedChunks[] = cloned.combinedChunks[]
|
|
||||||
result.impl = SszMerkleizerImpl(
|
|
||||||
combinedChunks: cast[ptr UncheckedArray[Digest]](
|
|
||||||
addr result.combinedChunks[][0]),
|
|
||||||
topIndex: binaryTreeHeight(L) - 1,
|
|
||||||
totalChunks: cloned.totalChunks)
|
|
||||||
|
|
||||||
template addChunksAndGenMerkleProofs*(
|
|
||||||
merkleizer: var SszMerkleizer,
|
|
||||||
chunks: openArray[Digest]): seq[Digest] =
|
|
||||||
addChunksAndGenMerkleProofs(merkleizer.impl, chunks)
|
|
||||||
|
|
||||||
template addChunk*(merkleizer: var SszMerkleizer, data: openArray[byte]) =
|
|
||||||
addChunk(merkleizer.impl, data)
|
|
||||||
|
|
||||||
template totalChunks*(merkleizer: SszMerkleizer): uint64 =
|
|
||||||
merkleizer.impl.totalChunks
|
|
||||||
|
|
||||||
template getFinalHash*(merkleizer: SszMerkleizer): Digest =
|
|
||||||
merkleizer.impl.getFinalHash
|
|
||||||
|
|
||||||
template createMerkleizer*(totalElements: static Limit): SszMerkleizerImpl =
|
|
||||||
trs "CREATING A MERKLEIZER FOR ", totalElements
|
|
||||||
|
|
||||||
const treeHeight = binaryTreeHeight totalElements
|
|
||||||
var combinedChunks {.noInit.}: array[treeHeight, Digest]
|
|
||||||
|
|
||||||
let topIndex = treeHeight - 1
|
|
||||||
|
|
||||||
SszMerkleizerImpl(
|
|
||||||
combinedChunks: cast[ptr UncheckedArray[Digest]](addr combinedChunks),
|
|
||||||
topIndex: if (topIndex < 0): 0 else: topIndex,
|
|
||||||
totalChunks: 0)
|
|
||||||
|
|
||||||
func getFinalHash*(merkleizer: SszMerkleizerImpl): Digest =
|
|
||||||
if merkleizer.totalChunks == 0:
|
|
||||||
return zeroHashes[merkleizer.topIndex]
|
|
||||||
|
|
||||||
let
|
|
||||||
bottomHashIdx = firstOne(merkleizer.totalChunks) - 1
|
|
||||||
submittedChunksHeight = bitWidth(merkleizer.totalChunks - 1)
|
|
||||||
topHashIdx = merkleizer.topIndex
|
|
||||||
|
|
||||||
trs "BOTTOM HASH ", bottomHashIdx
|
|
||||||
trs "SUBMITTED HEIGHT ", submittedChunksHeight
|
|
||||||
trs "TOP HASH IDX ", topHashIdx
|
|
||||||
|
|
||||||
if bottomHashIdx != submittedChunksHeight:
|
|
||||||
# Our tree is not finished. We must complete the work in progress
|
|
||||||
# branches and then extend the tree to the right height.
|
|
||||||
result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx],
|
|
||||||
zeroHashes[bottomHashIdx])
|
|
||||||
|
|
||||||
for i in bottomHashIdx + 1 ..< topHashIdx:
|
|
||||||
if getBitLE(merkleizer.totalChunks, i):
|
|
||||||
result = mergeBranches(merkleizer.combinedChunks[i], result)
|
|
||||||
trs "COMBINED"
|
|
||||||
else:
|
|
||||||
result = mergeBranches(result, zeroHashes[i])
|
|
||||||
trs "COMBINED WITH ZERO"
|
|
||||||
|
|
||||||
elif bottomHashIdx == topHashIdx:
|
|
||||||
# We have a perfect tree (chunks == 2**n) at just the right height!
|
|
||||||
result = merkleizer.combinedChunks[bottomHashIdx]
|
|
||||||
else:
|
|
||||||
# We have a perfect tree of user chunks, but we have more work to
|
|
||||||
# do - we must extend it to reach the desired height
|
|
||||||
result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx],
|
|
||||||
zeroHashes[bottomHashIdx])
|
|
||||||
|
|
||||||
for i in bottomHashIdx + 1 ..< topHashIdx:
|
|
||||||
result = mergeBranches(result, zeroHashes[i])
|
|
||||||
|
|
||||||
func mixInLength*(root: Digest, length: int): Digest =
|
|
||||||
var dataLen: array[32, byte]
|
|
||||||
dataLen[0..<8] = uint64(length).toBytesLE()
|
|
||||||
mergeBranches(root, dataLen)
|
|
||||||
|
|
||||||
func hash_tree_root*(x: auto): Digest {.gcsafe, raises: [Defect].}
|
|
||||||
|
|
||||||
template merkleizeFields(totalElements: static Limit, body: untyped): Digest =
|
|
||||||
var merkleizer {.inject.} = createMerkleizer(totalElements)
|
|
||||||
|
|
||||||
template addField(field) =
|
|
||||||
let hash = hash_tree_root(field)
|
|
||||||
trs "MERKLEIZING FIELD ", astToStr(field), " = ", hash
|
|
||||||
addChunk(merkleizer, hash.data)
|
|
||||||
trs "CHUNK ADDED"
|
|
||||||
|
|
||||||
body
|
|
||||||
|
|
||||||
getFinalHash(merkleizer)
|
|
||||||
|
|
||||||
template writeBytesLE(chunk: var array[bytesPerChunk, byte], atParam: int,
|
|
||||||
val: SomeUnsignedInt) =
|
|
||||||
let at = atParam
|
|
||||||
chunk[at ..< at + sizeof(val)] = toBytesLE(val)
|
|
||||||
|
|
||||||
func chunkedHashTreeRootForBasicTypes[T](merkleizer: var SszMerkleizerImpl,
|
|
||||||
arr: openArray[T]): Digest =
|
|
||||||
static:
|
|
||||||
doAssert T is BasicType
|
|
||||||
doAssert bytesPerChunk mod sizeof(T) == 0
|
|
||||||
|
|
||||||
if arr.len == 0:
|
|
||||||
return getFinalHash(merkleizer)
|
|
||||||
|
|
||||||
when sizeof(T) == 1 or cpuEndian == littleEndian:
|
|
||||||
var
|
|
||||||
remainingBytes = when sizeof(T) == 1: arr.len
|
|
||||||
else: arr.len * sizeof(T)
|
|
||||||
pos = cast[ptr byte](unsafeAddr arr[0])
|
|
||||||
|
|
||||||
while remainingBytes >= bytesPerChunk:
|
|
||||||
merkleizer.addChunk(makeOpenArray(pos, bytesPerChunk))
|
|
||||||
pos = offset(pos, bytesPerChunk)
|
|
||||||
remainingBytes -= bytesPerChunk
|
|
||||||
|
|
||||||
if remainingBytes > 0:
|
|
||||||
merkleizer.addChunk(makeOpenArray(pos, remainingBytes))
|
|
||||||
|
|
||||||
else:
|
|
||||||
const valuesPerChunk = bytesPerChunk div sizeof(T)
|
|
||||||
|
|
||||||
var writtenValues = 0
|
|
||||||
|
|
||||||
var chunk: array[bytesPerChunk, byte]
|
|
||||||
while writtenValues < arr.len - valuesPerChunk:
|
|
||||||
for i in 0 ..< valuesPerChunk:
|
|
||||||
chunk.writeBytesLE(i * sizeof(T), arr[writtenValues + i])
|
|
||||||
merkleizer.addChunk chunk
|
|
||||||
inc writtenValues, valuesPerChunk
|
|
||||||
|
|
||||||
let remainingValues = arr.len - writtenValues
|
|
||||||
if remainingValues > 0:
|
|
||||||
var lastChunk: array[bytesPerChunk, byte]
|
|
||||||
for i in 0 ..< remainingValues:
|
|
||||||
lastChunk.writeBytesLE(i * sizeof(T), arr[writtenValues + i])
|
|
||||||
merkleizer.addChunk lastChunk
|
|
||||||
|
|
||||||
getFinalHash(merkleizer)
|
|
||||||
|
|
||||||
func bitListHashTreeRoot(merkleizer: var SszMerkleizerImpl, x: BitSeq): Digest =
|
|
||||||
# TODO: Switch to a simpler BitList representation and
|
|
||||||
# replace this with `chunkedHashTreeRoot`
|
|
||||||
var
|
|
||||||
totalBytes = bytes(x).len
|
|
||||||
lastCorrectedByte = bytes(x)[^1]
|
|
||||||
|
|
||||||
if lastCorrectedByte == byte(1):
|
|
||||||
if totalBytes == 1:
|
|
||||||
# This is an empty bit list.
|
|
||||||
# It should be hashed as a tree containing all zeros:
|
|
||||||
return mergeBranches(zeroHashes[merkleizer.topIndex],
|
|
||||||
zeroHashes[0]) # this is the mixed length
|
|
||||||
|
|
||||||
totalBytes -= 1
|
|
||||||
lastCorrectedByte = bytes(x)[^2]
|
|
||||||
else:
|
|
||||||
let markerPos = log2trunc(lastCorrectedByte)
|
|
||||||
lastCorrectedByte.clearBit(markerPos)
|
|
||||||
|
|
||||||
var
|
|
||||||
bytesInLastChunk = totalBytes mod bytesPerChunk
|
|
||||||
fullChunks = totalBytes div bytesPerChunk
|
|
||||||
|
|
||||||
if bytesInLastChunk == 0:
|
|
||||||
fullChunks -= 1
|
|
||||||
bytesInLastChunk = 32
|
|
||||||
|
|
||||||
for i in 0 ..< fullChunks:
|
|
||||||
let
|
|
||||||
chunkStartPos = i * bytesPerChunk
|
|
||||||
chunkEndPos = chunkStartPos + bytesPerChunk - 1
|
|
||||||
|
|
||||||
merkleizer.addChunk bytes(x).toOpenArray(chunkStartPos, chunkEndPos)
|
|
||||||
|
|
||||||
var
|
|
||||||
lastChunk: array[bytesPerChunk, byte]
|
|
||||||
chunkStartPos = fullChunks * bytesPerChunk
|
|
||||||
|
|
||||||
for i in 0 .. bytesInLastChunk - 2:
|
|
||||||
lastChunk[i] = bytes(x)[chunkStartPos + i]
|
|
||||||
|
|
||||||
lastChunk[bytesInLastChunk - 1] = lastCorrectedByte
|
|
||||||
|
|
||||||
merkleizer.addChunk lastChunk.toOpenArray(0, bytesInLastChunk - 1)
|
|
||||||
let contentsHash = merkleizer.getFinalHash
|
|
||||||
mixInLength contentsHash, x.len
|
|
||||||
|
|
||||||
func maxChunksCount(T: type, maxLen: Limit): Limit =
|
|
||||||
when T is BitList|BitArray:
|
|
||||||
(maxLen + bitsPerChunk - 1) div bitsPerChunk
|
|
||||||
elif T is array|List:
|
|
||||||
maxChunkIdx(ElemType(T), maxLen)
|
|
||||||
else:
|
|
||||||
unsupported T # This should never happen
|
|
||||||
|
|
||||||
func hashTreeRootAux[T](x: T): Digest =
|
|
||||||
when T is bool|char:
|
|
||||||
result.data[0] = byte(x)
|
|
||||||
elif T is SomeUnsignedInt:
|
|
||||||
when cpuEndian == bigEndian:
|
|
||||||
result.data[0..<sizeof(x)] = toBytesLE(x)
|
|
||||||
else:
|
|
||||||
copyMem(addr result.data[0], unsafeAddr x, sizeof x)
|
|
||||||
elif (when T is array: ElemType(T) is BasicType else: false):
|
|
||||||
type E = ElemType(T)
|
|
||||||
when sizeof(T) <= sizeof(result.data):
|
|
||||||
when E is byte|bool or cpuEndian == littleEndian:
|
|
||||||
copyMem(addr result.data[0], unsafeAddr x, sizeof x)
|
|
||||||
else:
|
|
||||||
var pos = 0
|
|
||||||
for e in x:
|
|
||||||
writeBytesLE(result.data, pos, e)
|
|
||||||
pos += sizeof(E)
|
|
||||||
else:
|
|
||||||
trs "FIXED TYPE; USE CHUNK STREAM"
|
|
||||||
var merkleizer = createMerkleizer(maxChunksCount(T, Limit x.len))
|
|
||||||
chunkedHashTreeRootForBasicTypes(merkleizer, x)
|
|
||||||
elif T is BitArray:
|
|
||||||
hashTreeRootAux(x.bytes)
|
|
||||||
elif T is array|object|tuple:
|
|
||||||
trs "MERKLEIZING FIELDS"
|
|
||||||
const totalFields = when T is array: len(x)
|
|
||||||
else: totalSerializedFields(T)
|
|
||||||
merkleizeFields(Limit totalFields):
|
|
||||||
x.enumerateSubFields(f):
|
|
||||||
addField f
|
|
||||||
#elif isCaseObject(T):
|
|
||||||
# # TODO implement this
|
|
||||||
else:
|
|
||||||
unsupported T
|
|
||||||
|
|
||||||
func hashTreeRootList(x: List|BitList): Digest =
|
|
||||||
const maxLen = static(x.maxLen)
|
|
||||||
type T = type(x)
|
|
||||||
const limit = maxChunksCount(T, maxLen)
|
|
||||||
var merkleizer = createMerkleizer(limit)
|
|
||||||
|
|
||||||
when x is BitList:
|
|
||||||
merkleizer.bitListHashTreeRoot(BitSeq x)
|
|
||||||
else:
|
|
||||||
type E = ElemType(T)
|
|
||||||
let contentsHash = when E is BasicType:
|
|
||||||
chunkedHashTreeRootForBasicTypes(merkleizer, asSeq x)
|
|
||||||
else:
|
|
||||||
for elem in x:
|
|
||||||
let elemHash = hash_tree_root(elem)
|
|
||||||
merkleizer.addChunk(elemHash.data)
|
|
||||||
merkleizer.getFinalHash()
|
|
||||||
mixInLength(contentsHash, x.len)
|
|
||||||
|
|
||||||
func hash_tree_root*(x: auto): Digest {.raises: [Defect].} =
|
|
||||||
trs "STARTING HASH TREE ROOT FOR TYPE ", name(type(x))
|
|
||||||
mixin toSszType
|
|
||||||
|
|
||||||
result =
|
|
||||||
when x is List|BitList:
|
|
||||||
hashTreeRootList(x)
|
|
||||||
else:
|
|
||||||
hashTreeRootAux toSszType(x)
|
|
||||||
|
|
||||||
trs "HASH TREE ROOT FOR ", name(type x), " = ", "0x", $result
|
|
||||||
|
|
||||||
# https://github.com/ethereum/consensus-specs/blob/dev/ssz/merkle-proofs.md#get_generalized_index_length
|
|
||||||
func getGeneralizedIndexLength(x: uint64): int =
|
|
||||||
log2trunc(x)
|
|
||||||
|
|
||||||
# https://github.com/ethereum/consensus-specs/blob/dev/ssz/merkle-proofs.md#get_generalized_index_bit
|
|
||||||
func getGeneralizedIndexBit(index: uint64, position: uint64): bool =
|
|
||||||
(index and (1'u64 shl position)) > 0
|
|
||||||
|
|
||||||
# validates merkle proof. Provided index should be a generalized index of leaf node
|
|
||||||
# as defined in: https://github.com/ethereum/consensus-specs/blob/dev/ssz/merkle-proofs.md#generalized-merkle-tree-index
|
|
||||||
func isValidProof*(leaf: Digest, proof: openArray[Digest],
|
|
||||||
index: uint64, root: Digest): bool =
|
|
||||||
if len(proof) == getGeneralizedIndexLength(index):
|
|
||||||
var
|
|
||||||
value = leaf
|
|
||||||
|
|
||||||
for i, digest in proof:
|
|
||||||
value =
|
|
||||||
if getGeneralizedIndexBit(index, uint64 i):
|
|
||||||
mergeBranches(digest, value)
|
|
||||||
else:
|
|
||||||
mergeBranches(value, digest)
|
|
||||||
|
|
||||||
value == root
|
|
||||||
else:
|
|
||||||
false
|
|
||||||
|
|
||||||
proc slice[T](x: openArray[T]): seq[T] = x.toSeq()
|
|
||||||
|
|
||||||
# Helper functions to get proof for any element of a list
|
|
||||||
proc getProofForAllListElements*(list: List): seq[Digest] =
|
|
||||||
type T = type(list)
|
|
||||||
type E = ElemType(T)
|
|
||||||
# basic types have different chunking rules
|
|
||||||
static:
|
|
||||||
doAssert (E is not BasicType)
|
|
||||||
var digests: seq[Digest] = @[]
|
|
||||||
for e in list:
|
|
||||||
let root = hash_tree_root(e)
|
|
||||||
digests.add(root)
|
|
||||||
var merk = createMerkleizer(list.maxLen)
|
|
||||||
merk.addChunksAndGenMerkleProofs(digests)
|
|
||||||
|
|
||||||
proc getProofWithIdx*(list: List, allProofs: seq[Digest], idx: int): seq[Digest] =
|
|
||||||
let treeHeight = binaryTreeHeight(list.maxLen)
|
|
||||||
let startPos = idx * treeHeight
|
|
||||||
let endPos = startPos + treeHeight - 2
|
|
||||||
slice(allProofs.toOpenArray(startPos, endPos))
|
|
||||||
|
|
||||||
proc generateAndGetProofWithIdx*(list: List, idx: int): seq[Digest] =
|
|
||||||
let allProofs = getProofForAllListElements(list)
|
|
||||||
getProofWithIdx(list, allProofs, idx)
|
|
||||||
|
|
@ -1,247 +0,0 @@
|
||||||
# nim-eth - Limited SSZ implementation
|
|
||||||
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
|
||||||
# Licensed and distributed under either of
|
|
||||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
|
||||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
|
||||||
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
|
||||||
|
|
||||||
{.push raises: [Defect].}
|
|
||||||
|
|
||||||
## SSZ serialization for core SSZ types, as specified in:
|
|
||||||
# https://github.com/ethereum/consensus-specs/blob/v1.0.1/ssz/simple-serialize.md#serialization
|
|
||||||
|
|
||||||
import
|
|
||||||
std/[typetraits, options],
|
|
||||||
stew/[endians2, leb128, objects],
|
|
||||||
serialization, serialization/testing/tracing,
|
|
||||||
./bytes_reader, ./types
|
|
||||||
|
|
||||||
export
|
|
||||||
serialization, types, bytes_reader
|
|
||||||
|
|
||||||
type
|
|
||||||
SszReader* = object
|
|
||||||
stream: InputStream
|
|
||||||
|
|
||||||
SszWriter* = object
|
|
||||||
stream: OutputStream
|
|
||||||
|
|
||||||
SizePrefixed*[T] = distinct T
|
|
||||||
SszMaxSizeExceeded* = object of SerializationError
|
|
||||||
|
|
||||||
VarSizedWriterCtx = object
|
|
||||||
fixedParts: WriteCursor
|
|
||||||
offset: int
|
|
||||||
|
|
||||||
FixedSizedWriterCtx = object
|
|
||||||
|
|
||||||
serializationFormat SSZ
|
|
||||||
|
|
||||||
SSZ.setReader SszReader
|
|
||||||
SSZ.setWriter SszWriter, PreferredOutput = seq[byte]
|
|
||||||
|
|
||||||
template sizePrefixed*[TT](x: TT): untyped =
|
|
||||||
type T = TT
|
|
||||||
SizePrefixed[T](x)
|
|
||||||
|
|
||||||
proc init*(T: type SszReader, stream: InputStream): T {.raises: [Defect].} =
|
|
||||||
T(stream: stream)
|
|
||||||
|
|
||||||
proc writeFixedSized(s: var (OutputStream|WriteCursor), x: auto)
|
|
||||||
{.raises: [Defect, IOError].} =
|
|
||||||
mixin toSszType
|
|
||||||
|
|
||||||
when x is byte:
|
|
||||||
s.write x
|
|
||||||
elif x is bool:
|
|
||||||
s.write byte(ord(x))
|
|
||||||
elif x is UintN:
|
|
||||||
when cpuEndian == bigEndian:
|
|
||||||
s.write toBytesLE(x)
|
|
||||||
else:
|
|
||||||
s.writeMemCopy x
|
|
||||||
elif x is array:
|
|
||||||
when x[0] is byte:
|
|
||||||
trs "APPENDING FIXED SIZE BYTES", x
|
|
||||||
s.write x
|
|
||||||
else:
|
|
||||||
for elem in x:
|
|
||||||
trs "WRITING FIXED SIZE ARRAY ELEMENT"
|
|
||||||
s.writeFixedSized toSszType(elem)
|
|
||||||
elif x is tuple|object:
|
|
||||||
enumInstanceSerializedFields(x, fieldName, field):
|
|
||||||
trs "WRITING FIXED SIZE FIELD", fieldName
|
|
||||||
s.writeFixedSized toSszType(field)
|
|
||||||
else:
|
|
||||||
unsupported x.type
|
|
||||||
|
|
||||||
template writeOffset(cursor: var WriteCursor, offset: int) =
|
|
||||||
write cursor, toBytesLE(uint32 offset)
|
|
||||||
|
|
||||||
template supports*(_: type SSZ, T: type): bool =
|
|
||||||
mixin toSszType
|
|
||||||
anonConst compiles(fixedPortionSize toSszType(declval T))
|
|
||||||
|
|
||||||
func init*(T: type SszWriter, stream: OutputStream): T {.raises: [Defect].} =
|
|
||||||
result.stream = stream
|
|
||||||
|
|
||||||
proc writeVarSizeType(w: var SszWriter, value: auto)
|
|
||||||
{.gcsafe, raises: [Defect, IOError].}
|
|
||||||
|
|
||||||
proc beginRecord*(w: var SszWriter, TT: type): auto {.raises: [Defect].} =
|
|
||||||
type T = TT
|
|
||||||
when isFixedSize(T):
|
|
||||||
FixedSizedWriterCtx()
|
|
||||||
else:
|
|
||||||
const offset = when T is array: len(T) * offsetSize
|
|
||||||
else: fixedPortionSize(T)
|
|
||||||
VarSizedWriterCtx(offset: offset,
|
|
||||||
fixedParts: w.stream.delayFixedSizeWrite(offset))
|
|
||||||
|
|
||||||
template writeField*(w: var SszWriter,
|
|
||||||
ctx: var auto,
|
|
||||||
fieldName: string,
|
|
||||||
field: auto) =
|
|
||||||
mixin toSszType
|
|
||||||
when ctx is FixedSizedWriterCtx:
|
|
||||||
writeFixedSized(w.stream, toSszType(field))
|
|
||||||
else:
|
|
||||||
type FieldType = type toSszType(field)
|
|
||||||
|
|
||||||
when isFixedSize(FieldType):
|
|
||||||
writeFixedSized(ctx.fixedParts, toSszType(field))
|
|
||||||
else:
|
|
||||||
trs "WRITING OFFSET ", ctx.offset, " FOR ", fieldName
|
|
||||||
writeOffset(ctx.fixedParts, ctx.offset)
|
|
||||||
let initPos = w.stream.pos
|
|
||||||
trs "WRITING VAR SIZE VALUE OF TYPE ", name(FieldType)
|
|
||||||
when FieldType is BitList:
|
|
||||||
trs "BIT SEQ ", bytes(field)
|
|
||||||
writeVarSizeType(w, toSszType(field))
|
|
||||||
ctx.offset += w.stream.pos - initPos
|
|
||||||
|
|
||||||
template endRecord*(w: var SszWriter, ctx: var auto) =
|
|
||||||
when ctx is VarSizedWriterCtx:
|
|
||||||
finalize ctx.fixedParts
|
|
||||||
|
|
||||||
proc writeSeq[T](w: var SszWriter, value: seq[T])
|
|
||||||
{.raises: [Defect, IOError].} =
|
|
||||||
# Please note that `writeSeq` exists in order to reduce the code bloat
|
|
||||||
# produced from generic instantiations of the unique `List[N, T]` types.
|
|
||||||
when isFixedSize(T):
|
|
||||||
trs "WRITING LIST WITH FIXED SIZE ELEMENTS"
|
|
||||||
for elem in value:
|
|
||||||
w.stream.writeFixedSized toSszType(elem)
|
|
||||||
trs "DONE"
|
|
||||||
else:
|
|
||||||
trs "WRITING LIST WITH VAR SIZE ELEMENTS"
|
|
||||||
var offset = value.len * offsetSize
|
|
||||||
var cursor = w.stream.delayFixedSizeWrite offset
|
|
||||||
for elem in value:
|
|
||||||
cursor.writeFixedSized uint32(offset)
|
|
||||||
let initPos = w.stream.pos
|
|
||||||
w.writeVarSizeType toSszType(elem)
|
|
||||||
offset += w.stream.pos - initPos
|
|
||||||
finalize cursor
|
|
||||||
trs "DONE"
|
|
||||||
|
|
||||||
proc writeVarSizeType(w: var SszWriter, value: auto)
|
|
||||||
{.raises: [Defect, IOError].} =
|
|
||||||
trs "STARTING VAR SIZE TYPE"
|
|
||||||
|
|
||||||
when value is List:
|
|
||||||
# We reduce code bloat by forwarding all `List` types to a general `seq[T]`
|
|
||||||
# proc.
|
|
||||||
writeSeq(w, asSeq value)
|
|
||||||
elif value is BitList:
|
|
||||||
# ATTENTION! We can reuse `writeSeq` only as long as our BitList type is
|
|
||||||
# implemented to internally match the binary representation of SSZ BitLists
|
|
||||||
# in memory.
|
|
||||||
writeSeq(w, bytes value)
|
|
||||||
elif value is object|tuple|array:
|
|
||||||
trs "WRITING OBJECT OR ARRAY"
|
|
||||||
var ctx = beginRecord(w, type value)
|
|
||||||
enumerateSubFields(value, field):
|
|
||||||
writeField w, ctx, astToStr(field), field
|
|
||||||
endRecord w, ctx
|
|
||||||
else:
|
|
||||||
unsupported type(value)
|
|
||||||
|
|
||||||
proc writeValue*(w: var SszWriter, x: auto)
|
|
||||||
{.gcsafe, raises: [Defect, IOError].} =
|
|
||||||
mixin toSszType
|
|
||||||
type T = type toSszType(x)
|
|
||||||
|
|
||||||
when isFixedSize(T):
|
|
||||||
w.stream.writeFixedSized toSszType(x)
|
|
||||||
else:
|
|
||||||
w.writeVarSizeType toSszType(x)
|
|
||||||
|
|
||||||
func sszSize*(value: auto): int {.gcsafe, raises: [Defect].}
|
|
||||||
|
|
||||||
func sszSizeForVarSizeList[T](value: openArray[T]): int =
|
|
||||||
mixin toSszType
|
|
||||||
result = len(value) * offsetSize
|
|
||||||
for elem in value:
|
|
||||||
result += sszSize(toSszType elem)
|
|
||||||
|
|
||||||
func sszSize*(value: auto): int {.gcsafe, raises: [Defect].} =
|
|
||||||
mixin toSszType
|
|
||||||
type T = type toSszType(value)
|
|
||||||
|
|
||||||
when isFixedSize(T):
|
|
||||||
anonConst fixedPortionSize(T)
|
|
||||||
|
|
||||||
elif T is array|List:
|
|
||||||
type E = ElemType(T)
|
|
||||||
when isFixedSize(E):
|
|
||||||
len(value) * anonConst(fixedPortionSize(E))
|
|
||||||
elif T is HashArray:
|
|
||||||
sszSizeForVarSizeList(value.data)
|
|
||||||
elif T is array:
|
|
||||||
sszSizeForVarSizeList(value)
|
|
||||||
else:
|
|
||||||
sszSizeForVarSizeList(asSeq value)
|
|
||||||
|
|
||||||
elif T is BitList:
|
|
||||||
return len(bytes(value))
|
|
||||||
|
|
||||||
elif T is object|tuple:
|
|
||||||
result = anonConst fixedPortionSize(T)
|
|
||||||
enumInstanceSerializedFields(value, _{.used.}, field):
|
|
||||||
type FieldType = type toSszType(field)
|
|
||||||
when not isFixedSize(FieldType):
|
|
||||||
result += sszSize(toSszType field)
|
|
||||||
|
|
||||||
else:
|
|
||||||
unsupported T
|
|
||||||
|
|
||||||
proc writeValue*[T](w: var SszWriter, x: SizePrefixed[T])
|
|
||||||
{.raises: [Defect, IOError].} =
|
|
||||||
var cursor = w.stream.delayVarSizeWrite(Leb128.maxLen(uint64))
|
|
||||||
let initPos = w.stream.pos
|
|
||||||
w.writeValue T(x)
|
|
||||||
let length = toBytes(uint64(w.stream.pos - initPos), Leb128)
|
|
||||||
cursor.finalWrite length.toOpenArray()
|
|
||||||
|
|
||||||
proc readValue*[T](r: var SszReader, val: var T)
|
|
||||||
{.raises: [Defect, SszError, IOError].} =
|
|
||||||
when isFixedSize(T):
|
|
||||||
const minimalSize = fixedPortionSize(T)
|
|
||||||
if r.stream.readable(minimalSize):
|
|
||||||
readSszValue(r.stream.read(minimalSize), val)
|
|
||||||
else:
|
|
||||||
raise newException(MalformedSszError, "SSZ input of insufficient size")
|
|
||||||
else:
|
|
||||||
# TODO(zah) Read the fixed portion first and precisely measure the
|
|
||||||
# size of the dynamic portion to consume the right number of bytes.
|
|
||||||
readSszValue(r.stream.read(r.stream.len.get), val)
|
|
||||||
|
|
||||||
proc readSszBytes*[T](data: openArray[byte], val: var T) {.
|
|
||||||
raises: [Defect, MalformedSszError, SszSizeMismatchError].} =
|
|
||||||
when isFixedSize(T):
|
|
||||||
const minimalSize = fixedPortionSize(T)
|
|
||||||
if data.len < minimalSize:
|
|
||||||
raise newException(MalformedSszError, "SSZ input of insufficient size")
|
|
||||||
|
|
||||||
readSszValue(data, val)
|
|
||||||
|
|
@ -1,302 +0,0 @@
|
||||||
# nim-eth - Limited SSZ implementation
|
|
||||||
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
|
||||||
# Licensed and distributed under either of
|
|
||||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
|
||||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
|
||||||
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
|
||||||
|
|
||||||
{.push raises: [Defect].}
|
|
||||||
|
|
||||||
import
|
|
||||||
std/[tables, options, typetraits, strformat],
|
|
||||||
stew/shims/macros, stew/[byteutils, bitops2, objects],
|
|
||||||
nimcrypto/hash, serialization/[object_serialization, errors],
|
|
||||||
./bitseqs
|
|
||||||
|
|
||||||
export bitseqs
|
|
||||||
|
|
||||||
const
|
|
||||||
offsetSize* = 4
|
|
||||||
bytesPerChunk* = 32
|
|
||||||
|
|
||||||
type
|
|
||||||
UintN* = SomeUnsignedInt
|
|
||||||
BasicType* = bool|UintN
|
|
||||||
|
|
||||||
Limit* = int64
|
|
||||||
|
|
||||||
List*[T; maxLen: static Limit] = distinct seq[T]
|
|
||||||
BitList*[maxLen: static Limit] = distinct BitSeq
|
|
||||||
Digest* = MDigest[32 * 8]
|
|
||||||
|
|
||||||
# Note for readers:
|
|
||||||
# We use `array` for `Vector` and
|
|
||||||
# `BitArray` for `BitVector`
|
|
||||||
|
|
||||||
SszError* = object of SerializationError
|
|
||||||
|
|
||||||
MalformedSszError* = object of SszError
|
|
||||||
|
|
||||||
SszSizeMismatchError* = object of SszError
|
|
||||||
deserializedType*: cstring
|
|
||||||
actualSszSize*: int
|
|
||||||
elementSize*: int
|
|
||||||
|
|
||||||
# A few index types from here onwards:
|
|
||||||
# * dataIdx - leaf index starting from 0 to maximum length of collection
|
|
||||||
# * chunkIdx - leaf data index after chunking starting from 0
|
|
||||||
# * vIdx - virtual index in merkle tree - the root is found at index 1, its
|
|
||||||
# two children at 2, 3 then 4, 5, 6, 7 etc
|
|
||||||
|
|
||||||
func nextPow2Int64(x: int64): int64 =
|
|
||||||
# TODO the nextPow2 in bitops2 works with uint64 - there's a bug in the nim
|
|
||||||
# compiler preventing it to be used - it seems that a conversion to
|
|
||||||
# uint64 cannot be done with the static maxLen :(
|
|
||||||
var v = x - 1
|
|
||||||
|
|
||||||
# round down, make sure all bits are 1 below the threshold, then add 1
|
|
||||||
v = v or v shr 1
|
|
||||||
v = v or v shr 2
|
|
||||||
v = v or v shr 4
|
|
||||||
when bitsof(x) > 8:
|
|
||||||
v = v or v shr 8
|
|
||||||
when bitsof(x) > 16:
|
|
||||||
v = v or v shr 16
|
|
||||||
when bitsof(x) > 32:
|
|
||||||
v = v or v shr 32
|
|
||||||
|
|
||||||
v + 1
|
|
||||||
|
|
||||||
template dataPerChunk(T: type): int =
|
|
||||||
# How many data items fit in a chunk
|
|
||||||
when T is BasicType:
|
|
||||||
bytesPerChunk div sizeof(T)
|
|
||||||
else:
|
|
||||||
1
|
|
||||||
|
|
||||||
template chunkIdx*(T: type, dataIdx: int64): int64 =
|
|
||||||
# Given a data index, which chunk does it belong to?
|
|
||||||
dataIdx div dataPerChunk(T)
|
|
||||||
|
|
||||||
template maxChunkIdx*(T: type, maxLen: Limit): int64 =
|
|
||||||
# Given a number of data items, how many chunks are needed?
|
|
||||||
# TODO compiler bug:
|
|
||||||
# beacon_chain/ssz/types.nim(75, 53) Error: cannot generate code for: maxLen
|
|
||||||
# nextPow2(chunkIdx(T, maxLen + dataPerChunk(T) - 1).uint64).int64
|
|
||||||
nextPow2Int64(chunkIdx(T, maxLen.int64 + dataPerChunk(T) - 1))
|
|
||||||
|
|
||||||
template asSeq*(x: List): auto = distinctBase(x)
|
|
||||||
|
|
||||||
template init*[T](L: type List, x: seq[T], N: static Limit): auto =
|
|
||||||
List[T, N](x)
|
|
||||||
|
|
||||||
template init*[T, N](L: type List[T, N], x: seq[T]): auto =
|
|
||||||
List[T, N](x)
|
|
||||||
|
|
||||||
template `$`*(x: List): auto = $(distinctBase x)
|
|
||||||
template len*(x: List): auto = len(distinctBase x)
|
|
||||||
template low*(x: List): auto = low(distinctBase x)
|
|
||||||
template high*(x: List): auto = high(distinctBase x)
|
|
||||||
template `[]`*(x: List, idx: auto): untyped = distinctBase(x)[idx]
|
|
||||||
template `[]=`*(x: var List, idx: auto, val: auto) = distinctBase(x)[idx] = val
|
|
||||||
template `==`*(a, b: List): bool = distinctBase(a) == distinctBase(b)
|
|
||||||
|
|
||||||
template `&`*(a, b: List): auto = (type(a)(distinctBase(a) & distinctBase(b)))
|
|
||||||
|
|
||||||
template items* (x: List): untyped = items(distinctBase x)
|
|
||||||
template pairs* (x: List): untyped = pairs(distinctBase x)
|
|
||||||
template mitems*(x: var List): untyped = mitems(distinctBase x)
|
|
||||||
template mpairs*(x: var List): untyped = mpairs(distinctBase x)
|
|
||||||
|
|
||||||
template contains* (x: List, val: auto): untyped = contains(distinctBase x, val)
|
|
||||||
|
|
||||||
proc add*(x: var List, val: auto): bool =
|
|
||||||
if x.len < x.maxLen:
|
|
||||||
add(distinctBase x, val)
|
|
||||||
true
|
|
||||||
else:
|
|
||||||
false
|
|
||||||
|
|
||||||
proc setLen*(x: var List, newLen: int): bool =
|
|
||||||
if newLen <= x.maxLen:
|
|
||||||
setLen(distinctBase x, newLen)
|
|
||||||
true
|
|
||||||
else:
|
|
||||||
false
|
|
||||||
|
|
||||||
template init*(L: type BitList, x: seq[byte], N: static Limit): auto =
|
|
||||||
BitList[N](data: x)
|
|
||||||
|
|
||||||
template init*[N](L: type BitList[N], x: seq[byte]): auto =
|
|
||||||
L(data: x)
|
|
||||||
|
|
||||||
template init*(T: type BitList, len: int): auto = T init(BitSeq, len)
|
|
||||||
template len*(x: BitList): auto = len(BitSeq(x))
|
|
||||||
template bytes*(x: BitList): auto = seq[byte](x)
|
|
||||||
template `[]`*(x: BitList, idx: auto): auto = BitSeq(x)[idx]
|
|
||||||
template `[]=`*(x: var BitList, idx: auto, val: bool) = BitSeq(x)[idx] = val
|
|
||||||
template `==`*(a, b: BitList): bool = BitSeq(a) == BitSeq(b)
|
|
||||||
template setBit*(x: var BitList, idx: Natural) = setBit(BitSeq(x), idx)
|
|
||||||
template clearBit*(x: var BitList, idx: Natural) = clearBit(BitSeq(x), idx)
|
|
||||||
template overlaps*(a, b: BitList): bool = overlaps(BitSeq(a), BitSeq(b))
|
|
||||||
template incl*(a: var BitList, b: BitList) = incl(BitSeq(a), BitSeq(b))
|
|
||||||
template isSubsetOf*(a, b: BitList): bool = isSubsetOf(BitSeq(a), BitSeq(b))
|
|
||||||
template isZeros*(x: BitList): bool = isZeros(BitSeq(x))
|
|
||||||
template countOnes*(x: BitList): int = countOnes(BitSeq(x))
|
|
||||||
template countZeros*(x: BitList): int = countZeros(BitSeq(x))
|
|
||||||
template countOverlap*(x, y: BitList): int = countOverlap(BitSeq(x), BitSeq(y))
|
|
||||||
template `$`*(a: BitList): string = $(BitSeq(a))
|
|
||||||
|
|
||||||
iterator items*(x: BitList): bool =
|
|
||||||
for i in 0 ..< x.len:
|
|
||||||
yield x[i]
|
|
||||||
|
|
||||||
macro unsupported*(T: typed): untyped =
|
|
||||||
# TODO: {.fatal.} breaks compilation even in `compiles()` context,
|
|
||||||
# so we use this macro instead. It's also much better at figuring
|
|
||||||
# out the actual type that was used in the instantiation.
|
|
||||||
# File both problems as issues.
|
|
||||||
error "SSZ serialization of the type " & humaneTypeName(T) & " is not supported"
|
|
||||||
|
|
||||||
template ElemType*(T: type array): untyped =
|
|
||||||
type(default(T)[low(T)])
|
|
||||||
|
|
||||||
template ElemType*(T: type seq): untyped =
|
|
||||||
type(default(T)[0])
|
|
||||||
|
|
||||||
template ElemType*(T0: type List): untyped =
|
|
||||||
T0.T
|
|
||||||
|
|
||||||
func isFixedSize*(T0: type): bool {.compileTime.} =
|
|
||||||
mixin toSszType, enumAllSerializedFields
|
|
||||||
|
|
||||||
type T = type toSszType(declval T0)
|
|
||||||
|
|
||||||
when T is BasicType:
|
|
||||||
return true
|
|
||||||
elif T is array:
|
|
||||||
return isFixedSize(ElemType(T))
|
|
||||||
elif T is object|tuple:
|
|
||||||
enumAllSerializedFields(T):
|
|
||||||
when not isFixedSize(FieldType):
|
|
||||||
return false
|
|
||||||
return true
|
|
||||||
|
|
||||||
func fixedPortionSize*(T0: type): int {.compileTime.} =
|
|
||||||
mixin enumAllSerializedFields, toSszType
|
|
||||||
|
|
||||||
type T = type toSszType(declval T0)
|
|
||||||
|
|
||||||
when T is BasicType: sizeof(T)
|
|
||||||
elif T is array:
|
|
||||||
type E = ElemType(T)
|
|
||||||
when isFixedSize(E): int(len(T)) * fixedPortionSize(E)
|
|
||||||
else: int(len(T)) * offsetSize
|
|
||||||
elif T is object|tuple:
|
|
||||||
enumAllSerializedFields(T):
|
|
||||||
when isFixedSize(FieldType):
|
|
||||||
result += fixedPortionSize(FieldType)
|
|
||||||
else:
|
|
||||||
result += offsetSize
|
|
||||||
else:
|
|
||||||
unsupported T0
|
|
||||||
|
|
||||||
# TODO This should have been an iterator, but the VM can't compile the
|
|
||||||
# code due to "too many registers required".
|
|
||||||
proc fieldInfos*(RecordType: type): seq[tuple[name: string,
|
|
||||||
offset: int,
|
|
||||||
fixedSize: int,
|
|
||||||
branchKey: string]] =
|
|
||||||
mixin enumAllSerializedFields
|
|
||||||
|
|
||||||
var
|
|
||||||
offsetInBranch = {"": 0}.toTable
|
|
||||||
nestedUnder = initTable[string, string]()
|
|
||||||
|
|
||||||
enumAllSerializedFields(RecordType):
|
|
||||||
const
|
|
||||||
isFixed = isFixedSize(FieldType)
|
|
||||||
fixedSize = when isFixed: fixedPortionSize(FieldType)
|
|
||||||
else: 0
|
|
||||||
branchKey = when fieldCaseDiscriminator.len == 0: ""
|
|
||||||
else: fieldCaseDiscriminator & ":" & $fieldCaseBranches
|
|
||||||
fieldSize = when isFixed: fixedSize
|
|
||||||
else: offsetSize
|
|
||||||
|
|
||||||
nestedUnder[fieldName] = branchKey
|
|
||||||
|
|
||||||
var fieldOffset: int
|
|
||||||
offsetInBranch.withValue(branchKey, val):
|
|
||||||
fieldOffset = val[]
|
|
||||||
val[] += fieldSize
|
|
||||||
do:
|
|
||||||
try:
|
|
||||||
let parentBranch = nestedUnder.getOrDefault(fieldCaseDiscriminator, "")
|
|
||||||
fieldOffset = offsetInBranch[parentBranch]
|
|
||||||
offsetInBranch[branchKey] = fieldOffset + fieldSize
|
|
||||||
except KeyError as e:
|
|
||||||
raiseAssert e.msg
|
|
||||||
|
|
||||||
result.add((fieldName, fieldOffset, fixedSize, branchKey))
|
|
||||||
|
|
||||||
func getFieldBoundingOffsetsImpl(RecordType: type, fieldName: static string):
|
|
||||||
tuple[fieldOffset, nextFieldOffset: int, isFirstOffset: bool]
|
|
||||||
{.compileTime.} =
|
|
||||||
result = (-1, -1, false)
|
|
||||||
var fieldBranchKey: string
|
|
||||||
var isFirstOffset = true
|
|
||||||
|
|
||||||
for f in fieldInfos(RecordType):
|
|
||||||
if fieldName == f.name:
|
|
||||||
result[0] = f.offset
|
|
||||||
if f.fixedSize > 0:
|
|
||||||
result[1] = result[0] + f.fixedSize
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
fieldBranchKey = f.branchKey
|
|
||||||
result.isFirstOffset = isFirstOffset
|
|
||||||
|
|
||||||
elif result[0] != -1 and
|
|
||||||
f.fixedSize == 0 and
|
|
||||||
f.branchKey == fieldBranchKey:
|
|
||||||
# We have found the next variable sized field
|
|
||||||
result[1] = f.offset
|
|
||||||
return
|
|
||||||
|
|
||||||
if f.fixedSize == 0:
|
|
||||||
isFirstOffset = false
|
|
||||||
|
|
||||||
func getFieldBoundingOffsets*(RecordType: type, fieldName: static string):
|
|
||||||
tuple[fieldOffset, nextFieldOffset: int, isFirstOffset: bool]
|
|
||||||
{.compileTime.} =
|
|
||||||
## Returns the start and end offsets of a field.
|
|
||||||
##
|
|
||||||
## For fixed-size fields, the start offset points to the first
|
|
||||||
## byte of the field and the end offset points to 1 byte past the
|
|
||||||
## end of the field.
|
|
||||||
##
|
|
||||||
## For variable-size fields, the returned offsets point to the
|
|
||||||
## statically known positions of the 32-bit offset values written
|
|
||||||
## within the SSZ object. You must read the 32-bit values stored
|
|
||||||
## at the these locations in order to obtain the actual offsets.
|
|
||||||
##
|
|
||||||
## For variable-size fields, the end offset may be -1 when the
|
|
||||||
## designated field is the last variable sized field within the
|
|
||||||
## object. Then the SSZ object boundary known at run-time marks
|
|
||||||
## the end of the variable-size field.
|
|
||||||
type T = RecordType
|
|
||||||
anonConst getFieldBoundingOffsetsImpl(T, fieldName)
|
|
||||||
|
|
||||||
template enumerateSubFields*(holder, fieldVar, body: untyped) =
|
|
||||||
when holder is array:
|
|
||||||
for fieldVar in holder: body
|
|
||||||
else:
|
|
||||||
enumInstanceSerializedFields(holder, _{.used.}, fieldVar): body
|
|
||||||
|
|
||||||
method formatMsg*(
|
|
||||||
err: ref SszSizeMismatchError,
|
|
||||||
filename: string): string {.gcsafe, raises: [Defect].} =
|
|
||||||
try:
|
|
||||||
&"SSZ size mismatch, element {err.elementSize}, actual {err.actualSszSize}, type {err.deserializedType}, file {filename}"
|
|
||||||
except CatchableError:
|
|
||||||
"SSZ size mismatch"
|
|
||||||
|
|
@ -1,3 +0,0 @@
|
||||||
import
|
|
||||||
./test_verification,
|
|
||||||
./test_proofs
|
|
||||||
|
|
@ -1,123 +0,0 @@
|
||||||
{.used.}
|
|
||||||
|
|
||||||
import
|
|
||||||
sequtils, unittest, math,
|
|
||||||
nimcrypto/[hash, sha2],
|
|
||||||
stew/endians2,
|
|
||||||
../eth/ssz/merkleization,
|
|
||||||
../eth/ssz/ssz_serialization,
|
|
||||||
../eth/ssz/merkle_tree
|
|
||||||
|
|
||||||
template toSszType(x: auto): auto =
|
|
||||||
x
|
|
||||||
|
|
||||||
proc h(a: openArray[byte]): Digest =
|
|
||||||
var h: sha256
|
|
||||||
h.init()
|
|
||||||
h.update(a)
|
|
||||||
h.finish()
|
|
||||||
|
|
||||||
type TestObject = object
|
|
||||||
digest: array[32, byte]
|
|
||||||
num: uint64
|
|
||||||
|
|
||||||
proc genObject(num: uint64): TestObject =
|
|
||||||
let numAsHash = h(num.toBytesLE())
|
|
||||||
TestObject(digest: numAsHash.data, num: num)
|
|
||||||
|
|
||||||
proc genNObjects(n: int): seq[TestObject] =
|
|
||||||
var objs = newSeq[TestObject]()
|
|
||||||
for i in 1..n:
|
|
||||||
let obj = genObject(uint64 i)
|
|
||||||
objs.add(obj)
|
|
||||||
objs
|
|
||||||
|
|
||||||
proc getGenIndex(idx: int, depth: uint64): uint64 =
|
|
||||||
uint64 (math.pow(2'f64, float64 depth) + float64 idx)
|
|
||||||
|
|
||||||
# Normal hash_tree_root add list length to final hash calculation. Proofs by default
|
|
||||||
# are generated without it. If necessary length of the list can be added manually
|
|
||||||
# at the end of the proof but here we are just hashing list with no mixin.
|
|
||||||
proc getListRootNoMixin(list: List): Digest =
|
|
||||||
var merk = createMerkleizer(list.maxLen)
|
|
||||||
for e in list:
|
|
||||||
let hash = hash_tree_root(e)
|
|
||||||
merk.addChunk(hash.data)
|
|
||||||
merk.getFinalHash()
|
|
||||||
|
|
||||||
type TestCase = object
|
|
||||||
numOfElements: int
|
|
||||||
limit: int
|
|
||||||
|
|
||||||
const TestCases = (
|
|
||||||
TestCase(numOfElements: 0, limit: 2),
|
|
||||||
TestCase(numOfElements: 1, limit: 2),
|
|
||||||
TestCase(numOfElements: 2, limit: 2),
|
|
||||||
|
|
||||||
TestCase(numOfElements: 0, limit: 4),
|
|
||||||
TestCase(numOfElements: 1, limit: 4),
|
|
||||||
TestCase(numOfElements: 2, limit: 4),
|
|
||||||
TestCase(numOfElements: 3, limit: 4),
|
|
||||||
TestCase(numOfElements: 4, limit: 4),
|
|
||||||
|
|
||||||
TestCase(numOfElements: 0, limit: 8),
|
|
||||||
TestCase(numOfElements: 1, limit: 8),
|
|
||||||
TestCase(numOfElements: 2, limit: 8),
|
|
||||||
TestCase(numOfElements: 3, limit: 8),
|
|
||||||
TestCase(numOfElements: 4, limit: 8),
|
|
||||||
TestCase(numOfElements: 5, limit: 8),
|
|
||||||
TestCase(numOfElements: 6, limit: 8),
|
|
||||||
TestCase(numOfElements: 7, limit: 8),
|
|
||||||
TestCase(numOfElements: 8, limit: 8),
|
|
||||||
|
|
||||||
TestCase(numOfElements: 0, limit: 16),
|
|
||||||
TestCase(numOfElements: 1, limit: 16),
|
|
||||||
TestCase(numOfElements: 2, limit: 16),
|
|
||||||
TestCase(numOfElements: 3, limit: 16),
|
|
||||||
TestCase(numOfElements: 4, limit: 16),
|
|
||||||
TestCase(numOfElements: 5, limit: 16),
|
|
||||||
TestCase(numOfElements: 6, limit: 16),
|
|
||||||
TestCase(numOfElements: 7, limit: 16),
|
|
||||||
TestCase(numOfElements: 16, limit: 16),
|
|
||||||
|
|
||||||
TestCase(numOfElements: 32, limit: 32),
|
|
||||||
|
|
||||||
TestCase(numOfElements: 64, limit: 64)
|
|
||||||
)
|
|
||||||
|
|
||||||
suite "Merkle Proof generation":
|
|
||||||
test "generation of proof for various tree sizes":
|
|
||||||
for testCase in TestCases.fields:
|
|
||||||
let testObjects = genNObjects(testCase.numOfElements)
|
|
||||||
let treeDepth = uint64 binaryTreeHeight(testCase.limit) - 1
|
|
||||||
|
|
||||||
# Create List and and genereate root by using merkelizer
|
|
||||||
let list = List.init(testObjects, testCase.limit)
|
|
||||||
let listRoot = getListRootNoMixin(list)
|
|
||||||
|
|
||||||
# Create sparse merkle tree from list elements and generate root
|
|
||||||
let listDigests = map(testObjects, proc(x: TestObject): Digest = hash_tree_root(x))
|
|
||||||
let tree = createTree(listDigests, treeDepth)
|
|
||||||
let treeHash = tree.hash()
|
|
||||||
|
|
||||||
# Assert that by using both methods we get same hash
|
|
||||||
check listRoot == treeHash
|
|
||||||
|
|
||||||
for i, e in list:
|
|
||||||
# generate proof by using merkelizer
|
|
||||||
let merkleizerProof = generateAndGetProofWithIdx(list, i)
|
|
||||||
# generate proof by sparse merkle tree
|
|
||||||
let sparseTreeProof = genProof(tree, uint64 i, treeDepth)
|
|
||||||
|
|
||||||
let leafHash = hash_tree_root(e)
|
|
||||||
let genIndex = getGenIndex(i, treeDepth)
|
|
||||||
|
|
||||||
# both proof are valid. If both are valid that means that both proof are
|
|
||||||
# effectivly the same
|
|
||||||
let isValidProof = isValidProof(leafHash , merkleizerProof, genIndex, listRoot)
|
|
||||||
let isValidProof1 = isValidProof(leafHash , sparseTreeProof, genIndex, listRoot)
|
|
||||||
|
|
||||||
check isValidProof
|
|
||||||
check isValidProof1
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,82 +0,0 @@
|
||||||
{.used.}
|
|
||||||
|
|
||||||
import
|
|
||||||
sequtils, unittest,
|
|
||||||
nimcrypto/[hash, sha2],
|
|
||||||
../eth/ssz/merkleization
|
|
||||||
|
|
||||||
type TestCase = object
|
|
||||||
root: string
|
|
||||||
proof: seq[string]
|
|
||||||
leaf: string
|
|
||||||
index: uint64
|
|
||||||
valid: bool
|
|
||||||
|
|
||||||
let testCases = @[
|
|
||||||
TestCase(
|
|
||||||
root: "2a23ef2b7a7221eaac2ffb3842a506a981c009ca6c2fcbf20adbc595e56f1a93",
|
|
||||||
proof: @[
|
|
||||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
|
||||||
"f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b"
|
|
||||||
],
|
|
||||||
leaf: "0100000000000000000000000000000000000000000000000000000000000000",
|
|
||||||
index: 4,
|
|
||||||
valid: true
|
|
||||||
),
|
|
||||||
TestCase(
|
|
||||||
root: "2a23ef2b7a7221eaac2ffb3842a506a981c009ca6c2fcbf20adbc595e56f1a93",
|
|
||||||
proof: @[
|
|
||||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
|
||||||
"f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b"
|
|
||||||
],
|
|
||||||
leaf: "0100000000000000000000000000000000000000000000000000000000000000",
|
|
||||||
index: 6,
|
|
||||||
valid: false
|
|
||||||
),
|
|
||||||
TestCase(
|
|
||||||
root: "2a23ef2b7a7221eaac2ffb3842a506a981c009ca6c2fcbf20adbc595e56f1a93",
|
|
||||||
proof: @[
|
|
||||||
"0100000000000000000000000000000000000000000000000000000000000000",
|
|
||||||
"f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b"
|
|
||||||
],
|
|
||||||
leaf: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
|
||||||
index: 5,
|
|
||||||
valid: true
|
|
||||||
),
|
|
||||||
TestCase(
|
|
||||||
root: "f1824b0084956084591ff4c91c11bcc94a40be82da280e5171932b967dd146e9",
|
|
||||||
proof: @[
|
|
||||||
"35210d64853aee79d03f30cf0f29c1398706cbbcacaf05ab9524f00070aec91e",
|
|
||||||
"f38a181470ef1eee90a29f0af0a9dba6b7e5d48af3c93c29b4f91fa11b777582"
|
|
||||||
],
|
|
||||||
leaf: "0100000000000000000000000000000000000000000000000000000000000000",
|
|
||||||
index: 7,
|
|
||||||
valid: true
|
|
||||||
),
|
|
||||||
TestCase(
|
|
||||||
root: "f1824b0084956084591ff4c91c11bcc94a40be82da280e5171932b967dd146e9",
|
|
||||||
proof: @[
|
|
||||||
"0000000000000000000000000000000000000000000000000000000000000000",
|
|
||||||
"0000000000000000000000000000000000000000000000000000000000000000",
|
|
||||||
"f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b",
|
|
||||||
"0100000000000000000000000000000000000000000000000000000000000000",
|
|
||||||
"f38a181470ef1eee90a29f0af0a9dba6b7e5d48af3c93c29b4f91fa11b777582"
|
|
||||||
],
|
|
||||||
leaf: "6001000000000000000000000000000000000000000000000000000000000000",
|
|
||||||
index: 49,
|
|
||||||
valid: true
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
suite "Merkle Proof verification":
|
|
||||||
test "correctly verify proof":
|
|
||||||
for testCase in testCases:
|
|
||||||
let root = MDigest[256].fromHex(testCase.root)
|
|
||||||
let proof = map(testCase.proof, proc(x: string): Digest = MDigest[256].fromHex(x))
|
|
||||||
let leaf = MDigest[256].fromHex(testCase.leaf)
|
|
||||||
let valid = isValidProof(leaf, proof, testCase.index, root)
|
|
||||||
|
|
||||||
if (testCase.valid):
|
|
||||||
check valid
|
|
||||||
else:
|
|
||||||
check (not valid)
|
|
||||||
Loading…
Reference in New Issue