mirror of https://github.com/status-im/nim-eth.git
remove outdated and incorrect SSZ code
This removes the outdated copy of the SSZ code. It became incorrect over time (e.g., empty SSZ list elements), and is no longer in use by GitHub projects: https://github.com/search?q=extension%3Anim+eth%2Fssz The canonical SSZ implementation resides at `nim-ssz-serialization`. Compared to `nim-eth`, these changes were made meanwhile: - `bitseqs` was extended with JSON serialization support, and with the new functions `isZero` and `countOnes`. - `bytes_reader` was renamed to `codec`, extended with a few additional SSZ type conversions as well as support for `SingleMemberUnion`. - The simplified merkle tree implementation in `merkle_tree.nim` was removed. It was not used by other projects. - `merkleization` was extended with support for `HashArray`, `HashList` and `SingleMemberUnion`. The `isValidProof` functionality has been moved to `nimbus-eth2` and replaced with the EF defined function `is_valid_merkle_branch`. The test was also moved to `nimbus-eth2`. There are no other GitHub projects using `isValidProof`: https://github.com/search?q=extension%3Anim+isValidProof Furthermore, a definition for `GeneralizedIndex` was added. - `ssz_serialization` was moved one directory up, and improved with bug fixes and `HashArray`, `HashList` and `SingleMemberUnion` support. - `types` was extended with JSON serialization and new type support for `Uint128`, `Uint256`, `HashArray`, `HashList` and `SingleMemberUnion`. There is also a new `getBit` function for `BitList`.
This commit is contained in:
Etan Kissling
parent
41d2d3c991
commit
3ce2d9a58e
|
|
@ -66,9 +66,6 @@ task test_trie, "Run trie tests":
|
|||
task test_db, "Run db tests":
|
||||
runTest("tests/db/all_tests")
|
||||
|
||||
task test_ssz, "Run ssz tests":
|
||||
runTest("tests/ssz/all_tests")
|
||||
|
||||
task test_utp, "Run utp tests":
|
||||
runTest("tests/utp/all_utp_tests")
|
||||
|
||||
|
|
@ -84,7 +81,6 @@ task test, "Run all tests":
|
|||
test_p2p_task()
|
||||
test_trie_task()
|
||||
test_db_task()
|
||||
test_ssz_task()
|
||||
test_utp_task()
|
||||
|
||||
task test_discv5_full, "Run discovery v5 and its dependencies tests":
|
||||
|
|
|
|||
|
|
@ -1,313 +0,0 @@
|
|||
# nim-eth
|
||||
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
||||
|
||||
{.push raises: [Defect].}
|
||||
|
||||
import
|
||||
stew/[bitops2, endians2, ptrops]
|
||||
|
||||
type
|
||||
Bytes = seq[byte]
|
||||
|
||||
BitSeq* = distinct Bytes
|
||||
## The current design of BitSeq tries to follow precisely
|
||||
## the bitwise representation of the SSZ bitlists.
|
||||
## This is a relatively compact representation, but as
|
||||
## evident from the code below, many of the operations
|
||||
## are not trivial.
|
||||
|
||||
BitArray*[bits: static int] = object
|
||||
bytes*: array[(bits + 7) div 8, byte]
|
||||
|
||||
func bitsLen*(bytes: openArray[byte]): int =
|
||||
let
|
||||
bytesCount = bytes.len
|
||||
lastByte = bytes[bytesCount - 1]
|
||||
markerPos = log2trunc(lastByte)
|
||||
|
||||
bytesCount * 8 - (8 - markerPos)
|
||||
|
||||
template len*(s: BitSeq): int =
|
||||
bitsLen(Bytes s)
|
||||
|
||||
template len*(a: BitArray): int =
|
||||
a.bits
|
||||
|
||||
func add*(s: var BitSeq, value: bool) =
|
||||
let
|
||||
lastBytePos = s.Bytes.len - 1
|
||||
lastByte = s.Bytes[lastBytePos]
|
||||
|
||||
if (lastByte and byte(128)) == 0:
|
||||
# There is at least one leading zero, so we have enough
|
||||
# room to store the new bit
|
||||
let markerPos = log2trunc(lastByte)
|
||||
s.Bytes[lastBytePos].changeBit markerPos, value
|
||||
s.Bytes[lastBytePos].setBit markerPos + 1
|
||||
else:
|
||||
s.Bytes[lastBytePos].changeBit 7, value
|
||||
s.Bytes.add byte(1)
|
||||
|
||||
func toBytesLE(x: uint): array[sizeof(x), byte] =
|
||||
# stew/endians2 supports explicitly sized uints only
|
||||
when sizeof(uint) == 4:
|
||||
static: doAssert sizeof(uint) == sizeof(uint32)
|
||||
toBytesLE(x.uint32)
|
||||
elif sizeof(uint) == 8:
|
||||
static: doAssert sizeof(uint) == sizeof(uint64)
|
||||
toBytesLE(x.uint64)
|
||||
else:
|
||||
static: doAssert false, "requires a 32-bit or 64-bit platform"
|
||||
|
||||
func loadLEBytes(WordType: type, bytes: openArray[byte]): WordType =
|
||||
# TODO: this is a temporary proc until the endians API is improved
|
||||
var shift = 0
|
||||
for b in bytes:
|
||||
result = result or (WordType(b) shl shift)
|
||||
shift += 8
|
||||
|
||||
func storeLEBytes(value: SomeUnsignedInt, dst: var openArray[byte]) =
|
||||
doAssert dst.len <= sizeof(value)
|
||||
let bytesLE = toBytesLE(value)
|
||||
copyMem(addr dst[0], unsafeAddr bytesLE[0], dst.len)
|
||||
|
||||
template loopOverWords(lhs, rhs: BitSeq,
|
||||
lhsIsVar, rhsIsVar: static bool,
|
||||
WordType: type,
|
||||
lhsBits, rhsBits, body: untyped) =
|
||||
const hasRhs = astToStr(lhs) != astToStr(rhs)
|
||||
|
||||
let bytesCount = len Bytes(lhs)
|
||||
when hasRhs: doAssert len(Bytes(rhs)) == bytesCount
|
||||
|
||||
var fullWordsCount = bytesCount div sizeof(WordType)
|
||||
let lastWordSize = bytesCount mod sizeof(WordType)
|
||||
|
||||
block:
|
||||
var lhsWord: WordType
|
||||
when hasRhs:
|
||||
var rhsWord: WordType
|
||||
var firstByteOfLastWord, lastByteOfLastWord: int
|
||||
|
||||
# TODO: Returning a `var` value from an iterator is always safe due to
|
||||
# the way inlining works, but currently the compiler reports an error
|
||||
# when a local variable escapes. We have to cheat it with this location
|
||||
# obfuscation through pointers:
|
||||
template lhsBits: auto = (addr(lhsWord))[]
|
||||
|
||||
when hasRhs:
|
||||
template rhsBits: auto = (addr(rhsWord))[]
|
||||
|
||||
template lastWordBytes(bitseq): auto =
|
||||
Bytes(bitseq).toOpenArray(firstByteOfLastWord, lastByteOfLastWord)
|
||||
|
||||
template initLastWords =
|
||||
lhsWord = loadLEBytes(WordType, lastWordBytes(lhs))
|
||||
when hasRhs: rhsWord = loadLEBytes(WordType, lastWordBytes(rhs))
|
||||
|
||||
if lastWordSize == 0:
|
||||
firstByteOfLastWord = bytesCount - sizeof(WordType)
|
||||
lastByteOfLastWord = bytesCount - 1
|
||||
dec fullWordsCount
|
||||
else:
|
||||
firstByteOfLastWord = bytesCount - lastWordSize
|
||||
lastByteOfLastWord = bytesCount - 1
|
||||
|
||||
initLastWords()
|
||||
let markerPos = log2trunc(lhsWord)
|
||||
when hasRhs: doAssert log2trunc(rhsWord) == markerPos
|
||||
|
||||
lhsWord.clearBit markerPos
|
||||
when hasRhs: rhsWord.clearBit markerPos
|
||||
|
||||
body
|
||||
|
||||
when lhsIsVar or rhsIsVar:
|
||||
let
|
||||
markerBit = uint(1 shl markerPos)
|
||||
mask = markerBit - 1'u
|
||||
|
||||
when lhsIsVar:
|
||||
let lhsEndResult = (lhsWord and mask) or markerBit
|
||||
storeLEBytes(lhsEndResult, lastWordBytes(lhs))
|
||||
|
||||
when rhsIsVar:
|
||||
let rhsEndResult = (rhsWord and mask) or markerBit
|
||||
storeLEBytes(rhsEndResult, lastWordBytes(rhs))
|
||||
|
||||
var lhsCurrAddr = cast[ptr WordType](unsafeAddr Bytes(lhs)[0])
|
||||
let lhsEndAddr = offset(lhsCurrAddr, fullWordsCount)
|
||||
when hasRhs:
|
||||
var rhsCurrAddr = cast[ptr WordType](unsafeAddr Bytes(rhs)[0])
|
||||
|
||||
while lhsCurrAddr < lhsEndAddr:
|
||||
template lhsBits: auto = lhsCurrAddr[]
|
||||
when hasRhs:
|
||||
template rhsBits: auto = rhsCurrAddr[]
|
||||
|
||||
body
|
||||
|
||||
lhsCurrAddr = offset(lhsCurrAddr, 1)
|
||||
when hasRhs: rhsCurrAddr = offset(rhsCurrAddr, 1)
|
||||
|
||||
iterator words*(x: var BitSeq): var uint =
|
||||
loopOverWords(x, x, true, false, uint, word, wordB):
|
||||
yield word
|
||||
|
||||
iterator words*(x: BitSeq): uint =
|
||||
loopOverWords(x, x, false, false, uint, word, word):
|
||||
yield word
|
||||
|
||||
iterator words*(a, b: BitSeq): (uint, uint) =
|
||||
loopOverWords(a, b, false, false, uint, wordA, wordB):
|
||||
yield (wordA, wordB)
|
||||
|
||||
iterator words*(a: var BitSeq, b: BitSeq): (var uint, uint) =
|
||||
loopOverWords(a, b, true, false, uint, wordA, wordB):
|
||||
yield (wordA, wordB)
|
||||
|
||||
iterator words*(a, b: var BitSeq): (var uint, var uint) =
|
||||
loopOverWords(a, b, true, true, uint, wordA, wordB):
|
||||
yield (wordA, wordB)
|
||||
|
||||
func `[]`*(s: BitSeq, pos: Natural): bool {.inline.} =
|
||||
doAssert pos < s.len
|
||||
s.Bytes.getBit pos
|
||||
|
||||
func `[]=`*(s: var BitSeq, pos: Natural, value: bool) {.inline.} =
|
||||
doAssert pos < s.len
|
||||
s.Bytes.changeBit pos, value
|
||||
|
||||
func setBit*(s: var BitSeq, pos: Natural) {.inline.} =
|
||||
doAssert pos < s.len
|
||||
setBit s.Bytes, pos
|
||||
|
||||
func clearBit*(s: var BitSeq, pos: Natural) {.inline.} =
|
||||
doAssert pos < s.len
|
||||
clearBit s.Bytes, pos
|
||||
|
||||
func init*(T: type BitSeq, len: int): T =
|
||||
result = BitSeq newSeq[byte](1 + len div 8)
|
||||
Bytes(result).setBit len
|
||||
|
||||
func init*(T: type BitArray): T =
|
||||
# The default zero-initializatio is fine
|
||||
discard
|
||||
|
||||
template `[]`*(a: BitArray, pos: Natural): bool =
|
||||
getBit a.bytes, pos
|
||||
|
||||
template `[]=`*(a: var BitArray, pos: Natural, value: bool) =
|
||||
changeBit a.bytes, pos, value
|
||||
|
||||
template setBit*(a: var BitArray, pos: Natural) =
|
||||
setBit a.bytes, pos
|
||||
|
||||
template clearBit*(a: var BitArray, pos: Natural) =
|
||||
clearBit a.bytes, pos
|
||||
|
||||
# TODO: Submit this to the standard library as `cmp`
|
||||
# At the moment, it doesn't work quite well because Nim selects
|
||||
# the generic cmp[T] from the system module instead of choosing
|
||||
# the openArray overload
|
||||
func compareArrays[T](a, b: openArray[T]): int =
|
||||
result = cmp(a.len, b.len)
|
||||
if result != 0: return
|
||||
|
||||
for i in 0 ..< a.len:
|
||||
result = cmp(a[i], b[i])
|
||||
if result != 0: return
|
||||
|
||||
template cmp*(a, b: BitSeq): int =
|
||||
compareArrays(Bytes a, Bytes b)
|
||||
|
||||
template `==`*(a, b: BitSeq): bool =
|
||||
cmp(a, b) == 0
|
||||
|
||||
func `$`*(a: BitSeq | BitArray): string =
|
||||
let length = a.len
|
||||
result = newStringOfCap(2 + length)
|
||||
result.add "0b"
|
||||
for i in countdown(length - 1, 0):
|
||||
result.add if a[i]: '1' else: '0'
|
||||
|
||||
func incl*(tgt: var BitSeq, src: BitSeq) =
|
||||
# Update `tgt` to include the bits of `src`, as if applying `or` to each bit
|
||||
doAssert tgt.len == src.len
|
||||
for tgtWord, srcWord in words(tgt, src):
|
||||
tgtWord = tgtWord or srcWord
|
||||
|
||||
func overlaps*(a, b: BitSeq): bool =
|
||||
for wa, wb in words(a, b):
|
||||
if (wa and wb) != 0:
|
||||
return true
|
||||
|
||||
func countOverlap*(a, b: BitSeq): int =
|
||||
var res = 0
|
||||
for wa, wb in words(a, b):
|
||||
res += countOnes(wa and wb)
|
||||
res
|
||||
|
||||
func isSubsetOf*(a, b: BitSeq): bool =
|
||||
let alen = a.len
|
||||
doAssert b.len == alen
|
||||
for i in 0 ..< alen:
|
||||
if a[i] and not b[i]:
|
||||
return false
|
||||
true
|
||||
|
||||
func isZeros*(x: BitSeq): bool =
|
||||
for w in words(x):
|
||||
if w != 0: return false
|
||||
return true
|
||||
|
||||
func countOnes*(x: BitSeq): int =
|
||||
# Count the number of set bits
|
||||
var res = 0
|
||||
for w in words(x):
|
||||
res += w.countOnes()
|
||||
res
|
||||
|
||||
func clear*(x: var BitSeq) =
|
||||
for w in words(x):
|
||||
w = 0
|
||||
|
||||
func countZeros*(x: BitSeq): int =
|
||||
x.len() - x.countOnes()
|
||||
|
||||
template bytes*(x: BitSeq): untyped =
|
||||
seq[byte](x)
|
||||
|
||||
iterator items*(x: BitArray): bool =
|
||||
for i in 0..<x.bits:
|
||||
yield x[i]
|
||||
|
||||
iterator pairs*(x: BitArray): (int, bool) =
|
||||
for i in 0..<x.bits:
|
||||
yield (i, x[i])
|
||||
|
||||
func incl*(a: var BitArray, b: BitArray) =
|
||||
# Update `a` to include the bits of `b`, as if applying `or` to each bit
|
||||
for i in 0..<a.bytes.len:
|
||||
a[i] = a[i] or b[i]
|
||||
|
||||
func clear*(a: var BitArray) =
|
||||
for b in a.bytes.mitems(): b = 0
|
||||
|
||||
# Set operations
|
||||
func `+`*(a, b: BitArray): BitArray =
|
||||
for i in 0..<a.bytes.len:
|
||||
result.bytes[i] = a.bytes[i] or b.bytes[i]
|
||||
|
||||
func `-`*(a, b: BitArray): BitArray =
|
||||
for i in 0..<a.bytes.len:
|
||||
result.bytes[i] = a.bytes[i] and (not b.bytes[i])
|
||||
|
||||
iterator oneIndices*(a: BitArray): int =
|
||||
for i in 0..<a.len:
|
||||
if a[i]: yield i
|
||||
|
||||
|
|
@ -1,218 +0,0 @@
|
|||
# nim-eth - Limited SSZ implementation
|
||||
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
||||
|
||||
{.push raises: [Defect].}
|
||||
|
||||
import
|
||||
std/[typetraits, options],
|
||||
stew/[endians2, objects],
|
||||
./types
|
||||
|
||||
template raiseIncorrectSize*(T: type) =
|
||||
const typeName = name(T)
|
||||
raise newException(MalformedSszError,
|
||||
"SSZ " & typeName & " input of incorrect size")
|
||||
|
||||
template setOutputSize[R, T](a: var array[R, T], length: int) =
|
||||
if length != a.len:
|
||||
raiseIncorrectSize a.type
|
||||
|
||||
proc setOutputSize(list: var List, length: int) {.raises: [SszError, Defect].} =
|
||||
if not list.setLen length:
|
||||
raise newException(MalformedSszError, "SSZ list maximum size exceeded")
|
||||
|
||||
# fromSszBytes copies the wire representation to a Nim variable,
|
||||
# assuming there's enough data in the buffer
|
||||
func fromSszBytes*(T: type UintN, data: openArray[byte]):
|
||||
T {.raises: [MalformedSszError, Defect].} =
|
||||
## Convert directly to bytes the size of the int. (e.g. ``uint16 = 2 bytes``)
|
||||
## All integers are serialized as **little endian**.
|
||||
if data.len != sizeof(result):
|
||||
raiseIncorrectSize T
|
||||
|
||||
T.fromBytesLE(data)
|
||||
|
||||
func fromSszBytes*(T: type bool, data: openArray[byte]):
|
||||
T {.raises: [MalformedSszError, Defect].} =
|
||||
# Strict: only allow 0 or 1
|
||||
if data.len != 1 or byte(data[0]) > byte(1):
|
||||
raise newException(MalformedSszError, "invalid boolean value")
|
||||
data[0] == 1
|
||||
|
||||
template fromSszBytes*(T: type BitSeq, bytes: openArray[byte]): auto =
|
||||
BitSeq @bytes
|
||||
|
||||
proc `[]`[T, U, V](s: openArray[T], x: HSlice[U, V]) {.error:
|
||||
"Please don't use openArray's [] as it allocates a result sequence".}
|
||||
|
||||
template checkForForbiddenBits(ResulType: type,
|
||||
input: openArray[byte],
|
||||
expectedBits: static int64) =
|
||||
## This checks if the input contains any bits set above the maximum
|
||||
## sized allowed. We only need to check the last byte to verify this:
|
||||
const bitsInLastByte = (expectedBits mod 8)
|
||||
when bitsInLastByte != 0:
|
||||
# As an example, if there are 3 bits expected in the last byte,
|
||||
# we calculate a bitmask equal to 11111000. If the input has any
|
||||
# raised bits in range of the bitmask, this would be a violation
|
||||
# of the size of the BitArray:
|
||||
const forbiddenBitsMask = byte(byte(0xff) shl bitsInLastByte)
|
||||
|
||||
if (input[^1] and forbiddenBitsMask) != 0:
|
||||
raiseIncorrectSize ResulType
|
||||
|
||||
func readSszValue*[T](input: openArray[byte], val: var T)
|
||||
{.raises: [SszError, Defect].} =
|
||||
mixin fromSszBytes, toSszType
|
||||
|
||||
template readOffsetUnchecked(n: int): uint32 {.used.}=
|
||||
fromSszBytes(uint32, input.toOpenArray(n, n + offsetSize - 1))
|
||||
|
||||
template readOffset(n: int): int {.used.} =
|
||||
let offset = readOffsetUnchecked(n)
|
||||
if offset > input.len.uint32:
|
||||
raise newException(MalformedSszError, "SSZ list element offset points past the end of the input")
|
||||
int(offset)
|
||||
|
||||
when val is BitList:
|
||||
if input.len == 0:
|
||||
raise newException(MalformedSszError, "Invalid empty SSZ BitList value")
|
||||
|
||||
# Since our BitLists have an in-memory representation that precisely
|
||||
# matches their SSZ encoding, we can deserialize them as regular Lists:
|
||||
const maxExpectedSize = (val.maxLen div 8) + 1
|
||||
type MatchingListType = List[byte, maxExpectedSize]
|
||||
|
||||
when false:
|
||||
# TODO: Nim doesn't like this simple type coercion,
|
||||
# we'll rely on `cast` for now (see below)
|
||||
readSszValue(input, MatchingListType val)
|
||||
else:
|
||||
static:
|
||||
# As a sanity check, we verify that the coercion is accepted by the compiler:
|
||||
doAssert MatchingListType(val) is MatchingListType
|
||||
readSszValue(input, cast[ptr MatchingListType](addr val)[])
|
||||
|
||||
let resultBytesCount = len bytes(val)
|
||||
|
||||
if bytes(val)[resultBytesCount - 1] == 0:
|
||||
raise newException(MalformedSszError, "SSZ BitList is not properly terminated")
|
||||
|
||||
if resultBytesCount == maxExpectedSize:
|
||||
checkForForbiddenBits(T, input, val.maxLen + 1)
|
||||
|
||||
elif val is List|array:
|
||||
type E = type val[0]
|
||||
|
||||
when E is byte:
|
||||
val.setOutputSize input.len
|
||||
if input.len > 0:
|
||||
copyMem(addr val[0], unsafeAddr input[0], input.len)
|
||||
|
||||
elif isFixedSize(E):
|
||||
const elemSize = fixedPortionSize(E)
|
||||
if input.len mod elemSize != 0:
|
||||
var ex = new SszSizeMismatchError
|
||||
ex.deserializedType = cstring typetraits.name(T)
|
||||
ex.actualSszSize = input.len
|
||||
ex.elementSize = elemSize
|
||||
raise ex
|
||||
val.setOutputSize input.len div elemSize
|
||||
for i in 0 ..< val.len:
|
||||
let offset = i * elemSize
|
||||
readSszValue(input.toOpenArray(offset, offset + elemSize - 1), val[i])
|
||||
|
||||
else:
|
||||
if input.len == 0:
|
||||
# This is an empty list.
|
||||
# The default initialization of the return value is fine.
|
||||
val.setOutputSize 0
|
||||
return
|
||||
elif input.len < offsetSize:
|
||||
raise newException(MalformedSszError, "SSZ input of insufficient size")
|
||||
|
||||
var offset = readOffset 0
|
||||
let resultLen = offset div offsetSize
|
||||
|
||||
if resultLen == 0:
|
||||
# If there are too many elements, other constraints detect problems
|
||||
# (not monotonically increasing, past end of input, or last element
|
||||
# not matching up with its nextOffset properly)
|
||||
raise newException(MalformedSszError, "SSZ list incorrectly encoded of zero length")
|
||||
|
||||
val.setOutputSize resultLen
|
||||
for i in 1 ..< resultLen:
|
||||
let nextOffset = readOffset(i * offsetSize)
|
||||
if nextOffset <= offset:
|
||||
raise newException(MalformedSszError, "SSZ list element offsets are not monotonically increasing")
|
||||
else:
|
||||
readSszValue(input.toOpenArray(offset, nextOffset - 1), val[i - 1])
|
||||
offset = nextOffset
|
||||
|
||||
readSszValue(input.toOpenArray(offset, input.len - 1), val[resultLen - 1])
|
||||
|
||||
elif val is UintN|bool:
|
||||
val = fromSszBytes(T, input)
|
||||
|
||||
elif val is BitArray:
|
||||
if sizeof(val) != input.len:
|
||||
raiseIncorrectSize(T)
|
||||
checkForForbiddenBits(T, input, val.bits)
|
||||
copyMem(addr val.bytes[0], unsafeAddr input[0], input.len)
|
||||
|
||||
elif val is object|tuple:
|
||||
let inputLen = uint32 input.len
|
||||
const minimallyExpectedSize = uint32 fixedPortionSize(T)
|
||||
|
||||
if inputLen < minimallyExpectedSize:
|
||||
raise newException(MalformedSszError, "SSZ input of insufficient size")
|
||||
|
||||
enumInstanceSerializedFields(val, fieldName, field):
|
||||
const boundingOffsets = getFieldBoundingOffsets(T, fieldName)
|
||||
|
||||
# type FieldType = type field # buggy
|
||||
# For some reason, Nim gets confused about the alias here. This could be a
|
||||
# generics caching issue caused by the use of distinct types. Such an
|
||||
# issue is very scary in general.
|
||||
# The bug can be seen with the two List[uint64, N] types that exist in
|
||||
# the spec, with different N.
|
||||
|
||||
type SszType = type toSszType(declval type(field))
|
||||
|
||||
when isFixedSize(SszType):
|
||||
const
|
||||
startOffset = boundingOffsets[0]
|
||||
endOffset = boundingOffsets[1]
|
||||
else:
|
||||
let
|
||||
startOffset = readOffsetUnchecked(boundingOffsets[0])
|
||||
endOffset = if boundingOffsets[1] == -1: inputLen
|
||||
else: readOffsetUnchecked(boundingOffsets[1])
|
||||
|
||||
when boundingOffsets.isFirstOffset:
|
||||
if startOffset != minimallyExpectedSize:
|
||||
raise newException(MalformedSszError, "SSZ object dynamic portion starts at invalid offset")
|
||||
|
||||
if startOffset > endOffset:
|
||||
raise newException(MalformedSszError, "SSZ field offsets are not monotonically increasing")
|
||||
elif endOffset > inputLen:
|
||||
raise newException(MalformedSszError, "SSZ field offset points past the end of the input")
|
||||
elif startOffset < minimallyExpectedSize:
|
||||
raise newException(MalformedSszError, "SSZ field offset points outside bounding offsets")
|
||||
|
||||
# TODO The extra type escaping here is a work-around for a Nim issue:
|
||||
when type(field) is type(SszType):
|
||||
readSszValue(
|
||||
input.toOpenArray(int(startOffset), int(endOffset - 1)),
|
||||
field)
|
||||
else:
|
||||
field = fromSszBytes(
|
||||
type(field),
|
||||
input.toOpenArray(int(startOffset), int(endOffset - 1)))
|
||||
|
||||
else:
|
||||
unsupported T
|
||||
|
|
@ -1,111 +0,0 @@
|
|||
{.push raises: [Defect].}
|
||||
|
||||
import
|
||||
math, sequtils, ssz_serialization, options, algorithm,
|
||||
nimcrypto/hash,
|
||||
../common/eth_types, ./types, ./merkleization
|
||||
|
||||
const maxTreeDepth: uint64 = 32
|
||||
const empty: seq[Digest] = @[]
|
||||
|
||||
type
|
||||
MerkleNodeType = enum
|
||||
LeafType,
|
||||
NodeType,
|
||||
ZeroType
|
||||
|
||||
MerkleNode = ref object
|
||||
case kind: MerkleNodeType
|
||||
of LeafType:
|
||||
digest: Digest
|
||||
of NodeType:
|
||||
innerDigest: Digest
|
||||
left: MerkleNode
|
||||
right: MerkleNode
|
||||
of ZeroType:
|
||||
depth: uint64
|
||||
|
||||
func zeroNodes(): seq[MerkleNode] =
|
||||
var nodes = newSeq[MerkleNode]()
|
||||
for i in 0..maxTreeDepth:
|
||||
nodes.add(MerkleNode(kind: ZeroType, depth: i))
|
||||
return nodes
|
||||
|
||||
let zNodes = zeroNodes()
|
||||
|
||||
# This look like something that should be in standard lib.
|
||||
func splitAt[T](s: openArray[T], idx: uint64): (seq[T], seq[T]) =
|
||||
var lSeq = newSeq[T]()
|
||||
var rSeq = newSeq[T]()
|
||||
for i, e in s:
|
||||
if (uint64(i) < idx):
|
||||
lSeq.add(e)
|
||||
else:
|
||||
rSeq.add(e)
|
||||
(lSeq, rSeq)
|
||||
|
||||
func splitLeaves(l: openArray[Digest], cap: uint64): (seq[Digest], seq[Digest]) =
|
||||
if (uint64(len(l)) <= cap):
|
||||
(l.toSeq(), empty)
|
||||
else:
|
||||
splitAt(l, cap)
|
||||
|
||||
proc getSubTrees(node: MerkleNode): Option[(MerkleNode, MerkleNode)] =
|
||||
case node.kind
|
||||
of LeafType:
|
||||
return none[(MerkleNode, MerkleNode)]()
|
||||
of NodeType:
|
||||
return some((node.left, node.right))
|
||||
of ZeroType:
|
||||
if node.depth == 0:
|
||||
return none[(MerkleNode, MerkleNode)]()
|
||||
else:
|
||||
return some((zNodes[node.depth - 1], zNodes[node.depth - 1]))
|
||||
|
||||
func hash*(node: MerkleNode): Digest =
|
||||
case node.kind
|
||||
of LeafType:
|
||||
node.digest
|
||||
of NodeType:
|
||||
node.innerDigest
|
||||
of ZeroType:
|
||||
zeroHashes[node.depth]
|
||||
|
||||
func getCapacityAtDepth(depth: uint64): uint64 =
|
||||
uint64 math.pow(2'f64, float64 depth)
|
||||
|
||||
func createTree*(leaves: openArray[Digest], depth: uint64): MerkleNode =
|
||||
if len(leaves) == 0:
|
||||
return MerkleNode(kind: ZeroType, depth: depth)
|
||||
elif depth == 0:
|
||||
return MerkleNode(kind: LeafType, digest: leaves[0])
|
||||
else:
|
||||
let nexLevelDepth = depth - 1
|
||||
let subCap = getCapacityAtDepth(nexLevelDepth)
|
||||
let (left, right) = splitLeaves(leaves, subCap)
|
||||
let leftTree = createTree(left, nexLevelDepth)
|
||||
let rightTree = createTree(right, nexLevelDepth)
|
||||
let finalHash = mergeBranches(leftTree.hash(), rightTree.hash())
|
||||
return MerkleNode(kind: NodeType, innerDigest: finalHash, left: leftTree, right: rightTree)
|
||||
|
||||
proc genProof*(tree: MerkleNode, idx: uint64, treeDepth: uint64): seq[Digest] =
|
||||
var proof = newSeq[Digest]()
|
||||
var currNode = tree
|
||||
var currDepth = treeDepth
|
||||
while currDepth > 0:
|
||||
let ithBit = (idx shr (currDepth - 1)) and 1
|
||||
# should be safe to call unsafeGet() as leaves are on lowest level, and depth is
|
||||
# always larger than 0
|
||||
let (left, right) = getSubTrees(currNode).unsafeGet()
|
||||
if ithBit == 1:
|
||||
proof.add(left.hash())
|
||||
currNode = right
|
||||
else:
|
||||
proof.add(right.hash())
|
||||
currNode = left
|
||||
currDepth = currDepth - 1
|
||||
|
||||
proof.reverse()
|
||||
proof
|
||||
|
||||
# TODO add method to add leaf to the exisiting tree
|
||||
|
|
@ -1,660 +0,0 @@
|
|||
# ssz_serialization
|
||||
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
||||
|
||||
# This module contains the parts necessary to create a merkle hash from the core
|
||||
# SSZ types outlined in the spec:
|
||||
# https://github.com/ethereum/consensus-specs/blob/v1.0.1/ssz/simple-serialize.md#merkleization
|
||||
|
||||
{.push raises: [Defect].}
|
||||
|
||||
import
|
||||
math, sequtils,
|
||||
stew/[bitops2, endians2, ptrops],
|
||||
stew/ranges/ptr_arith, nimcrypto/[hash, sha2],
|
||||
serialization/testing/tracing,
|
||||
"."/[bitseqs, types]
|
||||
|
||||
export
|
||||
types
|
||||
|
||||
when hasSerializationTracing:
|
||||
import stew/byteutils, typetraits
|
||||
|
||||
const
|
||||
zero64 = default array[64, byte]
|
||||
bitsPerChunk = bytesPerChunk * 8
|
||||
|
||||
func binaryTreeHeight*(totalElements: Limit): int =
|
||||
bitWidth nextPow2(uint64 totalElements)
|
||||
|
||||
type
|
||||
SszMerkleizerImpl = object
|
||||
combinedChunks: ptr UncheckedArray[Digest]
|
||||
totalChunks: uint64
|
||||
topIndex: int
|
||||
|
||||
SszMerkleizer*[limit: static[Limit]] = object
|
||||
combinedChunks: ref array[binaryTreeHeight limit, Digest]
|
||||
impl: SszMerkleizerImpl
|
||||
|
||||
template chunks*(m: SszMerkleizerImpl): openArray[Digest] =
|
||||
m.combinedChunks.toOpenArray(0, m.topIndex)
|
||||
|
||||
template getChunkCount*(m: SszMerkleizer): uint64 =
|
||||
m.impl.totalChunks
|
||||
|
||||
template getCombinedChunks*(m: SszMerkleizer): openArray[Digest] =
|
||||
toOpenArray(m.impl.combinedChunks, 0, m.impl.topIndex)
|
||||
|
||||
type DigestCtx* = sha2.sha256
|
||||
|
||||
template computeDigest*(body: untyped): Digest =
|
||||
## This little helper will init the hash function and return the sliced
|
||||
## hash:
|
||||
## let hashOfData = withHash: h.update(data)
|
||||
when nimvm:
|
||||
# In SSZ, computeZeroHashes require compile-time SHA256
|
||||
block:
|
||||
var h {.inject.}: sha256
|
||||
init(h)
|
||||
body
|
||||
finish(h)
|
||||
else:
|
||||
block:
|
||||
var h {.inject, noInit.}: DigestCtx
|
||||
init(h)
|
||||
body
|
||||
finish(h)
|
||||
|
||||
func digest(a: openArray[byte]): Digest =
|
||||
result = computeDigest:
|
||||
h.update(a)
|
||||
|
||||
func digest(a, b: openArray[byte]): Digest =
|
||||
result = computeDigest:
|
||||
trs "DIGESTING ARRAYS ", toHex(a), " ", toHex(b)
|
||||
trs toHex(a)
|
||||
trs toHex(b)
|
||||
|
||||
h.update a
|
||||
h.update b
|
||||
trs "HASH RESULT ", result
|
||||
|
||||
func digest(a, b, c: openArray[byte]): Digest =
|
||||
result = computeDigest:
|
||||
trs "DIGESTING ARRAYS ", toHex(a), " ", toHex(b), " ", toHex(c)
|
||||
|
||||
h.update a
|
||||
h.update b
|
||||
h.update c
|
||||
trs "HASH RESULT ", result
|
||||
|
||||
func mergeBranches(existing: Digest, newData: openArray[byte]): Digest =
|
||||
trs "MERGING BRANCHES OPEN ARRAY"
|
||||
|
||||
let paddingBytes = bytesPerChunk - newData.len
|
||||
digest(existing.data, newData, zero64.toOpenArray(0, paddingBytes - 1))
|
||||
|
||||
template mergeBranches(existing: Digest, newData: array[32, byte]): Digest =
|
||||
trs "MERGING BRANCHES ARRAY"
|
||||
digest(existing.data, newData)
|
||||
|
||||
template mergeBranches*(a, b: Digest): Digest =
|
||||
trs "MERGING BRANCHES DIGEST"
|
||||
digest(a.data, b.data)
|
||||
|
||||
func computeZeroHashes: array[sizeof(Limit) * 8, Digest] =
|
||||
result[0] = Digest()
|
||||
for i in 1 .. result.high:
|
||||
result[i] = mergeBranches(result[i - 1], result[i - 1])
|
||||
|
||||
const zeroHashes* = computeZeroHashes()
|
||||
|
||||
func addChunk*(merkleizer: var SszMerkleizerImpl, data: openArray[byte]) =
|
||||
doAssert data.len > 0 and data.len <= bytesPerChunk
|
||||
|
||||
if getBitLE(merkleizer.totalChunks, 0):
|
||||
var hash = mergeBranches(merkleizer.combinedChunks[0], data)
|
||||
|
||||
for i in 1 .. merkleizer.topIndex:
|
||||
trs "ITERATING"
|
||||
if getBitLE(merkleizer.totalChunks, i):
|
||||
trs "CALLING MERGE BRANCHES"
|
||||
hash = mergeBranches(merkleizer.combinedChunks[i], hash)
|
||||
else:
|
||||
trs "WRITING FRESH CHUNK AT ", i, " = ", hash
|
||||
merkleizer.combinedChunks[i] = hash
|
||||
break
|
||||
else:
|
||||
let paddingBytes = bytesPerChunk - data.len
|
||||
|
||||
merkleizer.combinedChunks[0].data[0..<data.len] = data
|
||||
merkleizer.combinedChunks[0].data[data.len..<bytesPerChunk] =
|
||||
zero64.toOpenArray(0, paddingBytes - 1)
|
||||
|
||||
trs "WROTE BASE CHUNK ",
|
||||
toHex(merkleizer.combinedChunks[0].data), " ", data.len
|
||||
|
||||
inc merkleizer.totalChunks
|
||||
|
||||
template isOdd(x: SomeNumber): bool =
|
||||
(x and 1) != 0
|
||||
|
||||
func addChunkAndGenMerkleProof*(merkleizer: var SszMerkleizerImpl,
|
||||
hash: Digest,
|
||||
outProof: var openArray[Digest]) =
|
||||
var
|
||||
hashWrittenToMerkleizer = false
|
||||
hash = hash
|
||||
|
||||
doAssert merkleizer.topIndex < outProof.len
|
||||
|
||||
for level in 0 .. merkleizer.topIndex:
|
||||
if getBitLE(merkleizer.totalChunks, level):
|
||||
outProof[level] = merkleizer.combinedChunks[level]
|
||||
hash = mergeBranches(merkleizer.combinedChunks[level], hash)
|
||||
else:
|
||||
if not hashWrittenToMerkleizer:
|
||||
merkleizer.combinedChunks[level] = hash
|
||||
hashWrittenToMerkleizer = true
|
||||
outProof[level] = zeroHashes[level]
|
||||
hash = mergeBranches(hash, zeroHashes[level])
|
||||
|
||||
merkleizer.totalChunks += 1
|
||||
|
||||
func completeStartedChunk(merkleizer: var SszMerkleizerImpl,
|
||||
hash: Digest, atLevel: int) =
|
||||
when false:
|
||||
let
|
||||
insertedChunksCount = 1'u64 shl (atLevel - 1)
|
||||
chunksStateMask = (insertedChunksCount shl 1) - 1
|
||||
doAssert (merkleizer.totalChunks and chunksStateMask) == insertedChunksCount
|
||||
|
||||
var hash = hash
|
||||
for i in atLevel .. merkleizer.topIndex:
|
||||
if getBitLE(merkleizer.totalChunks, i):
|
||||
hash = mergeBranches(merkleizer.combinedChunks[i], hash)
|
||||
else:
|
||||
merkleizer.combinedChunks[i] = hash
|
||||
break
|
||||
|
||||
func addChunksAndGenMerkleProofs*(merkleizer: var SszMerkleizerImpl,
|
||||
chunks: openArray[Digest]): seq[Digest] =
|
||||
doAssert chunks.len > 0 and merkleizer.topIndex > 0
|
||||
|
||||
let proofHeight = merkleizer.topIndex + 1
|
||||
result = newSeq[Digest](chunks.len * proofHeight)
|
||||
|
||||
if chunks.len == 1:
|
||||
merkleizer.addChunkAndGenMerkleProof(chunks[0], result)
|
||||
return
|
||||
|
||||
let newTotalChunks = merkleizer.totalChunks + chunks.len.uint64
|
||||
|
||||
var
|
||||
# A perfect binary tree will take either `chunks.len * 2` values if the
|
||||
# number of elements in the base layer is odd and `chunks.len * 2 - 1`
|
||||
# otherwise. Each row may also need a single extra element at most if
|
||||
# it must be combined with the existing values in the Merkleizer:
|
||||
merkleTree = newSeqOfCap[Digest](chunks.len + merkleizer.topIndex)
|
||||
inRowIdx = merkleizer.totalChunks
|
||||
postUpdateInRowIdx = newTotalChunks
|
||||
zeroMixed = false
|
||||
|
||||
template writeResult(chunkIdx, level: int, chunk: Digest) =
|
||||
result[chunkIdx * proofHeight + level] = chunk
|
||||
|
||||
# We'll start by generating the first row of the merkle tree.
|
||||
var currPairEnd = if inRowIdx.isOdd:
|
||||
# an odd chunk number means that we must combine the
|
||||
# hash with the existing pending sibling hash in the
|
||||
# merkleizer.
|
||||
writeResult(0, 0, merkleizer.combinedChunks[0])
|
||||
merkleTree.add mergeBranches(merkleizer.combinedChunks[0], chunks[0])
|
||||
|
||||
# TODO: can we immediately write this out?
|
||||
merkleizer.completeStartedChunk(merkleTree[^1], 1)
|
||||
2
|
||||
else:
|
||||
1
|
||||
|
||||
if postUpdateInRowIdx.isOdd:
|
||||
merkleizer.combinedChunks[0] = chunks[^1]
|
||||
|
||||
while currPairEnd < chunks.len:
|
||||
writeResult(currPairEnd - 1, 0, chunks[currPairEnd])
|
||||
writeResult(currPairEnd, 0, chunks[currPairEnd - 1])
|
||||
merkleTree.add mergeBranches(chunks[currPairEnd - 1],
|
||||
chunks[currPairEnd])
|
||||
currPairEnd += 2
|
||||
|
||||
if currPairEnd - 1 < chunks.len:
|
||||
zeroMixed = true
|
||||
writeResult(currPairEnd - 1, 0, zeroHashes[0])
|
||||
merkleTree.add mergeBranches(chunks[currPairEnd - 1],
|
||||
zeroHashes[0])
|
||||
var
|
||||
level = 0
|
||||
baseChunksPerElement = 1
|
||||
treeRowStart = 0
|
||||
rowLen = merkleTree.len
|
||||
|
||||
template writeProofs(rowChunkIdx: int, hash: Digest) =
|
||||
let
|
||||
startAbsIdx = (inRowIdx.int + rowChunkIdx) * baseChunksPerElement
|
||||
endAbsIdx = startAbsIdx + baseChunksPerElement
|
||||
startResIdx = max(startAbsIdx - merkleizer.totalChunks.int, 0)
|
||||
endResIdx = min(endAbsIdx - merkleizer.totalChunks.int, chunks.len)
|
||||
|
||||
for resultPos in startResIdx ..< endResIdx:
|
||||
writeResult(resultPos, level, hash)
|
||||
|
||||
if rowLen > 1:
|
||||
while level < merkleizer.topIndex:
|
||||
inc level
|
||||
baseChunksPerElement *= 2
|
||||
inRowIdx = inRowIdx div 2
|
||||
postUpdateInRowIdx = postUpdateInRowIdx div 2
|
||||
|
||||
var currPairEnd = if inRowIdx.isOdd:
|
||||
# an odd chunk number means that we must combine the
|
||||
# hash with the existing pending sibling hash in the
|
||||
# merkleizer.
|
||||
writeProofs(0, merkleizer.combinedChunks[level])
|
||||
merkleTree.add mergeBranches(merkleizer.combinedChunks[level],
|
||||
merkleTree[treeRowStart])
|
||||
|
||||
# TODO: can we immediately write this out?
|
||||
merkleizer.completeStartedChunk(merkleTree[^1], level + 1)
|
||||
2
|
||||
else:
|
||||
1
|
||||
|
||||
if postUpdateInRowIdx.isOdd:
|
||||
merkleizer.combinedChunks[level] = merkleTree[treeRowStart + rowLen -
|
||||
ord(zeroMixed) - 1]
|
||||
while currPairEnd < rowLen:
|
||||
writeProofs(currPairEnd - 1, merkleTree[treeRowStart + currPairEnd])
|
||||
writeProofs(currPairEnd, merkleTree[treeRowStart + currPairEnd - 1])
|
||||
merkleTree.add mergeBranches(merkleTree[treeRowStart + currPairEnd - 1],
|
||||
merkleTree[treeRowStart + currPairEnd])
|
||||
currPairEnd += 2
|
||||
|
||||
if currPairEnd - 1 < rowLen:
|
||||
zeroMixed = true
|
||||
writeProofs(currPairEnd - 1, zeroHashes[level])
|
||||
merkleTree.add mergeBranches(merkleTree[treeRowStart + currPairEnd - 1],
|
||||
zeroHashes[level])
|
||||
|
||||
treeRowStart += rowLen
|
||||
rowLen = merkleTree.len - treeRowStart
|
||||
|
||||
if rowLen == 1:
|
||||
break
|
||||
|
||||
doAssert rowLen == 1
|
||||
|
||||
if (inRowIdx and 2) != 0:
|
||||
merkleizer.completeStartedChunk(
|
||||
mergeBranches(merkleizer.combinedChunks[level + 1], merkleTree[^1]),
|
||||
level + 2)
|
||||
|
||||
if (not zeroMixed) and (postUpdateInRowIdx and 2) != 0:
|
||||
merkleizer.combinedChunks[level + 1] = merkleTree[^1]
|
||||
|
||||
while level < merkleizer.topIndex:
|
||||
inc level
|
||||
baseChunksPerElement *= 2
|
||||
inRowIdx = inRowIdx div 2
|
||||
|
||||
let hash = if getBitLE(merkleizer.totalChunks, level):
|
||||
merkleizer.combinedChunks[level]
|
||||
else:
|
||||
zeroHashes[level]
|
||||
|
||||
writeProofs(0, hash)
|
||||
|
||||
merkleizer.totalChunks = newTotalChunks
|
||||
|
||||
proc init*(S: type SszMerkleizer): S =
|
||||
new result.combinedChunks
|
||||
result.impl = SszMerkleizerImpl(
|
||||
combinedChunks: cast[ptr UncheckedArray[Digest]](
|
||||
addr result.combinedChunks[][0]),
|
||||
topIndex: binaryTreeHeight(result.limit) - 1,
|
||||
totalChunks: 0)
|
||||
|
||||
proc init*(S: type SszMerkleizer,
|
||||
combinedChunks: openArray[Digest],
|
||||
totalChunks: uint64): S =
|
||||
new result.combinedChunks
|
||||
result.combinedChunks[][0 ..< combinedChunks.len] = combinedChunks
|
||||
result.impl = SszMerkleizerImpl(
|
||||
combinedChunks: cast[ptr UncheckedArray[Digest]](
|
||||
addr result.combinedChunks[][0]),
|
||||
topIndex: binaryTreeHeight(result.limit) - 1,
|
||||
totalChunks: totalChunks)
|
||||
|
||||
proc copy*[L: static[Limit]](cloned: SszMerkleizer[L]): SszMerkleizer[L] =
|
||||
new result.combinedChunks
|
||||
result.combinedChunks[] = cloned.combinedChunks[]
|
||||
result.impl = SszMerkleizerImpl(
|
||||
combinedChunks: cast[ptr UncheckedArray[Digest]](
|
||||
addr result.combinedChunks[][0]),
|
||||
topIndex: binaryTreeHeight(L) - 1,
|
||||
totalChunks: cloned.totalChunks)
|
||||
|
||||
template addChunksAndGenMerkleProofs*(
|
||||
merkleizer: var SszMerkleizer,
|
||||
chunks: openArray[Digest]): seq[Digest] =
|
||||
addChunksAndGenMerkleProofs(merkleizer.impl, chunks)
|
||||
|
||||
template addChunk*(merkleizer: var SszMerkleizer, data: openArray[byte]) =
|
||||
addChunk(merkleizer.impl, data)
|
||||
|
||||
template totalChunks*(merkleizer: SszMerkleizer): uint64 =
|
||||
merkleizer.impl.totalChunks
|
||||
|
||||
template getFinalHash*(merkleizer: SszMerkleizer): Digest =
|
||||
merkleizer.impl.getFinalHash
|
||||
|
||||
template createMerkleizer*(totalElements: static Limit): SszMerkleizerImpl =
|
||||
trs "CREATING A MERKLEIZER FOR ", totalElements
|
||||
|
||||
const treeHeight = binaryTreeHeight totalElements
|
||||
var combinedChunks {.noInit.}: array[treeHeight, Digest]
|
||||
|
||||
let topIndex = treeHeight - 1
|
||||
|
||||
SszMerkleizerImpl(
|
||||
combinedChunks: cast[ptr UncheckedArray[Digest]](addr combinedChunks),
|
||||
topIndex: if (topIndex < 0): 0 else: topIndex,
|
||||
totalChunks: 0)
|
||||
|
||||
func getFinalHash*(merkleizer: SszMerkleizerImpl): Digest =
|
||||
if merkleizer.totalChunks == 0:
|
||||
return zeroHashes[merkleizer.topIndex]
|
||||
|
||||
let
|
||||
bottomHashIdx = firstOne(merkleizer.totalChunks) - 1
|
||||
submittedChunksHeight = bitWidth(merkleizer.totalChunks - 1)
|
||||
topHashIdx = merkleizer.topIndex
|
||||
|
||||
trs "BOTTOM HASH ", bottomHashIdx
|
||||
trs "SUBMITTED HEIGHT ", submittedChunksHeight
|
||||
trs "TOP HASH IDX ", topHashIdx
|
||||
|
||||
if bottomHashIdx != submittedChunksHeight:
|
||||
# Our tree is not finished. We must complete the work in progress
|
||||
# branches and then extend the tree to the right height.
|
||||
result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx],
|
||||
zeroHashes[bottomHashIdx])
|
||||
|
||||
for i in bottomHashIdx + 1 ..< topHashIdx:
|
||||
if getBitLE(merkleizer.totalChunks, i):
|
||||
result = mergeBranches(merkleizer.combinedChunks[i], result)
|
||||
trs "COMBINED"
|
||||
else:
|
||||
result = mergeBranches(result, zeroHashes[i])
|
||||
trs "COMBINED WITH ZERO"
|
||||
|
||||
elif bottomHashIdx == topHashIdx:
|
||||
# We have a perfect tree (chunks == 2**n) at just the right height!
|
||||
result = merkleizer.combinedChunks[bottomHashIdx]
|
||||
else:
|
||||
# We have a perfect tree of user chunks, but we have more work to
|
||||
# do - we must extend it to reach the desired height
|
||||
result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx],
|
||||
zeroHashes[bottomHashIdx])
|
||||
|
||||
for i in bottomHashIdx + 1 ..< topHashIdx:
|
||||
result = mergeBranches(result, zeroHashes[i])
|
||||
|
||||
func mixInLength*(root: Digest, length: int): Digest =
|
||||
var dataLen: array[32, byte]
|
||||
dataLen[0..<8] = uint64(length).toBytesLE()
|
||||
mergeBranches(root, dataLen)
|
||||
|
||||
func hash_tree_root*(x: auto): Digest {.gcsafe, raises: [Defect].}
|
||||
|
||||
template merkleizeFields(totalElements: static Limit, body: untyped): Digest =
|
||||
var merkleizer {.inject.} = createMerkleizer(totalElements)
|
||||
|
||||
template addField(field) =
|
||||
let hash = hash_tree_root(field)
|
||||
trs "MERKLEIZING FIELD ", astToStr(field), " = ", hash
|
||||
addChunk(merkleizer, hash.data)
|
||||
trs "CHUNK ADDED"
|
||||
|
||||
body
|
||||
|
||||
getFinalHash(merkleizer)
|
||||
|
||||
template writeBytesLE(chunk: var array[bytesPerChunk, byte], atParam: int,
|
||||
val: SomeUnsignedInt) =
|
||||
let at = atParam
|
||||
chunk[at ..< at + sizeof(val)] = toBytesLE(val)
|
||||
|
||||
func chunkedHashTreeRootForBasicTypes[T](merkleizer: var SszMerkleizerImpl,
|
||||
arr: openArray[T]): Digest =
|
||||
static:
|
||||
doAssert T is BasicType
|
||||
doAssert bytesPerChunk mod sizeof(T) == 0
|
||||
|
||||
if arr.len == 0:
|
||||
return getFinalHash(merkleizer)
|
||||
|
||||
when sizeof(T) == 1 or cpuEndian == littleEndian:
|
||||
var
|
||||
remainingBytes = when sizeof(T) == 1: arr.len
|
||||
else: arr.len * sizeof(T)
|
||||
pos = cast[ptr byte](unsafeAddr arr[0])
|
||||
|
||||
while remainingBytes >= bytesPerChunk:
|
||||
merkleizer.addChunk(makeOpenArray(pos, bytesPerChunk))
|
||||
pos = offset(pos, bytesPerChunk)
|
||||
remainingBytes -= bytesPerChunk
|
||||
|
||||
if remainingBytes > 0:
|
||||
merkleizer.addChunk(makeOpenArray(pos, remainingBytes))
|
||||
|
||||
else:
|
||||
const valuesPerChunk = bytesPerChunk div sizeof(T)
|
||||
|
||||
var writtenValues = 0
|
||||
|
||||
var chunk: array[bytesPerChunk, byte]
|
||||
while writtenValues < arr.len - valuesPerChunk:
|
||||
for i in 0 ..< valuesPerChunk:
|
||||
chunk.writeBytesLE(i * sizeof(T), arr[writtenValues + i])
|
||||
merkleizer.addChunk chunk
|
||||
inc writtenValues, valuesPerChunk
|
||||
|
||||
let remainingValues = arr.len - writtenValues
|
||||
if remainingValues > 0:
|
||||
var lastChunk: array[bytesPerChunk, byte]
|
||||
for i in 0 ..< remainingValues:
|
||||
lastChunk.writeBytesLE(i * sizeof(T), arr[writtenValues + i])
|
||||
merkleizer.addChunk lastChunk
|
||||
|
||||
getFinalHash(merkleizer)
|
||||
|
||||
func bitListHashTreeRoot(merkleizer: var SszMerkleizerImpl, x: BitSeq): Digest =
|
||||
# TODO: Switch to a simpler BitList representation and
|
||||
# replace this with `chunkedHashTreeRoot`
|
||||
var
|
||||
totalBytes = bytes(x).len
|
||||
lastCorrectedByte = bytes(x)[^1]
|
||||
|
||||
if lastCorrectedByte == byte(1):
|
||||
if totalBytes == 1:
|
||||
# This is an empty bit list.
|
||||
# It should be hashed as a tree containing all zeros:
|
||||
return mergeBranches(zeroHashes[merkleizer.topIndex],
|
||||
zeroHashes[0]) # this is the mixed length
|
||||
|
||||
totalBytes -= 1
|
||||
lastCorrectedByte = bytes(x)[^2]
|
||||
else:
|
||||
let markerPos = log2trunc(lastCorrectedByte)
|
||||
lastCorrectedByte.clearBit(markerPos)
|
||||
|
||||
var
|
||||
bytesInLastChunk = totalBytes mod bytesPerChunk
|
||||
fullChunks = totalBytes div bytesPerChunk
|
||||
|
||||
if bytesInLastChunk == 0:
|
||||
fullChunks -= 1
|
||||
bytesInLastChunk = 32
|
||||
|
||||
for i in 0 ..< fullChunks:
|
||||
let
|
||||
chunkStartPos = i * bytesPerChunk
|
||||
chunkEndPos = chunkStartPos + bytesPerChunk - 1
|
||||
|
||||
merkleizer.addChunk bytes(x).toOpenArray(chunkStartPos, chunkEndPos)
|
||||
|
||||
var
|
||||
lastChunk: array[bytesPerChunk, byte]
|
||||
chunkStartPos = fullChunks * bytesPerChunk
|
||||
|
||||
for i in 0 .. bytesInLastChunk - 2:
|
||||
lastChunk[i] = bytes(x)[chunkStartPos + i]
|
||||
|
||||
lastChunk[bytesInLastChunk - 1] = lastCorrectedByte
|
||||
|
||||
merkleizer.addChunk lastChunk.toOpenArray(0, bytesInLastChunk - 1)
|
||||
let contentsHash = merkleizer.getFinalHash
|
||||
mixInLength contentsHash, x.len
|
||||
|
||||
func maxChunksCount(T: type, maxLen: Limit): Limit =
|
||||
when T is BitList|BitArray:
|
||||
(maxLen + bitsPerChunk - 1) div bitsPerChunk
|
||||
elif T is array|List:
|
||||
maxChunkIdx(ElemType(T), maxLen)
|
||||
else:
|
||||
unsupported T # This should never happen
|
||||
|
||||
func hashTreeRootAux[T](x: T): Digest =
|
||||
when T is bool|char:
|
||||
result.data[0] = byte(x)
|
||||
elif T is SomeUnsignedInt:
|
||||
when cpuEndian == bigEndian:
|
||||
result.data[0..<sizeof(x)] = toBytesLE(x)
|
||||
else:
|
||||
copyMem(addr result.data[0], unsafeAddr x, sizeof x)
|
||||
elif (when T is array: ElemType(T) is BasicType else: false):
|
||||
type E = ElemType(T)
|
||||
when sizeof(T) <= sizeof(result.data):
|
||||
when E is byte|bool or cpuEndian == littleEndian:
|
||||
copyMem(addr result.data[0], unsafeAddr x, sizeof x)
|
||||
else:
|
||||
var pos = 0
|
||||
for e in x:
|
||||
writeBytesLE(result.data, pos, e)
|
||||
pos += sizeof(E)
|
||||
else:
|
||||
trs "FIXED TYPE; USE CHUNK STREAM"
|
||||
var merkleizer = createMerkleizer(maxChunksCount(T, Limit x.len))
|
||||
chunkedHashTreeRootForBasicTypes(merkleizer, x)
|
||||
elif T is BitArray:
|
||||
hashTreeRootAux(x.bytes)
|
||||
elif T is array|object|tuple:
|
||||
trs "MERKLEIZING FIELDS"
|
||||
const totalFields = when T is array: len(x)
|
||||
else: totalSerializedFields(T)
|
||||
merkleizeFields(Limit totalFields):
|
||||
x.enumerateSubFields(f):
|
||||
addField f
|
||||
#elif isCaseObject(T):
|
||||
# # TODO implement this
|
||||
else:
|
||||
unsupported T
|
||||
|
||||
func hashTreeRootList(x: List|BitList): Digest =
|
||||
const maxLen = static(x.maxLen)
|
||||
type T = type(x)
|
||||
const limit = maxChunksCount(T, maxLen)
|
||||
var merkleizer = createMerkleizer(limit)
|
||||
|
||||
when x is BitList:
|
||||
merkleizer.bitListHashTreeRoot(BitSeq x)
|
||||
else:
|
||||
type E = ElemType(T)
|
||||
let contentsHash = when E is BasicType:
|
||||
chunkedHashTreeRootForBasicTypes(merkleizer, asSeq x)
|
||||
else:
|
||||
for elem in x:
|
||||
let elemHash = hash_tree_root(elem)
|
||||
merkleizer.addChunk(elemHash.data)
|
||||
merkleizer.getFinalHash()
|
||||
mixInLength(contentsHash, x.len)
|
||||
|
||||
func hash_tree_root*(x: auto): Digest {.raises: [Defect].} =
|
||||
trs "STARTING HASH TREE ROOT FOR TYPE ", name(type(x))
|
||||
mixin toSszType
|
||||
|
||||
result =
|
||||
when x is List|BitList:
|
||||
hashTreeRootList(x)
|
||||
else:
|
||||
hashTreeRootAux toSszType(x)
|
||||
|
||||
trs "HASH TREE ROOT FOR ", name(type x), " = ", "0x", $result
|
||||
|
||||
# https://github.com/ethereum/consensus-specs/blob/dev/ssz/merkle-proofs.md#get_generalized_index_length
|
||||
func getGeneralizedIndexLength(x: uint64): int =
|
||||
log2trunc(x)
|
||||
|
||||
# https://github.com/ethereum/consensus-specs/blob/dev/ssz/merkle-proofs.md#get_generalized_index_bit
|
||||
func getGeneralizedIndexBit(index: uint64, position: uint64): bool =
|
||||
(index and (1'u64 shl position)) > 0
|
||||
|
||||
# validates merkle proof. Provided index should be a generalized index of leaf node
|
||||
# as defined in: https://github.com/ethereum/consensus-specs/blob/dev/ssz/merkle-proofs.md#generalized-merkle-tree-index
|
||||
func isValidProof*(leaf: Digest, proof: openArray[Digest],
|
||||
index: uint64, root: Digest): bool =
|
||||
if len(proof) == getGeneralizedIndexLength(index):
|
||||
var
|
||||
value = leaf
|
||||
|
||||
for i, digest in proof:
|
||||
value =
|
||||
if getGeneralizedIndexBit(index, uint64 i):
|
||||
mergeBranches(digest, value)
|
||||
else:
|
||||
mergeBranches(value, digest)
|
||||
|
||||
value == root
|
||||
else:
|
||||
false
|
||||
|
||||
proc slice[T](x: openArray[T]): seq[T] = x.toSeq()
|
||||
|
||||
# Helper functions to get proof for any element of a list
|
||||
proc getProofForAllListElements*(list: List): seq[Digest] =
|
||||
type T = type(list)
|
||||
type E = ElemType(T)
|
||||
# basic types have different chunking rules
|
||||
static:
|
||||
doAssert (E is not BasicType)
|
||||
var digests: seq[Digest] = @[]
|
||||
for e in list:
|
||||
let root = hash_tree_root(e)
|
||||
digests.add(root)
|
||||
var merk = createMerkleizer(list.maxLen)
|
||||
merk.addChunksAndGenMerkleProofs(digests)
|
||||
|
||||
proc getProofWithIdx*(list: List, allProofs: seq[Digest], idx: int): seq[Digest] =
|
||||
let treeHeight = binaryTreeHeight(list.maxLen)
|
||||
let startPos = idx * treeHeight
|
||||
let endPos = startPos + treeHeight - 2
|
||||
slice(allProofs.toOpenArray(startPos, endPos))
|
||||
|
||||
proc generateAndGetProofWithIdx*(list: List, idx: int): seq[Digest] =
|
||||
let allProofs = getProofForAllListElements(list)
|
||||
getProofWithIdx(list, allProofs, idx)
|
||||
|
|
@ -1,247 +0,0 @@
|
|||
# nim-eth - Limited SSZ implementation
|
||||
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
||||
|
||||
{.push raises: [Defect].}
|
||||
|
||||
## SSZ serialization for core SSZ types, as specified in:
|
||||
# https://github.com/ethereum/consensus-specs/blob/v1.0.1/ssz/simple-serialize.md#serialization
|
||||
|
||||
import
|
||||
std/[typetraits, options],
|
||||
stew/[endians2, leb128, objects],
|
||||
serialization, serialization/testing/tracing,
|
||||
./bytes_reader, ./types
|
||||
|
||||
export
|
||||
serialization, types, bytes_reader
|
||||
|
||||
type
|
||||
SszReader* = object
|
||||
stream: InputStream
|
||||
|
||||
SszWriter* = object
|
||||
stream: OutputStream
|
||||
|
||||
SizePrefixed*[T] = distinct T
|
||||
SszMaxSizeExceeded* = object of SerializationError
|
||||
|
||||
VarSizedWriterCtx = object
|
||||
fixedParts: WriteCursor
|
||||
offset: int
|
||||
|
||||
FixedSizedWriterCtx = object
|
||||
|
||||
serializationFormat SSZ
|
||||
|
||||
SSZ.setReader SszReader
|
||||
SSZ.setWriter SszWriter, PreferredOutput = seq[byte]
|
||||
|
||||
template sizePrefixed*[TT](x: TT): untyped =
|
||||
type T = TT
|
||||
SizePrefixed[T](x)
|
||||
|
||||
proc init*(T: type SszReader, stream: InputStream): T {.raises: [Defect].} =
|
||||
T(stream: stream)
|
||||
|
||||
proc writeFixedSized(s: var (OutputStream|WriteCursor), x: auto)
|
||||
{.raises: [Defect, IOError].} =
|
||||
mixin toSszType
|
||||
|
||||
when x is byte:
|
||||
s.write x
|
||||
elif x is bool:
|
||||
s.write byte(ord(x))
|
||||
elif x is UintN:
|
||||
when cpuEndian == bigEndian:
|
||||
s.write toBytesLE(x)
|
||||
else:
|
||||
s.writeMemCopy x
|
||||
elif x is array:
|
||||
when x[0] is byte:
|
||||
trs "APPENDING FIXED SIZE BYTES", x
|
||||
s.write x
|
||||
else:
|
||||
for elem in x:
|
||||
trs "WRITING FIXED SIZE ARRAY ELEMENT"
|
||||
s.writeFixedSized toSszType(elem)
|
||||
elif x is tuple|object:
|
||||
enumInstanceSerializedFields(x, fieldName, field):
|
||||
trs "WRITING FIXED SIZE FIELD", fieldName
|
||||
s.writeFixedSized toSszType(field)
|
||||
else:
|
||||
unsupported x.type
|
||||
|
||||
template writeOffset(cursor: var WriteCursor, offset: int) =
|
||||
write cursor, toBytesLE(uint32 offset)
|
||||
|
||||
template supports*(_: type SSZ, T: type): bool =
|
||||
mixin toSszType
|
||||
anonConst compiles(fixedPortionSize toSszType(declval T))
|
||||
|
||||
func init*(T: type SszWriter, stream: OutputStream): T {.raises: [Defect].} =
|
||||
result.stream = stream
|
||||
|
||||
proc writeVarSizeType(w: var SszWriter, value: auto)
|
||||
{.gcsafe, raises: [Defect, IOError].}
|
||||
|
||||
proc beginRecord*(w: var SszWriter, TT: type): auto {.raises: [Defect].} =
|
||||
type T = TT
|
||||
when isFixedSize(T):
|
||||
FixedSizedWriterCtx()
|
||||
else:
|
||||
const offset = when T is array: len(T) * offsetSize
|
||||
else: fixedPortionSize(T)
|
||||
VarSizedWriterCtx(offset: offset,
|
||||
fixedParts: w.stream.delayFixedSizeWrite(offset))
|
||||
|
||||
template writeField*(w: var SszWriter,
|
||||
ctx: var auto,
|
||||
fieldName: string,
|
||||
field: auto) =
|
||||
mixin toSszType
|
||||
when ctx is FixedSizedWriterCtx:
|
||||
writeFixedSized(w.stream, toSszType(field))
|
||||
else:
|
||||
type FieldType = type toSszType(field)
|
||||
|
||||
when isFixedSize(FieldType):
|
||||
writeFixedSized(ctx.fixedParts, toSszType(field))
|
||||
else:
|
||||
trs "WRITING OFFSET ", ctx.offset, " FOR ", fieldName
|
||||
writeOffset(ctx.fixedParts, ctx.offset)
|
||||
let initPos = w.stream.pos
|
||||
trs "WRITING VAR SIZE VALUE OF TYPE ", name(FieldType)
|
||||
when FieldType is BitList:
|
||||
trs "BIT SEQ ", bytes(field)
|
||||
writeVarSizeType(w, toSszType(field))
|
||||
ctx.offset += w.stream.pos - initPos
|
||||
|
||||
template endRecord*(w: var SszWriter, ctx: var auto) =
|
||||
when ctx is VarSizedWriterCtx:
|
||||
finalize ctx.fixedParts
|
||||
|
||||
proc writeSeq[T](w: var SszWriter, value: seq[T])
|
||||
{.raises: [Defect, IOError].} =
|
||||
# Please note that `writeSeq` exists in order to reduce the code bloat
|
||||
# produced from generic instantiations of the unique `List[N, T]` types.
|
||||
when isFixedSize(T):
|
||||
trs "WRITING LIST WITH FIXED SIZE ELEMENTS"
|
||||
for elem in value:
|
||||
w.stream.writeFixedSized toSszType(elem)
|
||||
trs "DONE"
|
||||
else:
|
||||
trs "WRITING LIST WITH VAR SIZE ELEMENTS"
|
||||
var offset = value.len * offsetSize
|
||||
var cursor = w.stream.delayFixedSizeWrite offset
|
||||
for elem in value:
|
||||
cursor.writeFixedSized uint32(offset)
|
||||
let initPos = w.stream.pos
|
||||
w.writeVarSizeType toSszType(elem)
|
||||
offset += w.stream.pos - initPos
|
||||
finalize cursor
|
||||
trs "DONE"
|
||||
|
||||
proc writeVarSizeType(w: var SszWriter, value: auto)
|
||||
{.raises: [Defect, IOError].} =
|
||||
trs "STARTING VAR SIZE TYPE"
|
||||
|
||||
when value is List:
|
||||
# We reduce code bloat by forwarding all `List` types to a general `seq[T]`
|
||||
# proc.
|
||||
writeSeq(w, asSeq value)
|
||||
elif value is BitList:
|
||||
# ATTENTION! We can reuse `writeSeq` only as long as our BitList type is
|
||||
# implemented to internally match the binary representation of SSZ BitLists
|
||||
# in memory.
|
||||
writeSeq(w, bytes value)
|
||||
elif value is object|tuple|array:
|
||||
trs "WRITING OBJECT OR ARRAY"
|
||||
var ctx = beginRecord(w, type value)
|
||||
enumerateSubFields(value, field):
|
||||
writeField w, ctx, astToStr(field), field
|
||||
endRecord w, ctx
|
||||
else:
|
||||
unsupported type(value)
|
||||
|
||||
proc writeValue*(w: var SszWriter, x: auto)
|
||||
{.gcsafe, raises: [Defect, IOError].} =
|
||||
mixin toSszType
|
||||
type T = type toSszType(x)
|
||||
|
||||
when isFixedSize(T):
|
||||
w.stream.writeFixedSized toSszType(x)
|
||||
else:
|
||||
w.writeVarSizeType toSszType(x)
|
||||
|
||||
func sszSize*(value: auto): int {.gcsafe, raises: [Defect].}
|
||||
|
||||
func sszSizeForVarSizeList[T](value: openArray[T]): int =
|
||||
mixin toSszType
|
||||
result = len(value) * offsetSize
|
||||
for elem in value:
|
||||
result += sszSize(toSszType elem)
|
||||
|
||||
func sszSize*(value: auto): int {.gcsafe, raises: [Defect].} =
|
||||
mixin toSszType
|
||||
type T = type toSszType(value)
|
||||
|
||||
when isFixedSize(T):
|
||||
anonConst fixedPortionSize(T)
|
||||
|
||||
elif T is array|List:
|
||||
type E = ElemType(T)
|
||||
when isFixedSize(E):
|
||||
len(value) * anonConst(fixedPortionSize(E))
|
||||
elif T is HashArray:
|
||||
sszSizeForVarSizeList(value.data)
|
||||
elif T is array:
|
||||
sszSizeForVarSizeList(value)
|
||||
else:
|
||||
sszSizeForVarSizeList(asSeq value)
|
||||
|
||||
elif T is BitList:
|
||||
return len(bytes(value))
|
||||
|
||||
elif T is object|tuple:
|
||||
result = anonConst fixedPortionSize(T)
|
||||
enumInstanceSerializedFields(value, _{.used.}, field):
|
||||
type FieldType = type toSszType(field)
|
||||
when not isFixedSize(FieldType):
|
||||
result += sszSize(toSszType field)
|
||||
|
||||
else:
|
||||
unsupported T
|
||||
|
||||
proc writeValue*[T](w: var SszWriter, x: SizePrefixed[T])
|
||||
{.raises: [Defect, IOError].} =
|
||||
var cursor = w.stream.delayVarSizeWrite(Leb128.maxLen(uint64))
|
||||
let initPos = w.stream.pos
|
||||
w.writeValue T(x)
|
||||
let length = toBytes(uint64(w.stream.pos - initPos), Leb128)
|
||||
cursor.finalWrite length.toOpenArray()
|
||||
|
||||
proc readValue*[T](r: var SszReader, val: var T)
|
||||
{.raises: [Defect, SszError, IOError].} =
|
||||
when isFixedSize(T):
|
||||
const minimalSize = fixedPortionSize(T)
|
||||
if r.stream.readable(minimalSize):
|
||||
readSszValue(r.stream.read(minimalSize), val)
|
||||
else:
|
||||
raise newException(MalformedSszError, "SSZ input of insufficient size")
|
||||
else:
|
||||
# TODO(zah) Read the fixed portion first and precisely measure the
|
||||
# size of the dynamic portion to consume the right number of bytes.
|
||||
readSszValue(r.stream.read(r.stream.len.get), val)
|
||||
|
||||
proc readSszBytes*[T](data: openArray[byte], val: var T) {.
|
||||
raises: [Defect, MalformedSszError, SszSizeMismatchError].} =
|
||||
when isFixedSize(T):
|
||||
const minimalSize = fixedPortionSize(T)
|
||||
if data.len < minimalSize:
|
||||
raise newException(MalformedSszError, "SSZ input of insufficient size")
|
||||
|
||||
readSszValue(data, val)
|
||||
|
|
@ -1,302 +0,0 @@
|
|||
# nim-eth - Limited SSZ implementation
|
||||
# Copyright (c) 2018-2021 Status Research & Development GmbH
|
||||
# Licensed and distributed under either of
|
||||
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
||||
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
||||
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
||||
|
||||
{.push raises: [Defect].}
|
||||
|
||||
import
|
||||
std/[tables, options, typetraits, strformat],
|
||||
stew/shims/macros, stew/[byteutils, bitops2, objects],
|
||||
nimcrypto/hash, serialization/[object_serialization, errors],
|
||||
./bitseqs
|
||||
|
||||
export bitseqs
|
||||
|
||||
const
|
||||
offsetSize* = 4
|
||||
bytesPerChunk* = 32
|
||||
|
||||
type
|
||||
UintN* = SomeUnsignedInt
|
||||
BasicType* = bool|UintN
|
||||
|
||||
Limit* = int64
|
||||
|
||||
List*[T; maxLen: static Limit] = distinct seq[T]
|
||||
BitList*[maxLen: static Limit] = distinct BitSeq
|
||||
Digest* = MDigest[32 * 8]
|
||||
|
||||
# Note for readers:
|
||||
# We use `array` for `Vector` and
|
||||
# `BitArray` for `BitVector`
|
||||
|
||||
SszError* = object of SerializationError
|
||||
|
||||
MalformedSszError* = object of SszError
|
||||
|
||||
SszSizeMismatchError* = object of SszError
|
||||
deserializedType*: cstring
|
||||
actualSszSize*: int
|
||||
elementSize*: int
|
||||
|
||||
# A few index types from here onwards:
|
||||
# * dataIdx - leaf index starting from 0 to maximum length of collection
|
||||
# * chunkIdx - leaf data index after chunking starting from 0
|
||||
# * vIdx - virtual index in merkle tree - the root is found at index 1, its
|
||||
# two children at 2, 3 then 4, 5, 6, 7 etc
|
||||
|
||||
func nextPow2Int64(x: int64): int64 =
|
||||
# TODO the nextPow2 in bitops2 works with uint64 - there's a bug in the nim
|
||||
# compiler preventing it to be used - it seems that a conversion to
|
||||
# uint64 cannot be done with the static maxLen :(
|
||||
var v = x - 1
|
||||
|
||||
# round down, make sure all bits are 1 below the threshold, then add 1
|
||||
v = v or v shr 1
|
||||
v = v or v shr 2
|
||||
v = v or v shr 4
|
||||
when bitsof(x) > 8:
|
||||
v = v or v shr 8
|
||||
when bitsof(x) > 16:
|
||||
v = v or v shr 16
|
||||
when bitsof(x) > 32:
|
||||
v = v or v shr 32
|
||||
|
||||
v + 1
|
||||
|
||||
template dataPerChunk(T: type): int =
|
||||
# How many data items fit in a chunk
|
||||
when T is BasicType:
|
||||
bytesPerChunk div sizeof(T)
|
||||
else:
|
||||
1
|
||||
|
||||
template chunkIdx*(T: type, dataIdx: int64): int64 =
|
||||
# Given a data index, which chunk does it belong to?
|
||||
dataIdx div dataPerChunk(T)
|
||||
|
||||
template maxChunkIdx*(T: type, maxLen: Limit): int64 =
|
||||
# Given a number of data items, how many chunks are needed?
|
||||
# TODO compiler bug:
|
||||
# beacon_chain/ssz/types.nim(75, 53) Error: cannot generate code for: maxLen
|
||||
# nextPow2(chunkIdx(T, maxLen + dataPerChunk(T) - 1).uint64).int64
|
||||
nextPow2Int64(chunkIdx(T, maxLen.int64 + dataPerChunk(T) - 1))
|
||||
|
||||
template asSeq*(x: List): auto = distinctBase(x)
|
||||
|
||||
template init*[T](L: type List, x: seq[T], N: static Limit): auto =
|
||||
List[T, N](x)
|
||||
|
||||
template init*[T, N](L: type List[T, N], x: seq[T]): auto =
|
||||
List[T, N](x)
|
||||
|
||||
template `$`*(x: List): auto = $(distinctBase x)
|
||||
template len*(x: List): auto = len(distinctBase x)
|
||||
template low*(x: List): auto = low(distinctBase x)
|
||||
template high*(x: List): auto = high(distinctBase x)
|
||||
template `[]`*(x: List, idx: auto): untyped = distinctBase(x)[idx]
|
||||
template `[]=`*(x: var List, idx: auto, val: auto) = distinctBase(x)[idx] = val
|
||||
template `==`*(a, b: List): bool = distinctBase(a) == distinctBase(b)
|
||||
|
||||
template `&`*(a, b: List): auto = (type(a)(distinctBase(a) & distinctBase(b)))
|
||||
|
||||
template items* (x: List): untyped = items(distinctBase x)
|
||||
template pairs* (x: List): untyped = pairs(distinctBase x)
|
||||
template mitems*(x: var List): untyped = mitems(distinctBase x)
|
||||
template mpairs*(x: var List): untyped = mpairs(distinctBase x)
|
||||
|
||||
template contains* (x: List, val: auto): untyped = contains(distinctBase x, val)
|
||||
|
||||
proc add*(x: var List, val: auto): bool =
|
||||
if x.len < x.maxLen:
|
||||
add(distinctBase x, val)
|
||||
true
|
||||
else:
|
||||
false
|
||||
|
||||
proc setLen*(x: var List, newLen: int): bool =
|
||||
if newLen <= x.maxLen:
|
||||
setLen(distinctBase x, newLen)
|
||||
true
|
||||
else:
|
||||
false
|
||||
|
||||
template init*(L: type BitList, x: seq[byte], N: static Limit): auto =
|
||||
BitList[N](data: x)
|
||||
|
||||
template init*[N](L: type BitList[N], x: seq[byte]): auto =
|
||||
L(data: x)
|
||||
|
||||
template init*(T: type BitList, len: int): auto = T init(BitSeq, len)
|
||||
template len*(x: BitList): auto = len(BitSeq(x))
|
||||
template bytes*(x: BitList): auto = seq[byte](x)
|
||||
template `[]`*(x: BitList, idx: auto): auto = BitSeq(x)[idx]
|
||||
template `[]=`*(x: var BitList, idx: auto, val: bool) = BitSeq(x)[idx] = val
|
||||
template `==`*(a, b: BitList): bool = BitSeq(a) == BitSeq(b)
|
||||
template setBit*(x: var BitList, idx: Natural) = setBit(BitSeq(x), idx)
|
||||
template clearBit*(x: var BitList, idx: Natural) = clearBit(BitSeq(x), idx)
|
||||
template overlaps*(a, b: BitList): bool = overlaps(BitSeq(a), BitSeq(b))
|
||||
template incl*(a: var BitList, b: BitList) = incl(BitSeq(a), BitSeq(b))
|
||||
template isSubsetOf*(a, b: BitList): bool = isSubsetOf(BitSeq(a), BitSeq(b))
|
||||
template isZeros*(x: BitList): bool = isZeros(BitSeq(x))
|
||||
template countOnes*(x: BitList): int = countOnes(BitSeq(x))
|
||||
template countZeros*(x: BitList): int = countZeros(BitSeq(x))
|
||||
template countOverlap*(x, y: BitList): int = countOverlap(BitSeq(x), BitSeq(y))
|
||||
template `$`*(a: BitList): string = $(BitSeq(a))
|
||||
|
||||
iterator items*(x: BitList): bool =
|
||||
for i in 0 ..< x.len:
|
||||
yield x[i]
|
||||
|
||||
macro unsupported*(T: typed): untyped =
|
||||
# TODO: {.fatal.} breaks compilation even in `compiles()` context,
|
||||
# so we use this macro instead. It's also much better at figuring
|
||||
# out the actual type that was used in the instantiation.
|
||||
# File both problems as issues.
|
||||
error "SSZ serialization of the type " & humaneTypeName(T) & " is not supported"
|
||||
|
||||
template ElemType*(T: type array): untyped =
|
||||
type(default(T)[low(T)])
|
||||
|
||||
template ElemType*(T: type seq): untyped =
|
||||
type(default(T)[0])
|
||||
|
||||
template ElemType*(T0: type List): untyped =
|
||||
T0.T
|
||||
|
||||
func isFixedSize*(T0: type): bool {.compileTime.} =
|
||||
mixin toSszType, enumAllSerializedFields
|
||||
|
||||
type T = type toSszType(declval T0)
|
||||
|
||||
when T is BasicType:
|
||||
return true
|
||||
elif T is array:
|
||||
return isFixedSize(ElemType(T))
|
||||
elif T is object|tuple:
|
||||
enumAllSerializedFields(T):
|
||||
when not isFixedSize(FieldType):
|
||||
return false
|
||||
return true
|
||||
|
||||
func fixedPortionSize*(T0: type): int {.compileTime.} =
|
||||
mixin enumAllSerializedFields, toSszType
|
||||
|
||||
type T = type toSszType(declval T0)
|
||||
|
||||
when T is BasicType: sizeof(T)
|
||||
elif T is array:
|
||||
type E = ElemType(T)
|
||||
when isFixedSize(E): int(len(T)) * fixedPortionSize(E)
|
||||
else: int(len(T)) * offsetSize
|
||||
elif T is object|tuple:
|
||||
enumAllSerializedFields(T):
|
||||
when isFixedSize(FieldType):
|
||||
result += fixedPortionSize(FieldType)
|
||||
else:
|
||||
result += offsetSize
|
||||
else:
|
||||
unsupported T0
|
||||
|
||||
# TODO This should have been an iterator, but the VM can't compile the
|
||||
# code due to "too many registers required".
|
||||
proc fieldInfos*(RecordType: type): seq[tuple[name: string,
|
||||
offset: int,
|
||||
fixedSize: int,
|
||||
branchKey: string]] =
|
||||
mixin enumAllSerializedFields
|
||||
|
||||
var
|
||||
offsetInBranch = {"": 0}.toTable
|
||||
nestedUnder = initTable[string, string]()
|
||||
|
||||
enumAllSerializedFields(RecordType):
|
||||
const
|
||||
isFixed = isFixedSize(FieldType)
|
||||
fixedSize = when isFixed: fixedPortionSize(FieldType)
|
||||
else: 0
|
||||
branchKey = when fieldCaseDiscriminator.len == 0: ""
|
||||
else: fieldCaseDiscriminator & ":" & $fieldCaseBranches
|
||||
fieldSize = when isFixed: fixedSize
|
||||
else: offsetSize
|
||||
|
||||
nestedUnder[fieldName] = branchKey
|
||||
|
||||
var fieldOffset: int
|
||||
offsetInBranch.withValue(branchKey, val):
|
||||
fieldOffset = val[]
|
||||
val[] += fieldSize
|
||||
do:
|
||||
try:
|
||||
let parentBranch = nestedUnder.getOrDefault(fieldCaseDiscriminator, "")
|
||||
fieldOffset = offsetInBranch[parentBranch]
|
||||
offsetInBranch[branchKey] = fieldOffset + fieldSize
|
||||
except KeyError as e:
|
||||
raiseAssert e.msg
|
||||
|
||||
result.add((fieldName, fieldOffset, fixedSize, branchKey))
|
||||
|
||||
func getFieldBoundingOffsetsImpl(RecordType: type, fieldName: static string):
|
||||
tuple[fieldOffset, nextFieldOffset: int, isFirstOffset: bool]
|
||||
{.compileTime.} =
|
||||
result = (-1, -1, false)
|
||||
var fieldBranchKey: string
|
||||
var isFirstOffset = true
|
||||
|
||||
for f in fieldInfos(RecordType):
|
||||
if fieldName == f.name:
|
||||
result[0] = f.offset
|
||||
if f.fixedSize > 0:
|
||||
result[1] = result[0] + f.fixedSize
|
||||
return
|
||||
else:
|
||||
fieldBranchKey = f.branchKey
|
||||
result.isFirstOffset = isFirstOffset
|
||||
|
||||
elif result[0] != -1 and
|
||||
f.fixedSize == 0 and
|
||||
f.branchKey == fieldBranchKey:
|
||||
# We have found the next variable sized field
|
||||
result[1] = f.offset
|
||||
return
|
||||
|
||||
if f.fixedSize == 0:
|
||||
isFirstOffset = false
|
||||
|
||||
func getFieldBoundingOffsets*(RecordType: type, fieldName: static string):
|
||||
tuple[fieldOffset, nextFieldOffset: int, isFirstOffset: bool]
|
||||
{.compileTime.} =
|
||||
## Returns the start and end offsets of a field.
|
||||
##
|
||||
## For fixed-size fields, the start offset points to the first
|
||||
## byte of the field and the end offset points to 1 byte past the
|
||||
## end of the field.
|
||||
##
|
||||
## For variable-size fields, the returned offsets point to the
|
||||
## statically known positions of the 32-bit offset values written
|
||||
## within the SSZ object. You must read the 32-bit values stored
|
||||
## at the these locations in order to obtain the actual offsets.
|
||||
##
|
||||
## For variable-size fields, the end offset may be -1 when the
|
||||
## designated field is the last variable sized field within the
|
||||
## object. Then the SSZ object boundary known at run-time marks
|
||||
## the end of the variable-size field.
|
||||
type T = RecordType
|
||||
anonConst getFieldBoundingOffsetsImpl(T, fieldName)
|
||||
|
||||
template enumerateSubFields*(holder, fieldVar, body: untyped) =
|
||||
when holder is array:
|
||||
for fieldVar in holder: body
|
||||
else:
|
||||
enumInstanceSerializedFields(holder, _{.used.}, fieldVar): body
|
||||
|
||||
method formatMsg*(
|
||||
err: ref SszSizeMismatchError,
|
||||
filename: string): string {.gcsafe, raises: [Defect].} =
|
||||
try:
|
||||
&"SSZ size mismatch, element {err.elementSize}, actual {err.actualSszSize}, type {err.deserializedType}, file {filename}"
|
||||
except CatchableError:
|
||||
"SSZ size mismatch"
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
import
|
||||
./test_verification,
|
||||
./test_proofs
|
||||
|
|
@ -1,123 +0,0 @@
|
|||
{.used.}
|
||||
|
||||
import
|
||||
sequtils, unittest, math,
|
||||
nimcrypto/[hash, sha2],
|
||||
stew/endians2,
|
||||
../eth/ssz/merkleization,
|
||||
../eth/ssz/ssz_serialization,
|
||||
../eth/ssz/merkle_tree
|
||||
|
||||
template toSszType(x: auto): auto =
|
||||
x
|
||||
|
||||
proc h(a: openArray[byte]): Digest =
|
||||
var h: sha256
|
||||
h.init()
|
||||
h.update(a)
|
||||
h.finish()
|
||||
|
||||
type TestObject = object
|
||||
digest: array[32, byte]
|
||||
num: uint64
|
||||
|
||||
proc genObject(num: uint64): TestObject =
|
||||
let numAsHash = h(num.toBytesLE())
|
||||
TestObject(digest: numAsHash.data, num: num)
|
||||
|
||||
proc genNObjects(n: int): seq[TestObject] =
|
||||
var objs = newSeq[TestObject]()
|
||||
for i in 1..n:
|
||||
let obj = genObject(uint64 i)
|
||||
objs.add(obj)
|
||||
objs
|
||||
|
||||
proc getGenIndex(idx: int, depth: uint64): uint64 =
|
||||
uint64 (math.pow(2'f64, float64 depth) + float64 idx)
|
||||
|
||||
# Normal hash_tree_root add list length to final hash calculation. Proofs by default
|
||||
# are generated without it. If necessary length of the list can be added manually
|
||||
# at the end of the proof but here we are just hashing list with no mixin.
|
||||
proc getListRootNoMixin(list: List): Digest =
|
||||
var merk = createMerkleizer(list.maxLen)
|
||||
for e in list:
|
||||
let hash = hash_tree_root(e)
|
||||
merk.addChunk(hash.data)
|
||||
merk.getFinalHash()
|
||||
|
||||
type TestCase = object
|
||||
numOfElements: int
|
||||
limit: int
|
||||
|
||||
const TestCases = (
|
||||
TestCase(numOfElements: 0, limit: 2),
|
||||
TestCase(numOfElements: 1, limit: 2),
|
||||
TestCase(numOfElements: 2, limit: 2),
|
||||
|
||||
TestCase(numOfElements: 0, limit: 4),
|
||||
TestCase(numOfElements: 1, limit: 4),
|
||||
TestCase(numOfElements: 2, limit: 4),
|
||||
TestCase(numOfElements: 3, limit: 4),
|
||||
TestCase(numOfElements: 4, limit: 4),
|
||||
|
||||
TestCase(numOfElements: 0, limit: 8),
|
||||
TestCase(numOfElements: 1, limit: 8),
|
||||
TestCase(numOfElements: 2, limit: 8),
|
||||
TestCase(numOfElements: 3, limit: 8),
|
||||
TestCase(numOfElements: 4, limit: 8),
|
||||
TestCase(numOfElements: 5, limit: 8),
|
||||
TestCase(numOfElements: 6, limit: 8),
|
||||
TestCase(numOfElements: 7, limit: 8),
|
||||
TestCase(numOfElements: 8, limit: 8),
|
||||
|
||||
TestCase(numOfElements: 0, limit: 16),
|
||||
TestCase(numOfElements: 1, limit: 16),
|
||||
TestCase(numOfElements: 2, limit: 16),
|
||||
TestCase(numOfElements: 3, limit: 16),
|
||||
TestCase(numOfElements: 4, limit: 16),
|
||||
TestCase(numOfElements: 5, limit: 16),
|
||||
TestCase(numOfElements: 6, limit: 16),
|
||||
TestCase(numOfElements: 7, limit: 16),
|
||||
TestCase(numOfElements: 16, limit: 16),
|
||||
|
||||
TestCase(numOfElements: 32, limit: 32),
|
||||
|
||||
TestCase(numOfElements: 64, limit: 64)
|
||||
)
|
||||
|
||||
suite "Merkle Proof generation":
|
||||
test "generation of proof for various tree sizes":
|
||||
for testCase in TestCases.fields:
|
||||
let testObjects = genNObjects(testCase.numOfElements)
|
||||
let treeDepth = uint64 binaryTreeHeight(testCase.limit) - 1
|
||||
|
||||
# Create List and and genereate root by using merkelizer
|
||||
let list = List.init(testObjects, testCase.limit)
|
||||
let listRoot = getListRootNoMixin(list)
|
||||
|
||||
# Create sparse merkle tree from list elements and generate root
|
||||
let listDigests = map(testObjects, proc(x: TestObject): Digest = hash_tree_root(x))
|
||||
let tree = createTree(listDigests, treeDepth)
|
||||
let treeHash = tree.hash()
|
||||
|
||||
# Assert that by using both methods we get same hash
|
||||
check listRoot == treeHash
|
||||
|
||||
for i, e in list:
|
||||
# generate proof by using merkelizer
|
||||
let merkleizerProof = generateAndGetProofWithIdx(list, i)
|
||||
# generate proof by sparse merkle tree
|
||||
let sparseTreeProof = genProof(tree, uint64 i, treeDepth)
|
||||
|
||||
let leafHash = hash_tree_root(e)
|
||||
let genIndex = getGenIndex(i, treeDepth)
|
||||
|
||||
# both proof are valid. If both are valid that means that both proof are
|
||||
# effectivly the same
|
||||
let isValidProof = isValidProof(leafHash , merkleizerProof, genIndex, listRoot)
|
||||
let isValidProof1 = isValidProof(leafHash , sparseTreeProof, genIndex, listRoot)
|
||||
|
||||
check isValidProof
|
||||
check isValidProof1
|
||||
|
||||
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
{.used.}
|
||||
|
||||
import
|
||||
sequtils, unittest,
|
||||
nimcrypto/[hash, sha2],
|
||||
../eth/ssz/merkleization
|
||||
|
||||
type TestCase = object
|
||||
root: string
|
||||
proof: seq[string]
|
||||
leaf: string
|
||||
index: uint64
|
||||
valid: bool
|
||||
|
||||
let testCases = @[
|
||||
TestCase(
|
||||
root: "2a23ef2b7a7221eaac2ffb3842a506a981c009ca6c2fcbf20adbc595e56f1a93",
|
||||
proof: @[
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b"
|
||||
],
|
||||
leaf: "0100000000000000000000000000000000000000000000000000000000000000",
|
||||
index: 4,
|
||||
valid: true
|
||||
),
|
||||
TestCase(
|
||||
root: "2a23ef2b7a7221eaac2ffb3842a506a981c009ca6c2fcbf20adbc595e56f1a93",
|
||||
proof: @[
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b"
|
||||
],
|
||||
leaf: "0100000000000000000000000000000000000000000000000000000000000000",
|
||||
index: 6,
|
||||
valid: false
|
||||
),
|
||||
TestCase(
|
||||
root: "2a23ef2b7a7221eaac2ffb3842a506a981c009ca6c2fcbf20adbc595e56f1a93",
|
||||
proof: @[
|
||||
"0100000000000000000000000000000000000000000000000000000000000000",
|
||||
"f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b"
|
||||
],
|
||||
leaf: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
index: 5,
|
||||
valid: true
|
||||
),
|
||||
TestCase(
|
||||
root: "f1824b0084956084591ff4c91c11bcc94a40be82da280e5171932b967dd146e9",
|
||||
proof: @[
|
||||
"35210d64853aee79d03f30cf0f29c1398706cbbcacaf05ab9524f00070aec91e",
|
||||
"f38a181470ef1eee90a29f0af0a9dba6b7e5d48af3c93c29b4f91fa11b777582"
|
||||
],
|
||||
leaf: "0100000000000000000000000000000000000000000000000000000000000000",
|
||||
index: 7,
|
||||
valid: true
|
||||
),
|
||||
TestCase(
|
||||
root: "f1824b0084956084591ff4c91c11bcc94a40be82da280e5171932b967dd146e9",
|
||||
proof: @[
|
||||
"0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b",
|
||||
"0100000000000000000000000000000000000000000000000000000000000000",
|
||||
"f38a181470ef1eee90a29f0af0a9dba6b7e5d48af3c93c29b4f91fa11b777582"
|
||||
],
|
||||
leaf: "6001000000000000000000000000000000000000000000000000000000000000",
|
||||
index: 49,
|
||||
valid: true
|
||||
)
|
||||
]
|
||||
|
||||
suite "Merkle Proof verification":
|
||||
test "correctly verify proof":
|
||||
for testCase in testCases:
|
||||
let root = MDigest[256].fromHex(testCase.root)
|
||||
let proof = map(testCase.proof, proc(x: string): Digest = MDigest[256].fromHex(x))
|
||||
let leaf = MDigest[256].fromHex(testCase.leaf)
|
||||
let valid = isValidProof(leaf, proof, testCase.index, root)
|
||||
|
||||
if (testCase.valid):
|
||||
check valid
|
||||
else:
|
||||
check (not valid)
|
||||
Loading…
Reference in New Issue