diff --git a/eth.nimble b/eth.nimble index d8ffd8f..af127cd 100644 --- a/eth.nimble +++ b/eth.nimble @@ -66,9 +66,6 @@ task test_trie, "Run trie tests": task test_db, "Run db tests": runTest("tests/db/all_tests") -task test_ssz, "Run ssz tests": - runTest("tests/ssz/all_tests") - task test_utp, "Run utp tests": runTest("tests/utp/all_utp_tests") @@ -84,7 +81,6 @@ task test, "Run all tests": test_p2p_task() test_trie_task() test_db_task() - test_ssz_task() test_utp_task() task test_discv5_full, "Run discovery v5 and its dependencies tests": diff --git a/eth/ssz/bitseqs.nim b/eth/ssz/bitseqs.nim deleted file mode 100644 index c4ca4f1..0000000 --- a/eth/ssz/bitseqs.nim +++ /dev/null @@ -1,313 +0,0 @@ -# nim-eth -# Copyright (c) 2018-2021 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -{.push raises: [Defect].} - -import - stew/[bitops2, endians2, ptrops] - -type - Bytes = seq[byte] - - BitSeq* = distinct Bytes - ## The current design of BitSeq tries to follow precisely - ## the bitwise representation of the SSZ bitlists. - ## This is a relatively compact representation, but as - ## evident from the code below, many of the operations - ## are not trivial. - - BitArray*[bits: static int] = object - bytes*: array[(bits + 7) div 8, byte] - -func bitsLen*(bytes: openArray[byte]): int = - let - bytesCount = bytes.len - lastByte = bytes[bytesCount - 1] - markerPos = log2trunc(lastByte) - - bytesCount * 8 - (8 - markerPos) - -template len*(s: BitSeq): int = - bitsLen(Bytes s) - -template len*(a: BitArray): int = - a.bits - -func add*(s: var BitSeq, value: bool) = - let - lastBytePos = s.Bytes.len - 1 - lastByte = s.Bytes[lastBytePos] - - if (lastByte and byte(128)) == 0: - # There is at least one leading zero, so we have enough - # room to store the new bit - let markerPos = log2trunc(lastByte) - s.Bytes[lastBytePos].changeBit markerPos, value - s.Bytes[lastBytePos].setBit markerPos + 1 - else: - s.Bytes[lastBytePos].changeBit 7, value - s.Bytes.add byte(1) - -func toBytesLE(x: uint): array[sizeof(x), byte] = - # stew/endians2 supports explicitly sized uints only - when sizeof(uint) == 4: - static: doAssert sizeof(uint) == sizeof(uint32) - toBytesLE(x.uint32) - elif sizeof(uint) == 8: - static: doAssert sizeof(uint) == sizeof(uint64) - toBytesLE(x.uint64) - else: - static: doAssert false, "requires a 32-bit or 64-bit platform" - -func loadLEBytes(WordType: type, bytes: openArray[byte]): WordType = - # TODO: this is a temporary proc until the endians API is improved - var shift = 0 - for b in bytes: - result = result or (WordType(b) shl shift) - shift += 8 - -func storeLEBytes(value: SomeUnsignedInt, dst: var openArray[byte]) = - doAssert dst.len <= sizeof(value) - let bytesLE = toBytesLE(value) - copyMem(addr dst[0], unsafeAddr bytesLE[0], dst.len) - -template loopOverWords(lhs, rhs: BitSeq, - lhsIsVar, rhsIsVar: static bool, - WordType: type, - lhsBits, rhsBits, body: untyped) = - const hasRhs = astToStr(lhs) != astToStr(rhs) - - let bytesCount = len Bytes(lhs) - when hasRhs: doAssert len(Bytes(rhs)) == bytesCount - - var fullWordsCount = bytesCount div sizeof(WordType) - let lastWordSize = bytesCount mod sizeof(WordType) - - block: - var lhsWord: WordType - when hasRhs: - var rhsWord: WordType - var firstByteOfLastWord, lastByteOfLastWord: int - - # TODO: Returning a `var` value from an iterator is always safe due to - # the way inlining works, but currently the compiler reports an error - # when a local variable escapes. We have to cheat it with this location - # obfuscation through pointers: - template lhsBits: auto = (addr(lhsWord))[] - - when hasRhs: - template rhsBits: auto = (addr(rhsWord))[] - - template lastWordBytes(bitseq): auto = - Bytes(bitseq).toOpenArray(firstByteOfLastWord, lastByteOfLastWord) - - template initLastWords = - lhsWord = loadLEBytes(WordType, lastWordBytes(lhs)) - when hasRhs: rhsWord = loadLEBytes(WordType, lastWordBytes(rhs)) - - if lastWordSize == 0: - firstByteOfLastWord = bytesCount - sizeof(WordType) - lastByteOfLastWord = bytesCount - 1 - dec fullWordsCount - else: - firstByteOfLastWord = bytesCount - lastWordSize - lastByteOfLastWord = bytesCount - 1 - - initLastWords() - let markerPos = log2trunc(lhsWord) - when hasRhs: doAssert log2trunc(rhsWord) == markerPos - - lhsWord.clearBit markerPos - when hasRhs: rhsWord.clearBit markerPos - - body - - when lhsIsVar or rhsIsVar: - let - markerBit = uint(1 shl markerPos) - mask = markerBit - 1'u - - when lhsIsVar: - let lhsEndResult = (lhsWord and mask) or markerBit - storeLEBytes(lhsEndResult, lastWordBytes(lhs)) - - when rhsIsVar: - let rhsEndResult = (rhsWord and mask) or markerBit - storeLEBytes(rhsEndResult, lastWordBytes(rhs)) - - var lhsCurrAddr = cast[ptr WordType](unsafeAddr Bytes(lhs)[0]) - let lhsEndAddr = offset(lhsCurrAddr, fullWordsCount) - when hasRhs: - var rhsCurrAddr = cast[ptr WordType](unsafeAddr Bytes(rhs)[0]) - - while lhsCurrAddr < lhsEndAddr: - template lhsBits: auto = lhsCurrAddr[] - when hasRhs: - template rhsBits: auto = rhsCurrAddr[] - - body - - lhsCurrAddr = offset(lhsCurrAddr, 1) - when hasRhs: rhsCurrAddr = offset(rhsCurrAddr, 1) - -iterator words*(x: var BitSeq): var uint = - loopOverWords(x, x, true, false, uint, word, wordB): - yield word - -iterator words*(x: BitSeq): uint = - loopOverWords(x, x, false, false, uint, word, word): - yield word - -iterator words*(a, b: BitSeq): (uint, uint) = - loopOverWords(a, b, false, false, uint, wordA, wordB): - yield (wordA, wordB) - -iterator words*(a: var BitSeq, b: BitSeq): (var uint, uint) = - loopOverWords(a, b, true, false, uint, wordA, wordB): - yield (wordA, wordB) - -iterator words*(a, b: var BitSeq): (var uint, var uint) = - loopOverWords(a, b, true, true, uint, wordA, wordB): - yield (wordA, wordB) - -func `[]`*(s: BitSeq, pos: Natural): bool {.inline.} = - doAssert pos < s.len - s.Bytes.getBit pos - -func `[]=`*(s: var BitSeq, pos: Natural, value: bool) {.inline.} = - doAssert pos < s.len - s.Bytes.changeBit pos, value - -func setBit*(s: var BitSeq, pos: Natural) {.inline.} = - doAssert pos < s.len - setBit s.Bytes, pos - -func clearBit*(s: var BitSeq, pos: Natural) {.inline.} = - doAssert pos < s.len - clearBit s.Bytes, pos - -func init*(T: type BitSeq, len: int): T = - result = BitSeq newSeq[byte](1 + len div 8) - Bytes(result).setBit len - -func init*(T: type BitArray): T = - # The default zero-initializatio is fine - discard - -template `[]`*(a: BitArray, pos: Natural): bool = - getBit a.bytes, pos - -template `[]=`*(a: var BitArray, pos: Natural, value: bool) = - changeBit a.bytes, pos, value - -template setBit*(a: var BitArray, pos: Natural) = - setBit a.bytes, pos - -template clearBit*(a: var BitArray, pos: Natural) = - clearBit a.bytes, pos - -# TODO: Submit this to the standard library as `cmp` -# At the moment, it doesn't work quite well because Nim selects -# the generic cmp[T] from the system module instead of choosing -# the openArray overload -func compareArrays[T](a, b: openArray[T]): int = - result = cmp(a.len, b.len) - if result != 0: return - - for i in 0 ..< a.len: - result = cmp(a[i], b[i]) - if result != 0: return - -template cmp*(a, b: BitSeq): int = - compareArrays(Bytes a, Bytes b) - -template `==`*(a, b: BitSeq): bool = - cmp(a, b) == 0 - -func `$`*(a: BitSeq | BitArray): string = - let length = a.len - result = newStringOfCap(2 + length) - result.add "0b" - for i in countdown(length - 1, 0): - result.add if a[i]: '1' else: '0' - -func incl*(tgt: var BitSeq, src: BitSeq) = - # Update `tgt` to include the bits of `src`, as if applying `or` to each bit - doAssert tgt.len == src.len - for tgtWord, srcWord in words(tgt, src): - tgtWord = tgtWord or srcWord - -func overlaps*(a, b: BitSeq): bool = - for wa, wb in words(a, b): - if (wa and wb) != 0: - return true - -func countOverlap*(a, b: BitSeq): int = - var res = 0 - for wa, wb in words(a, b): - res += countOnes(wa and wb) - res - -func isSubsetOf*(a, b: BitSeq): bool = - let alen = a.len - doAssert b.len == alen - for i in 0 ..< alen: - if a[i] and not b[i]: - return false - true - -func isZeros*(x: BitSeq): bool = - for w in words(x): - if w != 0: return false - return true - -func countOnes*(x: BitSeq): int = - # Count the number of set bits - var res = 0 - for w in words(x): - res += w.countOnes() - res - -func clear*(x: var BitSeq) = - for w in words(x): - w = 0 - -func countZeros*(x: BitSeq): int = - x.len() - x.countOnes() - -template bytes*(x: BitSeq): untyped = - seq[byte](x) - -iterator items*(x: BitArray): bool = - for i in 0.. byte(1): - raise newException(MalformedSszError, "invalid boolean value") - data[0] == 1 - -template fromSszBytes*(T: type BitSeq, bytes: openArray[byte]): auto = - BitSeq @bytes - -proc `[]`[T, U, V](s: openArray[T], x: HSlice[U, V]) {.error: - "Please don't use openArray's [] as it allocates a result sequence".} - -template checkForForbiddenBits(ResulType: type, - input: openArray[byte], - expectedBits: static int64) = - ## This checks if the input contains any bits set above the maximum - ## sized allowed. We only need to check the last byte to verify this: - const bitsInLastByte = (expectedBits mod 8) - when bitsInLastByte != 0: - # As an example, if there are 3 bits expected in the last byte, - # we calculate a bitmask equal to 11111000. If the input has any - # raised bits in range of the bitmask, this would be a violation - # of the size of the BitArray: - const forbiddenBitsMask = byte(byte(0xff) shl bitsInLastByte) - - if (input[^1] and forbiddenBitsMask) != 0: - raiseIncorrectSize ResulType - -func readSszValue*[T](input: openArray[byte], val: var T) - {.raises: [SszError, Defect].} = - mixin fromSszBytes, toSszType - - template readOffsetUnchecked(n: int): uint32 {.used.}= - fromSszBytes(uint32, input.toOpenArray(n, n + offsetSize - 1)) - - template readOffset(n: int): int {.used.} = - let offset = readOffsetUnchecked(n) - if offset > input.len.uint32: - raise newException(MalformedSszError, "SSZ list element offset points past the end of the input") - int(offset) - - when val is BitList: - if input.len == 0: - raise newException(MalformedSszError, "Invalid empty SSZ BitList value") - - # Since our BitLists have an in-memory representation that precisely - # matches their SSZ encoding, we can deserialize them as regular Lists: - const maxExpectedSize = (val.maxLen div 8) + 1 - type MatchingListType = List[byte, maxExpectedSize] - - when false: - # TODO: Nim doesn't like this simple type coercion, - # we'll rely on `cast` for now (see below) - readSszValue(input, MatchingListType val) - else: - static: - # As a sanity check, we verify that the coercion is accepted by the compiler: - doAssert MatchingListType(val) is MatchingListType - readSszValue(input, cast[ptr MatchingListType](addr val)[]) - - let resultBytesCount = len bytes(val) - - if bytes(val)[resultBytesCount - 1] == 0: - raise newException(MalformedSszError, "SSZ BitList is not properly terminated") - - if resultBytesCount == maxExpectedSize: - checkForForbiddenBits(T, input, val.maxLen + 1) - - elif val is List|array: - type E = type val[0] - - when E is byte: - val.setOutputSize input.len - if input.len > 0: - copyMem(addr val[0], unsafeAddr input[0], input.len) - - elif isFixedSize(E): - const elemSize = fixedPortionSize(E) - if input.len mod elemSize != 0: - var ex = new SszSizeMismatchError - ex.deserializedType = cstring typetraits.name(T) - ex.actualSszSize = input.len - ex.elementSize = elemSize - raise ex - val.setOutputSize input.len div elemSize - for i in 0 ..< val.len: - let offset = i * elemSize - readSszValue(input.toOpenArray(offset, offset + elemSize - 1), val[i]) - - else: - if input.len == 0: - # This is an empty list. - # The default initialization of the return value is fine. - val.setOutputSize 0 - return - elif input.len < offsetSize: - raise newException(MalformedSszError, "SSZ input of insufficient size") - - var offset = readOffset 0 - let resultLen = offset div offsetSize - - if resultLen == 0: - # If there are too many elements, other constraints detect problems - # (not monotonically increasing, past end of input, or last element - # not matching up with its nextOffset properly) - raise newException(MalformedSszError, "SSZ list incorrectly encoded of zero length") - - val.setOutputSize resultLen - for i in 1 ..< resultLen: - let nextOffset = readOffset(i * offsetSize) - if nextOffset <= offset: - raise newException(MalformedSszError, "SSZ list element offsets are not monotonically increasing") - else: - readSszValue(input.toOpenArray(offset, nextOffset - 1), val[i - 1]) - offset = nextOffset - - readSszValue(input.toOpenArray(offset, input.len - 1), val[resultLen - 1]) - - elif val is UintN|bool: - val = fromSszBytes(T, input) - - elif val is BitArray: - if sizeof(val) != input.len: - raiseIncorrectSize(T) - checkForForbiddenBits(T, input, val.bits) - copyMem(addr val.bytes[0], unsafeAddr input[0], input.len) - - elif val is object|tuple: - let inputLen = uint32 input.len - const minimallyExpectedSize = uint32 fixedPortionSize(T) - - if inputLen < minimallyExpectedSize: - raise newException(MalformedSszError, "SSZ input of insufficient size") - - enumInstanceSerializedFields(val, fieldName, field): - const boundingOffsets = getFieldBoundingOffsets(T, fieldName) - - # type FieldType = type field # buggy - # For some reason, Nim gets confused about the alias here. This could be a - # generics caching issue caused by the use of distinct types. Such an - # issue is very scary in general. - # The bug can be seen with the two List[uint64, N] types that exist in - # the spec, with different N. - - type SszType = type toSszType(declval type(field)) - - when isFixedSize(SszType): - const - startOffset = boundingOffsets[0] - endOffset = boundingOffsets[1] - else: - let - startOffset = readOffsetUnchecked(boundingOffsets[0]) - endOffset = if boundingOffsets[1] == -1: inputLen - else: readOffsetUnchecked(boundingOffsets[1]) - - when boundingOffsets.isFirstOffset: - if startOffset != minimallyExpectedSize: - raise newException(MalformedSszError, "SSZ object dynamic portion starts at invalid offset") - - if startOffset > endOffset: - raise newException(MalformedSszError, "SSZ field offsets are not monotonically increasing") - elif endOffset > inputLen: - raise newException(MalformedSszError, "SSZ field offset points past the end of the input") - elif startOffset < minimallyExpectedSize: - raise newException(MalformedSszError, "SSZ field offset points outside bounding offsets") - - # TODO The extra type escaping here is a work-around for a Nim issue: - when type(field) is type(SszType): - readSszValue( - input.toOpenArray(int(startOffset), int(endOffset - 1)), - field) - else: - field = fromSszBytes( - type(field), - input.toOpenArray(int(startOffset), int(endOffset - 1))) - - else: - unsupported T diff --git a/eth/ssz/merkle_tree.nim b/eth/ssz/merkle_tree.nim deleted file mode 100644 index 21977aa..0000000 --- a/eth/ssz/merkle_tree.nim +++ /dev/null @@ -1,111 +0,0 @@ -{.push raises: [Defect].} - -import - math, sequtils, ssz_serialization, options, algorithm, - nimcrypto/hash, - ../common/eth_types, ./types, ./merkleization - -const maxTreeDepth: uint64 = 32 -const empty: seq[Digest] = @[] - -type - MerkleNodeType = enum - LeafType, - NodeType, - ZeroType - - MerkleNode = ref object - case kind: MerkleNodeType - of LeafType: - digest: Digest - of NodeType: - innerDigest: Digest - left: MerkleNode - right: MerkleNode - of ZeroType: - depth: uint64 - -func zeroNodes(): seq[MerkleNode] = - var nodes = newSeq[MerkleNode]() - for i in 0..maxTreeDepth: - nodes.add(MerkleNode(kind: ZeroType, depth: i)) - return nodes - -let zNodes = zeroNodes() - -# This look like something that should be in standard lib. -func splitAt[T](s: openArray[T], idx: uint64): (seq[T], seq[T]) = - var lSeq = newSeq[T]() - var rSeq = newSeq[T]() - for i, e in s: - if (uint64(i) < idx): - lSeq.add(e) - else: - rSeq.add(e) - (lSeq, rSeq) - -func splitLeaves(l: openArray[Digest], cap: uint64): (seq[Digest], seq[Digest]) = - if (uint64(len(l)) <= cap): - (l.toSeq(), empty) - else: - splitAt(l, cap) - -proc getSubTrees(node: MerkleNode): Option[(MerkleNode, MerkleNode)] = - case node.kind - of LeafType: - return none[(MerkleNode, MerkleNode)]() - of NodeType: - return some((node.left, node.right)) - of ZeroType: - if node.depth == 0: - return none[(MerkleNode, MerkleNode)]() - else: - return some((zNodes[node.depth - 1], zNodes[node.depth - 1])) - -func hash*(node: MerkleNode): Digest = - case node.kind - of LeafType: - node.digest - of NodeType: - node.innerDigest - of ZeroType: - zeroHashes[node.depth] - -func getCapacityAtDepth(depth: uint64): uint64 = - uint64 math.pow(2'f64, float64 depth) - -func createTree*(leaves: openArray[Digest], depth: uint64): MerkleNode = - if len(leaves) == 0: - return MerkleNode(kind: ZeroType, depth: depth) - elif depth == 0: - return MerkleNode(kind: LeafType, digest: leaves[0]) - else: - let nexLevelDepth = depth - 1 - let subCap = getCapacityAtDepth(nexLevelDepth) - let (left, right) = splitLeaves(leaves, subCap) - let leftTree = createTree(left, nexLevelDepth) - let rightTree = createTree(right, nexLevelDepth) - let finalHash = mergeBranches(leftTree.hash(), rightTree.hash()) - return MerkleNode(kind: NodeType, innerDigest: finalHash, left: leftTree, right: rightTree) - -proc genProof*(tree: MerkleNode, idx: uint64, treeDepth: uint64): seq[Digest] = - var proof = newSeq[Digest]() - var currNode = tree - var currDepth = treeDepth - while currDepth > 0: - let ithBit = (idx shr (currDepth - 1)) and 1 - # should be safe to call unsafeGet() as leaves are on lowest level, and depth is - # always larger than 0 - let (left, right) = getSubTrees(currNode).unsafeGet() - if ithBit == 1: - proof.add(left.hash()) - currNode = right - else: - proof.add(right.hash()) - currNode = left - currDepth = currDepth - 1 - - proof.reverse() - proof - -# TODO add method to add leaf to the exisiting tree diff --git a/eth/ssz/merkleization.nim b/eth/ssz/merkleization.nim deleted file mode 100644 index 68f4740..0000000 --- a/eth/ssz/merkleization.nim +++ /dev/null @@ -1,660 +0,0 @@ -# ssz_serialization -# Copyright (c) 2018-2021 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -# This module contains the parts necessary to create a merkle hash from the core -# SSZ types outlined in the spec: -# https://github.com/ethereum/consensus-specs/blob/v1.0.1/ssz/simple-serialize.md#merkleization - -{.push raises: [Defect].} - -import - math, sequtils, - stew/[bitops2, endians2, ptrops], - stew/ranges/ptr_arith, nimcrypto/[hash, sha2], - serialization/testing/tracing, - "."/[bitseqs, types] - -export - types - -when hasSerializationTracing: - import stew/byteutils, typetraits - -const - zero64 = default array[64, byte] - bitsPerChunk = bytesPerChunk * 8 - -func binaryTreeHeight*(totalElements: Limit): int = - bitWidth nextPow2(uint64 totalElements) - -type - SszMerkleizerImpl = object - combinedChunks: ptr UncheckedArray[Digest] - totalChunks: uint64 - topIndex: int - - SszMerkleizer*[limit: static[Limit]] = object - combinedChunks: ref array[binaryTreeHeight limit, Digest] - impl: SszMerkleizerImpl - -template chunks*(m: SszMerkleizerImpl): openArray[Digest] = - m.combinedChunks.toOpenArray(0, m.topIndex) - -template getChunkCount*(m: SszMerkleizer): uint64 = - m.impl.totalChunks - -template getCombinedChunks*(m: SszMerkleizer): openArray[Digest] = - toOpenArray(m.impl.combinedChunks, 0, m.impl.topIndex) - -type DigestCtx* = sha2.sha256 - -template computeDigest*(body: untyped): Digest = - ## This little helper will init the hash function and return the sliced - ## hash: - ## let hashOfData = withHash: h.update(data) - when nimvm: - # In SSZ, computeZeroHashes require compile-time SHA256 - block: - var h {.inject.}: sha256 - init(h) - body - finish(h) - else: - block: - var h {.inject, noInit.}: DigestCtx - init(h) - body - finish(h) - -func digest(a: openArray[byte]): Digest = - result = computeDigest: - h.update(a) - -func digest(a, b: openArray[byte]): Digest = - result = computeDigest: - trs "DIGESTING ARRAYS ", toHex(a), " ", toHex(b) - trs toHex(a) - trs toHex(b) - - h.update a - h.update b - trs "HASH RESULT ", result - -func digest(a, b, c: openArray[byte]): Digest = - result = computeDigest: - trs "DIGESTING ARRAYS ", toHex(a), " ", toHex(b), " ", toHex(c) - - h.update a - h.update b - h.update c - trs "HASH RESULT ", result - -func mergeBranches(existing: Digest, newData: openArray[byte]): Digest = - trs "MERGING BRANCHES OPEN ARRAY" - - let paddingBytes = bytesPerChunk - newData.len - digest(existing.data, newData, zero64.toOpenArray(0, paddingBytes - 1)) - -template mergeBranches(existing: Digest, newData: array[32, byte]): Digest = - trs "MERGING BRANCHES ARRAY" - digest(existing.data, newData) - -template mergeBranches*(a, b: Digest): Digest = - trs "MERGING BRANCHES DIGEST" - digest(a.data, b.data) - -func computeZeroHashes: array[sizeof(Limit) * 8, Digest] = - result[0] = Digest() - for i in 1 .. result.high: - result[i] = mergeBranches(result[i - 1], result[i - 1]) - -const zeroHashes* = computeZeroHashes() - -func addChunk*(merkleizer: var SszMerkleizerImpl, data: openArray[byte]) = - doAssert data.len > 0 and data.len <= bytesPerChunk - - if getBitLE(merkleizer.totalChunks, 0): - var hash = mergeBranches(merkleizer.combinedChunks[0], data) - - for i in 1 .. merkleizer.topIndex: - trs "ITERATING" - if getBitLE(merkleizer.totalChunks, i): - trs "CALLING MERGE BRANCHES" - hash = mergeBranches(merkleizer.combinedChunks[i], hash) - else: - trs "WRITING FRESH CHUNK AT ", i, " = ", hash - merkleizer.combinedChunks[i] = hash - break - else: - let paddingBytes = bytesPerChunk - data.len - - merkleizer.combinedChunks[0].data[0.. 0 and merkleizer.topIndex > 0 - - let proofHeight = merkleizer.topIndex + 1 - result = newSeq[Digest](chunks.len * proofHeight) - - if chunks.len == 1: - merkleizer.addChunkAndGenMerkleProof(chunks[0], result) - return - - let newTotalChunks = merkleizer.totalChunks + chunks.len.uint64 - - var - # A perfect binary tree will take either `chunks.len * 2` values if the - # number of elements in the base layer is odd and `chunks.len * 2 - 1` - # otherwise. Each row may also need a single extra element at most if - # it must be combined with the existing values in the Merkleizer: - merkleTree = newSeqOfCap[Digest](chunks.len + merkleizer.topIndex) - inRowIdx = merkleizer.totalChunks - postUpdateInRowIdx = newTotalChunks - zeroMixed = false - - template writeResult(chunkIdx, level: int, chunk: Digest) = - result[chunkIdx * proofHeight + level] = chunk - - # We'll start by generating the first row of the merkle tree. - var currPairEnd = if inRowIdx.isOdd: - # an odd chunk number means that we must combine the - # hash with the existing pending sibling hash in the - # merkleizer. - writeResult(0, 0, merkleizer.combinedChunks[0]) - merkleTree.add mergeBranches(merkleizer.combinedChunks[0], chunks[0]) - - # TODO: can we immediately write this out? - merkleizer.completeStartedChunk(merkleTree[^1], 1) - 2 - else: - 1 - - if postUpdateInRowIdx.isOdd: - merkleizer.combinedChunks[0] = chunks[^1] - - while currPairEnd < chunks.len: - writeResult(currPairEnd - 1, 0, chunks[currPairEnd]) - writeResult(currPairEnd, 0, chunks[currPairEnd - 1]) - merkleTree.add mergeBranches(chunks[currPairEnd - 1], - chunks[currPairEnd]) - currPairEnd += 2 - - if currPairEnd - 1 < chunks.len: - zeroMixed = true - writeResult(currPairEnd - 1, 0, zeroHashes[0]) - merkleTree.add mergeBranches(chunks[currPairEnd - 1], - zeroHashes[0]) - var - level = 0 - baseChunksPerElement = 1 - treeRowStart = 0 - rowLen = merkleTree.len - - template writeProofs(rowChunkIdx: int, hash: Digest) = - let - startAbsIdx = (inRowIdx.int + rowChunkIdx) * baseChunksPerElement - endAbsIdx = startAbsIdx + baseChunksPerElement - startResIdx = max(startAbsIdx - merkleizer.totalChunks.int, 0) - endResIdx = min(endAbsIdx - merkleizer.totalChunks.int, chunks.len) - - for resultPos in startResIdx ..< endResIdx: - writeResult(resultPos, level, hash) - - if rowLen > 1: - while level < merkleizer.topIndex: - inc level - baseChunksPerElement *= 2 - inRowIdx = inRowIdx div 2 - postUpdateInRowIdx = postUpdateInRowIdx div 2 - - var currPairEnd = if inRowIdx.isOdd: - # an odd chunk number means that we must combine the - # hash with the existing pending sibling hash in the - # merkleizer. - writeProofs(0, merkleizer.combinedChunks[level]) - merkleTree.add mergeBranches(merkleizer.combinedChunks[level], - merkleTree[treeRowStart]) - - # TODO: can we immediately write this out? - merkleizer.completeStartedChunk(merkleTree[^1], level + 1) - 2 - else: - 1 - - if postUpdateInRowIdx.isOdd: - merkleizer.combinedChunks[level] = merkleTree[treeRowStart + rowLen - - ord(zeroMixed) - 1] - while currPairEnd < rowLen: - writeProofs(currPairEnd - 1, merkleTree[treeRowStart + currPairEnd]) - writeProofs(currPairEnd, merkleTree[treeRowStart + currPairEnd - 1]) - merkleTree.add mergeBranches(merkleTree[treeRowStart + currPairEnd - 1], - merkleTree[treeRowStart + currPairEnd]) - currPairEnd += 2 - - if currPairEnd - 1 < rowLen: - zeroMixed = true - writeProofs(currPairEnd - 1, zeroHashes[level]) - merkleTree.add mergeBranches(merkleTree[treeRowStart + currPairEnd - 1], - zeroHashes[level]) - - treeRowStart += rowLen - rowLen = merkleTree.len - treeRowStart - - if rowLen == 1: - break - - doAssert rowLen == 1 - - if (inRowIdx and 2) != 0: - merkleizer.completeStartedChunk( - mergeBranches(merkleizer.combinedChunks[level + 1], merkleTree[^1]), - level + 2) - - if (not zeroMixed) and (postUpdateInRowIdx and 2) != 0: - merkleizer.combinedChunks[level + 1] = merkleTree[^1] - - while level < merkleizer.topIndex: - inc level - baseChunksPerElement *= 2 - inRowIdx = inRowIdx div 2 - - let hash = if getBitLE(merkleizer.totalChunks, level): - merkleizer.combinedChunks[level] - else: - zeroHashes[level] - - writeProofs(0, hash) - - merkleizer.totalChunks = newTotalChunks - -proc init*(S: type SszMerkleizer): S = - new result.combinedChunks - result.impl = SszMerkleizerImpl( - combinedChunks: cast[ptr UncheckedArray[Digest]]( - addr result.combinedChunks[][0]), - topIndex: binaryTreeHeight(result.limit) - 1, - totalChunks: 0) - -proc init*(S: type SszMerkleizer, - combinedChunks: openArray[Digest], - totalChunks: uint64): S = - new result.combinedChunks - result.combinedChunks[][0 ..< combinedChunks.len] = combinedChunks - result.impl = SszMerkleizerImpl( - combinedChunks: cast[ptr UncheckedArray[Digest]]( - addr result.combinedChunks[][0]), - topIndex: binaryTreeHeight(result.limit) - 1, - totalChunks: totalChunks) - -proc copy*[L: static[Limit]](cloned: SszMerkleizer[L]): SszMerkleizer[L] = - new result.combinedChunks - result.combinedChunks[] = cloned.combinedChunks[] - result.impl = SszMerkleizerImpl( - combinedChunks: cast[ptr UncheckedArray[Digest]]( - addr result.combinedChunks[][0]), - topIndex: binaryTreeHeight(L) - 1, - totalChunks: cloned.totalChunks) - -template addChunksAndGenMerkleProofs*( - merkleizer: var SszMerkleizer, - chunks: openArray[Digest]): seq[Digest] = - addChunksAndGenMerkleProofs(merkleizer.impl, chunks) - -template addChunk*(merkleizer: var SszMerkleizer, data: openArray[byte]) = - addChunk(merkleizer.impl, data) - -template totalChunks*(merkleizer: SszMerkleizer): uint64 = - merkleizer.impl.totalChunks - -template getFinalHash*(merkleizer: SszMerkleizer): Digest = - merkleizer.impl.getFinalHash - -template createMerkleizer*(totalElements: static Limit): SszMerkleizerImpl = - trs "CREATING A MERKLEIZER FOR ", totalElements - - const treeHeight = binaryTreeHeight totalElements - var combinedChunks {.noInit.}: array[treeHeight, Digest] - - let topIndex = treeHeight - 1 - - SszMerkleizerImpl( - combinedChunks: cast[ptr UncheckedArray[Digest]](addr combinedChunks), - topIndex: if (topIndex < 0): 0 else: topIndex, - totalChunks: 0) - -func getFinalHash*(merkleizer: SszMerkleizerImpl): Digest = - if merkleizer.totalChunks == 0: - return zeroHashes[merkleizer.topIndex] - - let - bottomHashIdx = firstOne(merkleizer.totalChunks) - 1 - submittedChunksHeight = bitWidth(merkleizer.totalChunks - 1) - topHashIdx = merkleizer.topIndex - - trs "BOTTOM HASH ", bottomHashIdx - trs "SUBMITTED HEIGHT ", submittedChunksHeight - trs "TOP HASH IDX ", topHashIdx - - if bottomHashIdx != submittedChunksHeight: - # Our tree is not finished. We must complete the work in progress - # branches and then extend the tree to the right height. - result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx], - zeroHashes[bottomHashIdx]) - - for i in bottomHashIdx + 1 ..< topHashIdx: - if getBitLE(merkleizer.totalChunks, i): - result = mergeBranches(merkleizer.combinedChunks[i], result) - trs "COMBINED" - else: - result = mergeBranches(result, zeroHashes[i]) - trs "COMBINED WITH ZERO" - - elif bottomHashIdx == topHashIdx: - # We have a perfect tree (chunks == 2**n) at just the right height! - result = merkleizer.combinedChunks[bottomHashIdx] - else: - # We have a perfect tree of user chunks, but we have more work to - # do - we must extend it to reach the desired height - result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx], - zeroHashes[bottomHashIdx]) - - for i in bottomHashIdx + 1 ..< topHashIdx: - result = mergeBranches(result, zeroHashes[i]) - -func mixInLength*(root: Digest, length: int): Digest = - var dataLen: array[32, byte] - dataLen[0..<8] = uint64(length).toBytesLE() - mergeBranches(root, dataLen) - -func hash_tree_root*(x: auto): Digest {.gcsafe, raises: [Defect].} - -template merkleizeFields(totalElements: static Limit, body: untyped): Digest = - var merkleizer {.inject.} = createMerkleizer(totalElements) - - template addField(field) = - let hash = hash_tree_root(field) - trs "MERKLEIZING FIELD ", astToStr(field), " = ", hash - addChunk(merkleizer, hash.data) - trs "CHUNK ADDED" - - body - - getFinalHash(merkleizer) - -template writeBytesLE(chunk: var array[bytesPerChunk, byte], atParam: int, - val: SomeUnsignedInt) = - let at = atParam - chunk[at ..< at + sizeof(val)] = toBytesLE(val) - -func chunkedHashTreeRootForBasicTypes[T](merkleizer: var SszMerkleizerImpl, - arr: openArray[T]): Digest = - static: - doAssert T is BasicType - doAssert bytesPerChunk mod sizeof(T) == 0 - - if arr.len == 0: - return getFinalHash(merkleizer) - - when sizeof(T) == 1 or cpuEndian == littleEndian: - var - remainingBytes = when sizeof(T) == 1: arr.len - else: arr.len * sizeof(T) - pos = cast[ptr byte](unsafeAddr arr[0]) - - while remainingBytes >= bytesPerChunk: - merkleizer.addChunk(makeOpenArray(pos, bytesPerChunk)) - pos = offset(pos, bytesPerChunk) - remainingBytes -= bytesPerChunk - - if remainingBytes > 0: - merkleizer.addChunk(makeOpenArray(pos, remainingBytes)) - - else: - const valuesPerChunk = bytesPerChunk div sizeof(T) - - var writtenValues = 0 - - var chunk: array[bytesPerChunk, byte] - while writtenValues < arr.len - valuesPerChunk: - for i in 0 ..< valuesPerChunk: - chunk.writeBytesLE(i * sizeof(T), arr[writtenValues + i]) - merkleizer.addChunk chunk - inc writtenValues, valuesPerChunk - - let remainingValues = arr.len - writtenValues - if remainingValues > 0: - var lastChunk: array[bytesPerChunk, byte] - for i in 0 ..< remainingValues: - lastChunk.writeBytesLE(i * sizeof(T), arr[writtenValues + i]) - merkleizer.addChunk lastChunk - - getFinalHash(merkleizer) - -func bitListHashTreeRoot(merkleizer: var SszMerkleizerImpl, x: BitSeq): Digest = - # TODO: Switch to a simpler BitList representation and - # replace this with `chunkedHashTreeRoot` - var - totalBytes = bytes(x).len - lastCorrectedByte = bytes(x)[^1] - - if lastCorrectedByte == byte(1): - if totalBytes == 1: - # This is an empty bit list. - # It should be hashed as a tree containing all zeros: - return mergeBranches(zeroHashes[merkleizer.topIndex], - zeroHashes[0]) # this is the mixed length - - totalBytes -= 1 - lastCorrectedByte = bytes(x)[^2] - else: - let markerPos = log2trunc(lastCorrectedByte) - lastCorrectedByte.clearBit(markerPos) - - var - bytesInLastChunk = totalBytes mod bytesPerChunk - fullChunks = totalBytes div bytesPerChunk - - if bytesInLastChunk == 0: - fullChunks -= 1 - bytesInLastChunk = 32 - - for i in 0 ..< fullChunks: - let - chunkStartPos = i * bytesPerChunk - chunkEndPos = chunkStartPos + bytesPerChunk - 1 - - merkleizer.addChunk bytes(x).toOpenArray(chunkStartPos, chunkEndPos) - - var - lastChunk: array[bytesPerChunk, byte] - chunkStartPos = fullChunks * bytesPerChunk - - for i in 0 .. bytesInLastChunk - 2: - lastChunk[i] = bytes(x)[chunkStartPos + i] - - lastChunk[bytesInLastChunk - 1] = lastCorrectedByte - - merkleizer.addChunk lastChunk.toOpenArray(0, bytesInLastChunk - 1) - let contentsHash = merkleizer.getFinalHash - mixInLength contentsHash, x.len - -func maxChunksCount(T: type, maxLen: Limit): Limit = - when T is BitList|BitArray: - (maxLen + bitsPerChunk - 1) div bitsPerChunk - elif T is array|List: - maxChunkIdx(ElemType(T), maxLen) - else: - unsupported T # This should never happen - -func hashTreeRootAux[T](x: T): Digest = - when T is bool|char: - result.data[0] = byte(x) - elif T is SomeUnsignedInt: - when cpuEndian == bigEndian: - result.data[0.. 0 - -# validates merkle proof. Provided index should be a generalized index of leaf node -# as defined in: https://github.com/ethereum/consensus-specs/blob/dev/ssz/merkle-proofs.md#generalized-merkle-tree-index -func isValidProof*(leaf: Digest, proof: openArray[Digest], - index: uint64, root: Digest): bool = - if len(proof) == getGeneralizedIndexLength(index): - var - value = leaf - - for i, digest in proof: - value = - if getGeneralizedIndexBit(index, uint64 i): - mergeBranches(digest, value) - else: - mergeBranches(value, digest) - - value == root - else: - false - -proc slice[T](x: openArray[T]): seq[T] = x.toSeq() - -# Helper functions to get proof for any element of a list -proc getProofForAllListElements*(list: List): seq[Digest] = - type T = type(list) - type E = ElemType(T) - # basic types have different chunking rules - static: - doAssert (E is not BasicType) - var digests: seq[Digest] = @[] - for e in list: - let root = hash_tree_root(e) - digests.add(root) - var merk = createMerkleizer(list.maxLen) - merk.addChunksAndGenMerkleProofs(digests) - -proc getProofWithIdx*(list: List, allProofs: seq[Digest], idx: int): seq[Digest] = - let treeHeight = binaryTreeHeight(list.maxLen) - let startPos = idx * treeHeight - let endPos = startPos + treeHeight - 2 - slice(allProofs.toOpenArray(startPos, endPos)) - -proc generateAndGetProofWithIdx*(list: List, idx: int): seq[Digest] = - let allProofs = getProofForAllListElements(list) - getProofWithIdx(list, allProofs, idx) diff --git a/eth/ssz/ssz_serialization.nim b/eth/ssz/ssz_serialization.nim deleted file mode 100644 index 88fa7c7..0000000 --- a/eth/ssz/ssz_serialization.nim +++ /dev/null @@ -1,247 +0,0 @@ -# nim-eth - Limited SSZ implementation -# Copyright (c) 2018-2021 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -{.push raises: [Defect].} - -## SSZ serialization for core SSZ types, as specified in: -# https://github.com/ethereum/consensus-specs/blob/v1.0.1/ssz/simple-serialize.md#serialization - -import - std/[typetraits, options], - stew/[endians2, leb128, objects], - serialization, serialization/testing/tracing, - ./bytes_reader, ./types - -export - serialization, types, bytes_reader - -type - SszReader* = object - stream: InputStream - - SszWriter* = object - stream: OutputStream - - SizePrefixed*[T] = distinct T - SszMaxSizeExceeded* = object of SerializationError - - VarSizedWriterCtx = object - fixedParts: WriteCursor - offset: int - - FixedSizedWriterCtx = object - -serializationFormat SSZ - -SSZ.setReader SszReader -SSZ.setWriter SszWriter, PreferredOutput = seq[byte] - -template sizePrefixed*[TT](x: TT): untyped = - type T = TT - SizePrefixed[T](x) - -proc init*(T: type SszReader, stream: InputStream): T {.raises: [Defect].} = - T(stream: stream) - -proc writeFixedSized(s: var (OutputStream|WriteCursor), x: auto) - {.raises: [Defect, IOError].} = - mixin toSszType - - when x is byte: - s.write x - elif x is bool: - s.write byte(ord(x)) - elif x is UintN: - when cpuEndian == bigEndian: - s.write toBytesLE(x) - else: - s.writeMemCopy x - elif x is array: - when x[0] is byte: - trs "APPENDING FIXED SIZE BYTES", x - s.write x - else: - for elem in x: - trs "WRITING FIXED SIZE ARRAY ELEMENT" - s.writeFixedSized toSszType(elem) - elif x is tuple|object: - enumInstanceSerializedFields(x, fieldName, field): - trs "WRITING FIXED SIZE FIELD", fieldName - s.writeFixedSized toSszType(field) - else: - unsupported x.type - -template writeOffset(cursor: var WriteCursor, offset: int) = - write cursor, toBytesLE(uint32 offset) - -template supports*(_: type SSZ, T: type): bool = - mixin toSszType - anonConst compiles(fixedPortionSize toSszType(declval T)) - -func init*(T: type SszWriter, stream: OutputStream): T {.raises: [Defect].} = - result.stream = stream - -proc writeVarSizeType(w: var SszWriter, value: auto) - {.gcsafe, raises: [Defect, IOError].} - -proc beginRecord*(w: var SszWriter, TT: type): auto {.raises: [Defect].} = - type T = TT - when isFixedSize(T): - FixedSizedWriterCtx() - else: - const offset = when T is array: len(T) * offsetSize - else: fixedPortionSize(T) - VarSizedWriterCtx(offset: offset, - fixedParts: w.stream.delayFixedSizeWrite(offset)) - -template writeField*(w: var SszWriter, - ctx: var auto, - fieldName: string, - field: auto) = - mixin toSszType - when ctx is FixedSizedWriterCtx: - writeFixedSized(w.stream, toSszType(field)) - else: - type FieldType = type toSszType(field) - - when isFixedSize(FieldType): - writeFixedSized(ctx.fixedParts, toSszType(field)) - else: - trs "WRITING OFFSET ", ctx.offset, " FOR ", fieldName - writeOffset(ctx.fixedParts, ctx.offset) - let initPos = w.stream.pos - trs "WRITING VAR SIZE VALUE OF TYPE ", name(FieldType) - when FieldType is BitList: - trs "BIT SEQ ", bytes(field) - writeVarSizeType(w, toSszType(field)) - ctx.offset += w.stream.pos - initPos - -template endRecord*(w: var SszWriter, ctx: var auto) = - when ctx is VarSizedWriterCtx: - finalize ctx.fixedParts - -proc writeSeq[T](w: var SszWriter, value: seq[T]) - {.raises: [Defect, IOError].} = - # Please note that `writeSeq` exists in order to reduce the code bloat - # produced from generic instantiations of the unique `List[N, T]` types. - when isFixedSize(T): - trs "WRITING LIST WITH FIXED SIZE ELEMENTS" - for elem in value: - w.stream.writeFixedSized toSszType(elem) - trs "DONE" - else: - trs "WRITING LIST WITH VAR SIZE ELEMENTS" - var offset = value.len * offsetSize - var cursor = w.stream.delayFixedSizeWrite offset - for elem in value: - cursor.writeFixedSized uint32(offset) - let initPos = w.stream.pos - w.writeVarSizeType toSszType(elem) - offset += w.stream.pos - initPos - finalize cursor - trs "DONE" - -proc writeVarSizeType(w: var SszWriter, value: auto) - {.raises: [Defect, IOError].} = - trs "STARTING VAR SIZE TYPE" - - when value is List: - # We reduce code bloat by forwarding all `List` types to a general `seq[T]` - # proc. - writeSeq(w, asSeq value) - elif value is BitList: - # ATTENTION! We can reuse `writeSeq` only as long as our BitList type is - # implemented to internally match the binary representation of SSZ BitLists - # in memory. - writeSeq(w, bytes value) - elif value is object|tuple|array: - trs "WRITING OBJECT OR ARRAY" - var ctx = beginRecord(w, type value) - enumerateSubFields(value, field): - writeField w, ctx, astToStr(field), field - endRecord w, ctx - else: - unsupported type(value) - -proc writeValue*(w: var SszWriter, x: auto) - {.gcsafe, raises: [Defect, IOError].} = - mixin toSszType - type T = type toSszType(x) - - when isFixedSize(T): - w.stream.writeFixedSized toSszType(x) - else: - w.writeVarSizeType toSszType(x) - -func sszSize*(value: auto): int {.gcsafe, raises: [Defect].} - -func sszSizeForVarSizeList[T](value: openArray[T]): int = - mixin toSszType - result = len(value) * offsetSize - for elem in value: - result += sszSize(toSszType elem) - -func sszSize*(value: auto): int {.gcsafe, raises: [Defect].} = - mixin toSszType - type T = type toSszType(value) - - when isFixedSize(T): - anonConst fixedPortionSize(T) - - elif T is array|List: - type E = ElemType(T) - when isFixedSize(E): - len(value) * anonConst(fixedPortionSize(E)) - elif T is HashArray: - sszSizeForVarSizeList(value.data) - elif T is array: - sszSizeForVarSizeList(value) - else: - sszSizeForVarSizeList(asSeq value) - - elif T is BitList: - return len(bytes(value)) - - elif T is object|tuple: - result = anonConst fixedPortionSize(T) - enumInstanceSerializedFields(value, _{.used.}, field): - type FieldType = type toSszType(field) - when not isFixedSize(FieldType): - result += sszSize(toSszType field) - - else: - unsupported T - -proc writeValue*[T](w: var SszWriter, x: SizePrefixed[T]) - {.raises: [Defect, IOError].} = - var cursor = w.stream.delayVarSizeWrite(Leb128.maxLen(uint64)) - let initPos = w.stream.pos - w.writeValue T(x) - let length = toBytes(uint64(w.stream.pos - initPos), Leb128) - cursor.finalWrite length.toOpenArray() - -proc readValue*[T](r: var SszReader, val: var T) - {.raises: [Defect, SszError, IOError].} = - when isFixedSize(T): - const minimalSize = fixedPortionSize(T) - if r.stream.readable(minimalSize): - readSszValue(r.stream.read(minimalSize), val) - else: - raise newException(MalformedSszError, "SSZ input of insufficient size") - else: - # TODO(zah) Read the fixed portion first and precisely measure the - # size of the dynamic portion to consume the right number of bytes. - readSszValue(r.stream.read(r.stream.len.get), val) - -proc readSszBytes*[T](data: openArray[byte], val: var T) {. - raises: [Defect, MalformedSszError, SszSizeMismatchError].} = - when isFixedSize(T): - const minimalSize = fixedPortionSize(T) - if data.len < minimalSize: - raise newException(MalformedSszError, "SSZ input of insufficient size") - - readSszValue(data, val) diff --git a/eth/ssz/types.nim b/eth/ssz/types.nim deleted file mode 100644 index 12aeec4..0000000 --- a/eth/ssz/types.nim +++ /dev/null @@ -1,302 +0,0 @@ -# nim-eth - Limited SSZ implementation -# Copyright (c) 2018-2021 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -{.push raises: [Defect].} - -import - std/[tables, options, typetraits, strformat], - stew/shims/macros, stew/[byteutils, bitops2, objects], - nimcrypto/hash, serialization/[object_serialization, errors], - ./bitseqs - -export bitseqs - -const - offsetSize* = 4 - bytesPerChunk* = 32 - -type - UintN* = SomeUnsignedInt - BasicType* = bool|UintN - - Limit* = int64 - - List*[T; maxLen: static Limit] = distinct seq[T] - BitList*[maxLen: static Limit] = distinct BitSeq - Digest* = MDigest[32 * 8] - - # Note for readers: - # We use `array` for `Vector` and - # `BitArray` for `BitVector` - - SszError* = object of SerializationError - - MalformedSszError* = object of SszError - - SszSizeMismatchError* = object of SszError - deserializedType*: cstring - actualSszSize*: int - elementSize*: int - -# A few index types from here onwards: -# * dataIdx - leaf index starting from 0 to maximum length of collection -# * chunkIdx - leaf data index after chunking starting from 0 -# * vIdx - virtual index in merkle tree - the root is found at index 1, its -# two children at 2, 3 then 4, 5, 6, 7 etc - -func nextPow2Int64(x: int64): int64 = - # TODO the nextPow2 in bitops2 works with uint64 - there's a bug in the nim - # compiler preventing it to be used - it seems that a conversion to - # uint64 cannot be done with the static maxLen :( - var v = x - 1 - - # round down, make sure all bits are 1 below the threshold, then add 1 - v = v or v shr 1 - v = v or v shr 2 - v = v or v shr 4 - when bitsof(x) > 8: - v = v or v shr 8 - when bitsof(x) > 16: - v = v or v shr 16 - when bitsof(x) > 32: - v = v or v shr 32 - - v + 1 - -template dataPerChunk(T: type): int = - # How many data items fit in a chunk - when T is BasicType: - bytesPerChunk div sizeof(T) - else: - 1 - -template chunkIdx*(T: type, dataIdx: int64): int64 = - # Given a data index, which chunk does it belong to? - dataIdx div dataPerChunk(T) - -template maxChunkIdx*(T: type, maxLen: Limit): int64 = - # Given a number of data items, how many chunks are needed? - # TODO compiler bug: - # beacon_chain/ssz/types.nim(75, 53) Error: cannot generate code for: maxLen - # nextPow2(chunkIdx(T, maxLen + dataPerChunk(T) - 1).uint64).int64 - nextPow2Int64(chunkIdx(T, maxLen.int64 + dataPerChunk(T) - 1)) - -template asSeq*(x: List): auto = distinctBase(x) - -template init*[T](L: type List, x: seq[T], N: static Limit): auto = - List[T, N](x) - -template init*[T, N](L: type List[T, N], x: seq[T]): auto = - List[T, N](x) - -template `$`*(x: List): auto = $(distinctBase x) -template len*(x: List): auto = len(distinctBase x) -template low*(x: List): auto = low(distinctBase x) -template high*(x: List): auto = high(distinctBase x) -template `[]`*(x: List, idx: auto): untyped = distinctBase(x)[idx] -template `[]=`*(x: var List, idx: auto, val: auto) = distinctBase(x)[idx] = val -template `==`*(a, b: List): bool = distinctBase(a) == distinctBase(b) - -template `&`*(a, b: List): auto = (type(a)(distinctBase(a) & distinctBase(b))) - -template items* (x: List): untyped = items(distinctBase x) -template pairs* (x: List): untyped = pairs(distinctBase x) -template mitems*(x: var List): untyped = mitems(distinctBase x) -template mpairs*(x: var List): untyped = mpairs(distinctBase x) - -template contains* (x: List, val: auto): untyped = contains(distinctBase x, val) - -proc add*(x: var List, val: auto): bool = - if x.len < x.maxLen: - add(distinctBase x, val) - true - else: - false - -proc setLen*(x: var List, newLen: int): bool = - if newLen <= x.maxLen: - setLen(distinctBase x, newLen) - true - else: - false - -template init*(L: type BitList, x: seq[byte], N: static Limit): auto = - BitList[N](data: x) - -template init*[N](L: type BitList[N], x: seq[byte]): auto = - L(data: x) - -template init*(T: type BitList, len: int): auto = T init(BitSeq, len) -template len*(x: BitList): auto = len(BitSeq(x)) -template bytes*(x: BitList): auto = seq[byte](x) -template `[]`*(x: BitList, idx: auto): auto = BitSeq(x)[idx] -template `[]=`*(x: var BitList, idx: auto, val: bool) = BitSeq(x)[idx] = val -template `==`*(a, b: BitList): bool = BitSeq(a) == BitSeq(b) -template setBit*(x: var BitList, idx: Natural) = setBit(BitSeq(x), idx) -template clearBit*(x: var BitList, idx: Natural) = clearBit(BitSeq(x), idx) -template overlaps*(a, b: BitList): bool = overlaps(BitSeq(a), BitSeq(b)) -template incl*(a: var BitList, b: BitList) = incl(BitSeq(a), BitSeq(b)) -template isSubsetOf*(a, b: BitList): bool = isSubsetOf(BitSeq(a), BitSeq(b)) -template isZeros*(x: BitList): bool = isZeros(BitSeq(x)) -template countOnes*(x: BitList): int = countOnes(BitSeq(x)) -template countZeros*(x: BitList): int = countZeros(BitSeq(x)) -template countOverlap*(x, y: BitList): int = countOverlap(BitSeq(x), BitSeq(y)) -template `$`*(a: BitList): string = $(BitSeq(a)) - -iterator items*(x: BitList): bool = - for i in 0 ..< x.len: - yield x[i] - -macro unsupported*(T: typed): untyped = - # TODO: {.fatal.} breaks compilation even in `compiles()` context, - # so we use this macro instead. It's also much better at figuring - # out the actual type that was used in the instantiation. - # File both problems as issues. - error "SSZ serialization of the type " & humaneTypeName(T) & " is not supported" - -template ElemType*(T: type array): untyped = - type(default(T)[low(T)]) - -template ElemType*(T: type seq): untyped = - type(default(T)[0]) - -template ElemType*(T0: type List): untyped = - T0.T - -func isFixedSize*(T0: type): bool {.compileTime.} = - mixin toSszType, enumAllSerializedFields - - type T = type toSszType(declval T0) - - when T is BasicType: - return true - elif T is array: - return isFixedSize(ElemType(T)) - elif T is object|tuple: - enumAllSerializedFields(T): - when not isFixedSize(FieldType): - return false - return true - -func fixedPortionSize*(T0: type): int {.compileTime.} = - mixin enumAllSerializedFields, toSszType - - type T = type toSszType(declval T0) - - when T is BasicType: sizeof(T) - elif T is array: - type E = ElemType(T) - when isFixedSize(E): int(len(T)) * fixedPortionSize(E) - else: int(len(T)) * offsetSize - elif T is object|tuple: - enumAllSerializedFields(T): - when isFixedSize(FieldType): - result += fixedPortionSize(FieldType) - else: - result += offsetSize - else: - unsupported T0 - -# TODO This should have been an iterator, but the VM can't compile the -# code due to "too many registers required". -proc fieldInfos*(RecordType: type): seq[tuple[name: string, - offset: int, - fixedSize: int, - branchKey: string]] = - mixin enumAllSerializedFields - - var - offsetInBranch = {"": 0}.toTable - nestedUnder = initTable[string, string]() - - enumAllSerializedFields(RecordType): - const - isFixed = isFixedSize(FieldType) - fixedSize = when isFixed: fixedPortionSize(FieldType) - else: 0 - branchKey = when fieldCaseDiscriminator.len == 0: "" - else: fieldCaseDiscriminator & ":" & $fieldCaseBranches - fieldSize = when isFixed: fixedSize - else: offsetSize - - nestedUnder[fieldName] = branchKey - - var fieldOffset: int - offsetInBranch.withValue(branchKey, val): - fieldOffset = val[] - val[] += fieldSize - do: - try: - let parentBranch = nestedUnder.getOrDefault(fieldCaseDiscriminator, "") - fieldOffset = offsetInBranch[parentBranch] - offsetInBranch[branchKey] = fieldOffset + fieldSize - except KeyError as e: - raiseAssert e.msg - - result.add((fieldName, fieldOffset, fixedSize, branchKey)) - -func getFieldBoundingOffsetsImpl(RecordType: type, fieldName: static string): - tuple[fieldOffset, nextFieldOffset: int, isFirstOffset: bool] - {.compileTime.} = - result = (-1, -1, false) - var fieldBranchKey: string - var isFirstOffset = true - - for f in fieldInfos(RecordType): - if fieldName == f.name: - result[0] = f.offset - if f.fixedSize > 0: - result[1] = result[0] + f.fixedSize - return - else: - fieldBranchKey = f.branchKey - result.isFirstOffset = isFirstOffset - - elif result[0] != -1 and - f.fixedSize == 0 and - f.branchKey == fieldBranchKey: - # We have found the next variable sized field - result[1] = f.offset - return - - if f.fixedSize == 0: - isFirstOffset = false - -func getFieldBoundingOffsets*(RecordType: type, fieldName: static string): - tuple[fieldOffset, nextFieldOffset: int, isFirstOffset: bool] - {.compileTime.} = - ## Returns the start and end offsets of a field. - ## - ## For fixed-size fields, the start offset points to the first - ## byte of the field and the end offset points to 1 byte past the - ## end of the field. - ## - ## For variable-size fields, the returned offsets point to the - ## statically known positions of the 32-bit offset values written - ## within the SSZ object. You must read the 32-bit values stored - ## at the these locations in order to obtain the actual offsets. - ## - ## For variable-size fields, the end offset may be -1 when the - ## designated field is the last variable sized field within the - ## object. Then the SSZ object boundary known at run-time marks - ## the end of the variable-size field. - type T = RecordType - anonConst getFieldBoundingOffsetsImpl(T, fieldName) - -template enumerateSubFields*(holder, fieldVar, body: untyped) = - when holder is array: - for fieldVar in holder: body - else: - enumInstanceSerializedFields(holder, _{.used.}, fieldVar): body - -method formatMsg*( - err: ref SszSizeMismatchError, - filename: string): string {.gcsafe, raises: [Defect].} = - try: - &"SSZ size mismatch, element {err.elementSize}, actual {err.actualSszSize}, type {err.deserializedType}, file {filename}" - except CatchableError: - "SSZ size mismatch" diff --git a/tests/ssz/all_tests.nim b/tests/ssz/all_tests.nim deleted file mode 100644 index 2899b1c..0000000 --- a/tests/ssz/all_tests.nim +++ /dev/null @@ -1,3 +0,0 @@ -import - ./test_verification, - ./test_proofs diff --git a/tests/ssz/test_proofs.nim b/tests/ssz/test_proofs.nim deleted file mode 100644 index 6328b31..0000000 --- a/tests/ssz/test_proofs.nim +++ /dev/null @@ -1,123 +0,0 @@ -{.used.} - -import - sequtils, unittest, math, - nimcrypto/[hash, sha2], - stew/endians2, - ../eth/ssz/merkleization, - ../eth/ssz/ssz_serialization, - ../eth/ssz/merkle_tree - -template toSszType(x: auto): auto = - x - -proc h(a: openArray[byte]): Digest = - var h: sha256 - h.init() - h.update(a) - h.finish() - -type TestObject = object - digest: array[32, byte] - num: uint64 - -proc genObject(num: uint64): TestObject = - let numAsHash = h(num.toBytesLE()) - TestObject(digest: numAsHash.data, num: num) - -proc genNObjects(n: int): seq[TestObject] = - var objs = newSeq[TestObject]() - for i in 1..n: - let obj = genObject(uint64 i) - objs.add(obj) - objs - -proc getGenIndex(idx: int, depth: uint64): uint64 = - uint64 (math.pow(2'f64, float64 depth) + float64 idx) - -# Normal hash_tree_root add list length to final hash calculation. Proofs by default -# are generated without it. If necessary length of the list can be added manually -# at the end of the proof but here we are just hashing list with no mixin. -proc getListRootNoMixin(list: List): Digest = - var merk = createMerkleizer(list.maxLen) - for e in list: - let hash = hash_tree_root(e) - merk.addChunk(hash.data) - merk.getFinalHash() - -type TestCase = object - numOfElements: int - limit: int - -const TestCases = ( - TestCase(numOfElements: 0, limit: 2), - TestCase(numOfElements: 1, limit: 2), - TestCase(numOfElements: 2, limit: 2), - - TestCase(numOfElements: 0, limit: 4), - TestCase(numOfElements: 1, limit: 4), - TestCase(numOfElements: 2, limit: 4), - TestCase(numOfElements: 3, limit: 4), - TestCase(numOfElements: 4, limit: 4), - - TestCase(numOfElements: 0, limit: 8), - TestCase(numOfElements: 1, limit: 8), - TestCase(numOfElements: 2, limit: 8), - TestCase(numOfElements: 3, limit: 8), - TestCase(numOfElements: 4, limit: 8), - TestCase(numOfElements: 5, limit: 8), - TestCase(numOfElements: 6, limit: 8), - TestCase(numOfElements: 7, limit: 8), - TestCase(numOfElements: 8, limit: 8), - - TestCase(numOfElements: 0, limit: 16), - TestCase(numOfElements: 1, limit: 16), - TestCase(numOfElements: 2, limit: 16), - TestCase(numOfElements: 3, limit: 16), - TestCase(numOfElements: 4, limit: 16), - TestCase(numOfElements: 5, limit: 16), - TestCase(numOfElements: 6, limit: 16), - TestCase(numOfElements: 7, limit: 16), - TestCase(numOfElements: 16, limit: 16), - - TestCase(numOfElements: 32, limit: 32), - - TestCase(numOfElements: 64, limit: 64) -) - -suite "Merkle Proof generation": - test "generation of proof for various tree sizes": - for testCase in TestCases.fields: - let testObjects = genNObjects(testCase.numOfElements) - let treeDepth = uint64 binaryTreeHeight(testCase.limit) - 1 - - # Create List and and genereate root by using merkelizer - let list = List.init(testObjects, testCase.limit) - let listRoot = getListRootNoMixin(list) - - # Create sparse merkle tree from list elements and generate root - let listDigests = map(testObjects, proc(x: TestObject): Digest = hash_tree_root(x)) - let tree = createTree(listDigests, treeDepth) - let treeHash = tree.hash() - - # Assert that by using both methods we get same hash - check listRoot == treeHash - - for i, e in list: - # generate proof by using merkelizer - let merkleizerProof = generateAndGetProofWithIdx(list, i) - # generate proof by sparse merkle tree - let sparseTreeProof = genProof(tree, uint64 i, treeDepth) - - let leafHash = hash_tree_root(e) - let genIndex = getGenIndex(i, treeDepth) - - # both proof are valid. If both are valid that means that both proof are - # effectivly the same - let isValidProof = isValidProof(leafHash , merkleizerProof, genIndex, listRoot) - let isValidProof1 = isValidProof(leafHash , sparseTreeProof, genIndex, listRoot) - - check isValidProof - check isValidProof1 - - diff --git a/tests/ssz/test_verification.nim b/tests/ssz/test_verification.nim deleted file mode 100644 index 00c299b..0000000 --- a/tests/ssz/test_verification.nim +++ /dev/null @@ -1,82 +0,0 @@ -{.used.} - -import - sequtils, unittest, - nimcrypto/[hash, sha2], - ../eth/ssz/merkleization - -type TestCase = object - root: string - proof: seq[string] - leaf: string - index: uint64 - valid: bool - -let testCases = @[ - TestCase( - root: "2a23ef2b7a7221eaac2ffb3842a506a981c009ca6c2fcbf20adbc595e56f1a93", - proof: @[ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b" - ], - leaf: "0100000000000000000000000000000000000000000000000000000000000000", - index: 4, - valid: true - ), - TestCase( - root: "2a23ef2b7a7221eaac2ffb3842a506a981c009ca6c2fcbf20adbc595e56f1a93", - proof: @[ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b" - ], - leaf: "0100000000000000000000000000000000000000000000000000000000000000", - index: 6, - valid: false - ), - TestCase( - root: "2a23ef2b7a7221eaac2ffb3842a506a981c009ca6c2fcbf20adbc595e56f1a93", - proof: @[ - "0100000000000000000000000000000000000000000000000000000000000000", - "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b" - ], - leaf: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - index: 5, - valid: true - ), - TestCase( - root: "f1824b0084956084591ff4c91c11bcc94a40be82da280e5171932b967dd146e9", - proof: @[ - "35210d64853aee79d03f30cf0f29c1398706cbbcacaf05ab9524f00070aec91e", - "f38a181470ef1eee90a29f0af0a9dba6b7e5d48af3c93c29b4f91fa11b777582" - ], - leaf: "0100000000000000000000000000000000000000000000000000000000000000", - index: 7, - valid: true - ), - TestCase( - root: "f1824b0084956084591ff4c91c11bcc94a40be82da280e5171932b967dd146e9", - proof: @[ - "0000000000000000000000000000000000000000000000000000000000000000", - "0000000000000000000000000000000000000000000000000000000000000000", - "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b", - "0100000000000000000000000000000000000000000000000000000000000000", - "f38a181470ef1eee90a29f0af0a9dba6b7e5d48af3c93c29b4f91fa11b777582" - ], - leaf: "6001000000000000000000000000000000000000000000000000000000000000", - index: 49, - valid: true - ) -] - -suite "Merkle Proof verification": - test "correctly verify proof": - for testCase in testCases: - let root = MDigest[256].fromHex(testCase.root) - let proof = map(testCase.proof, proc(x: string): Digest = MDigest[256].fromHex(x)) - let leaf = MDigest[256].fromHex(testCase.leaf) - let valid = isValidProof(leaf, proof, testCase.index, root) - - if (testCase.valid): - check valid - else: - check (not valid)