From 03a70fbf36eb45f577f21f900ccae6d7538c9630 Mon Sep 17 00:00:00 2001 From: kdeme Date: Thu, 21 Oct 2021 22:27:06 +0200 Subject: [PATCH] Use nim-ssz-serialization module and rm local ssz code --- .gitmodules | 5 + beacon_chain/networking/network_metadata.nim | 2 +- beacon_chain/spec/datatypes/altair.nim | 2 +- beacon_chain/spec/datatypes/base.nim | 3 +- beacon_chain/spec/datatypes/merge.nim | 3 +- .../eth2_apis/eth2_json_rpc_serialization.nim | 2 +- beacon_chain/spec/eth2_merkleization.nim | 2 +- beacon_chain/spec/eth2_ssz_serialization.nim | 2 +- beacon_chain/spec/ssz_codec.nim | 2 +- beacon_chain/ssz/bitseqs.nim | 333 --------- beacon_chain/ssz/dynamic_navigator.nim | 163 ----- beacon_chain/ssz/merkleization.nim | 691 ------------------ beacon_chain/ssz/navigator.nim | 143 ---- beacon_chain/ssz/ssz_serialization.nim | 250 ------- tests/all_tests.nim | 2 - tests/helpers/debug_state.nim | 4 +- tests/ssz/all_tests.nim | 5 - tests/ssz/test_ssz_roundtrip.nim | 16 - tests/ssz/test_ssz_serialization.nim | 355 --------- tests/test_bitseqs.nim | 163 ----- vendor/nim-ssz-serialization | 1 + 21 files changed, 18 insertions(+), 2131 deletions(-) delete mode 100644 beacon_chain/ssz/bitseqs.nim delete mode 100644 beacon_chain/ssz/dynamic_navigator.nim delete mode 100644 beacon_chain/ssz/merkleization.nim delete mode 100644 beacon_chain/ssz/navigator.nim delete mode 100644 beacon_chain/ssz/ssz_serialization.nim delete mode 100644 tests/ssz/all_tests.nim delete mode 100644 tests/ssz/test_ssz_roundtrip.nim delete mode 100644 tests/ssz/test_ssz_serialization.nim delete mode 100644 tests/test_bitseqs.nim create mode 160000 vendor/nim-ssz-serialization diff --git a/.gitmodules b/.gitmodules index 29bb1fc78..6ae1c08f1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -209,3 +209,8 @@ [submodule "vendor/nim-taskpools"] path = vendor/nim-taskpools url = https://github.com/status-im/nim-taskpools +[submodule "vendor/nim-ssz-serialization"] + path = vendor/nim-ssz-serialization + url = https://github.com/status-im/nim-ssz-serialization.git + ignore = untracked + branch = master diff --git a/beacon_chain/networking/network_metadata.nim b/beacon_chain/networking/network_metadata.nim index 8f04ce2e7..0e0befc99 100644 --- a/beacon_chain/networking/network_metadata.nim +++ b/beacon_chain/networking/network_metadata.nim @@ -14,7 +14,7 @@ import web3/[ethtypes, conversions], chronicles, eth/common/eth_types_json_serialization, - ../ssz/[navigator], + ssz_serialization/navigator, ../spec/eth2_ssz_serialization, ../spec/datatypes/phase0 diff --git a/beacon_chain/spec/datatypes/altair.nim b/beacon_chain/spec/datatypes/altair.nim index b0d1633a5..ad5221484 100644 --- a/beacon_chain/spec/datatypes/altair.nim +++ b/beacon_chain/spec/datatypes/altair.nim @@ -33,7 +33,7 @@ import import ./base, ./phase0 export base -from ../../ssz/merkleization import GeneralizedIndex +from ssz_serialization/merkleization import GeneralizedIndex export merkleization.GeneralizedIndex const diff --git a/beacon_chain/spec/datatypes/base.nim b/beacon_chain/spec/datatypes/base.nim index 0bf9ab876..c09a687b0 100644 --- a/beacon_chain/spec/datatypes/base.nim +++ b/beacon_chain/spec/datatypes/base.nim @@ -35,7 +35,8 @@ import stew/[assign2, byteutils], chronicles, chronos/timer, - ../../version, ../../ssz/types as sszTypes, + ssz_serialization/types as sszTypes, + ../../version, ".."/[crypto, digest, presets] export diff --git a/beacon_chain/spec/datatypes/merge.nim b/beacon_chain/spec/datatypes/merge.nim index af795176d..62418cc5b 100644 --- a/beacon_chain/spec/datatypes/merge.nim +++ b/beacon_chain/spec/datatypes/merge.nim @@ -17,7 +17,8 @@ import stew/assign2, json_serialization, json_serialization/types as jsonTypes, - ../../ssz/types as sszTypes, ../digest, + ssz_serialization/types as sszTypes, + ../digest, ./phase0, ./altair, #web3/ethtypes, nimcrypto/utils diff --git a/beacon_chain/spec/eth2_apis/eth2_json_rpc_serialization.nim b/beacon_chain/spec/eth2_apis/eth2_json_rpc_serialization.nim index 04844c3fe..69c49364c 100644 --- a/beacon_chain/spec/eth2_apis/eth2_json_rpc_serialization.nim +++ b/beacon_chain/spec/eth2_apis/eth2_json_rpc_serialization.nim @@ -27,9 +27,9 @@ import # Nimble packages stew/byteutils, json_rpc/jsonmarshal, + ssz_serialization/types, # Local modules - ../../ssz/types, ../datatypes/base export jsonmarshal, base diff --git a/beacon_chain/spec/eth2_merkleization.nim b/beacon_chain/spec/eth2_merkleization.nim index 98e89bd47..634a7323b 100644 --- a/beacon_chain/spec/eth2_merkleization.nim +++ b/beacon_chain/spec/eth2_merkleization.nim @@ -10,8 +10,8 @@ # Import this module to get access to `hash_tree_root` for spec types import + ssz_serialization/merkleization, ./ssz_codec, - ../ssz/merkleization, ./datatypes/[phase0, altair] export ssz_codec, merkleization diff --git a/beacon_chain/spec/eth2_ssz_serialization.nim b/beacon_chain/spec/eth2_ssz_serialization.nim index 64b29b8e4..38e31de09 100644 --- a/beacon_chain/spec/eth2_ssz_serialization.nim +++ b/beacon_chain/spec/eth2_ssz_serialization.nim @@ -11,8 +11,8 @@ # ssz_serialization directly! To bypass root updates, use `readSszBytes` # without involving SSZ! import + ssz_serialization, ./ssz_codec, - ../ssz/ssz_serialization, ./datatypes/[phase0, altair, merge], ./eth2_merkleization diff --git a/beacon_chain/spec/ssz_codec.nim b/beacon_chain/spec/ssz_codec.nim index 01f6cef58..223291032 100644 --- a/beacon_chain/spec/ssz_codec.nim +++ b/beacon_chain/spec/ssz_codec.nim @@ -10,7 +10,7 @@ import std/[typetraits], - ../ssz/codec, + ssz_serialization/codec, ../spec/datatypes/[phase0, altair], ./eth2_merkleization diff --git a/beacon_chain/ssz/bitseqs.nim b/beacon_chain/ssz/bitseqs.nim deleted file mode 100644 index 08cc07cac..000000000 --- a/beacon_chain/ssz/bitseqs.nim +++ /dev/null @@ -1,333 +0,0 @@ -# beacon_chain -# Copyright (c) 2018-2020 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -{.push raises: [Defect].} - -import - stew/[bitops2, endians2, byteutils, ptrops], - json_serialization - -export json_serialization - -type - Bytes = seq[byte] - - BitSeq* = distinct Bytes - ## The current design of BitSeq tries to follow precisely - ## the bitwise representation of the SSZ bitlists. - ## This is a relatively compact representation, but as - ## evident from the code below, many of the operations - ## are not trivial. - - BitArray*[bits: static int] = object - bytes*: array[(bits + 7) div 8, byte] - -func bitsLen*(bytes: openArray[byte]): int = - let - bytesCount = bytes.len - lastByte = bytes[bytesCount - 1] - markerPos = log2trunc(lastByte) - - bytesCount * 8 - (8 - markerPos) - -template len*(s: BitSeq): int = - bitsLen(Bytes s) - -template len*(a: BitArray): int = - a.bits - -func add*(s: var BitSeq, value: bool) = - let - lastBytePos = s.Bytes.len - 1 - lastByte = s.Bytes[lastBytePos] - - if (lastByte and byte(128)) == 0: - # There is at least one leading zero, so we have enough - # room to store the new bit - let markerPos = log2trunc(lastByte) - s.Bytes[lastBytePos].changeBit markerPos, value - s.Bytes[lastBytePos].setBit markerPos + 1 - else: - s.Bytes[lastBytePos].changeBit 7, value - s.Bytes.add byte(1) - -func toBytesLE(x: uint): array[sizeof(x), byte] = - # stew/endians2 supports explicitly sized uints only - when sizeof(uint) == 4: - static: doAssert sizeof(uint) == sizeof(uint32) - toBytesLE(x.uint32) - elif sizeof(uint) == 8: - static: doAssert sizeof(uint) == sizeof(uint64) - toBytesLE(x.uint64) - else: - static: doAssert false, "requires a 32-bit or 64-bit platform" - -func loadLEBytes(WordType: type, bytes: openArray[byte]): WordType = - # TODO: this is a temporary proc until the endians API is improved - var shift = 0 - for b in bytes: - result = result or (WordType(b) shl shift) - shift += 8 - -func storeLEBytes(value: SomeUnsignedInt, dst: var openArray[byte]) = - doAssert dst.len <= sizeof(value) - let bytesLE = toBytesLE(value) - copyMem(addr dst[0], unsafeAddr bytesLE[0], dst.len) - -template loopOverWords(lhs, rhs: BitSeq, - lhsIsVar, rhsIsVar: static bool, - WordType: type, - lhsBits, rhsBits, body: untyped) = - const hasRhs = astToStr(lhs) != astToStr(rhs) - - let bytesCount = len Bytes(lhs) - when hasRhs: doAssert len(Bytes(rhs)) == bytesCount - - var fullWordsCount = bytesCount div sizeof(WordType) - let lastWordSize = bytesCount mod sizeof(WordType) - - block: - var lhsWord: WordType - when hasRhs: - var rhsWord: WordType - var firstByteOfLastWord, lastByteOfLastWord: int - - # TODO: Returning a `var` value from an iterator is always safe due to - # the way inlining works, but currently the compiler reports an error - # when a local variable escapes. We have to cheat it with this location - # obfuscation through pointers: - template lhsBits: auto = (addr(lhsWord))[] - - when hasRhs: - template rhsBits: auto = (addr(rhsWord))[] - - template lastWordBytes(bitseq): auto = - Bytes(bitseq).toOpenArray(firstByteOfLastWord, lastByteOfLastWord) - - template initLastWords = - lhsWord = loadLEBytes(WordType, lastWordBytes(lhs)) - when hasRhs: rhsWord = loadLEBytes(WordType, lastWordBytes(rhs)) - - if lastWordSize == 0: - firstByteOfLastWord = bytesCount - sizeof(WordType) - lastByteOfLastWord = bytesCount - 1 - dec fullWordsCount - else: - firstByteOfLastWord = bytesCount - lastWordSize - lastByteOfLastWord = bytesCount - 1 - - initLastWords() - let markerPos = log2trunc(lhsWord) - when hasRhs: doAssert log2trunc(rhsWord) == markerPos - - lhsWord.clearBit markerPos - when hasRhs: rhsWord.clearBit markerPos - - body - - when lhsIsVar or rhsIsVar: - let - markerBit = uint(1 shl markerPos) - mask = markerBit - 1'u - - when lhsIsVar: - let lhsEndResult = (lhsWord and mask) or markerBit - storeLEBytes(lhsEndResult, lastWordBytes(lhs)) - - when rhsIsVar: - let rhsEndResult = (rhsWord and mask) or markerBit - storeLEBytes(rhsEndResult, lastWordBytes(rhs)) - - var lhsCurrAddr = cast[ptr WordType](unsafeAddr Bytes(lhs)[0]) - let lhsEndAddr = offset(lhsCurrAddr, fullWordsCount) - when hasRhs: - var rhsCurrAddr = cast[ptr WordType](unsafeAddr Bytes(rhs)[0]) - - while lhsCurrAddr < lhsEndAddr: - template lhsBits: auto = lhsCurrAddr[] - when hasRhs: - template rhsBits: auto = rhsCurrAddr[] - - body - - lhsCurrAddr = offset(lhsCurrAddr, 1) - when hasRhs: rhsCurrAddr = offset(rhsCurrAddr, 1) - -iterator words*(x: var BitSeq): var uint = - loopOverWords(x, x, true, false, uint, word, wordB): - yield word - -iterator words*(x: BitSeq): uint = - loopOverWords(x, x, false, false, uint, word, word): - yield word - -iterator words*(a, b: BitSeq): (uint, uint) = - loopOverWords(a, b, false, false, uint, wordA, wordB): - yield (wordA, wordB) - -iterator words*(a: var BitSeq, b: BitSeq): (var uint, uint) = - loopOverWords(a, b, true, false, uint, wordA, wordB): - yield (wordA, wordB) - -iterator words*(a, b: var BitSeq): (var uint, var uint) = - loopOverWords(a, b, true, true, uint, wordA, wordB): - yield (wordA, wordB) - -func `[]`*(s: BitSeq, pos: Natural): bool {.inline.} = - doAssert pos < s.len - s.Bytes.getBit pos - -func `[]=`*(s: var BitSeq, pos: Natural, value: bool) {.inline.} = - doAssert pos < s.len - s.Bytes.changeBit pos, value - -func setBit*(s: var BitSeq, pos: Natural) {.inline.} = - doAssert pos < s.len - setBit s.Bytes, pos - -func clearBit*(s: var BitSeq, pos: Natural) {.inline.} = - doAssert pos < s.len - clearBit s.Bytes, pos - -func init*(T: type BitSeq, len: int): T = - result = BitSeq newSeq[byte](1 + len div 8) - Bytes(result).setBit len - -func init*(T: type BitArray): T = - # The default zero-initializatio is fine - discard - -template `[]`*(a: BitArray, pos: Natural): bool = - getBit a.bytes, pos - -template `[]=`*(a: var BitArray, pos: Natural, value: bool) = - changeBit a.bytes, pos, value - -template setBit*(a: var BitArray, pos: Natural) = - setBit a.bytes, pos - -template clearBit*(a: var BitArray, pos: Natural) = - clearBit a.bytes, pos - -# TODO: Submit this to the standard library as `cmp` -# At the moment, it doesn't work quite well because Nim selects -# the generic cmp[T] from the system module instead of choosing -# the openArray overload -func compareArrays[T](a, b: openArray[T]): int = - result = cmp(a.len, b.len) - if result != 0: return - - for i in 0 ..< a.len: - result = cmp(a[i], b[i]) - if result != 0: return - -template cmp*(a, b: BitSeq): int = - compareArrays(Bytes a, Bytes b) - -template `==`*(a, b: BitSeq): bool = - cmp(a, b) == 0 - -func `$`*(a: BitSeq | BitArray): string = - let length = a.len - result = newStringOfCap(2 + length) - result.add "0b" - for i in countdown(length - 1, 0): - result.add if a[i]: '1' else: '0' - -func incl*(tgt: var BitSeq, src: BitSeq) = - # Update `tgt` to include the bits of `src`, as if applying `or` to each bit - doAssert tgt.len == src.len - for tgtWord, srcWord in words(tgt, src): - tgtWord = tgtWord or srcWord - -func overlaps*(a, b: BitSeq): bool = - for wa, wb in words(a, b): - if (wa and wb) != 0: - return true - -func countOverlap*(a, b: BitSeq): int = - var res = 0 - for wa, wb in words(a, b): - res += countOnes(wa and wb) - res - -func isSubsetOf*(a, b: BitSeq): bool = - let alen = a.len - doAssert b.len == alen - for i in 0 ..< alen: - if a[i] and not b[i]: - return false - true - -func isZeros*(x: BitSeq): bool = - for w in words(x): - if w != 0: return false - return true - -func isZeros*(x: BitArray): bool = - x == default(type(x)) - -func countOnes*(x: BitSeq): int = - # Count the number of set bits - var res = 0 - for w in words(x): - res += w.countOnes() - res - -func clear*(x: var BitSeq) = - for w in words(x): - w = 0 - -func countZeros*(x: BitSeq): int = - x.len() - x.countOnes() - -template bytes*(x: BitSeq): untyped = - seq[byte](x) - -iterator items*(x: BitArray): bool = - for i in 0.. 0 and data.len <= bytesPerChunk - - if getBitLE(merkleizer.totalChunks, 0): - var hash = mergeBranches(merkleizer.combinedChunks[0], data) - - for i in 1 .. merkleizer.topIndex: - trs "ITERATING" - if getBitLE(merkleizer.totalChunks, i): - trs "CALLING MERGE BRANCHES" - hash = mergeBranches(merkleizer.combinedChunks[i], hash) - else: - trs "WRITING FRESH CHUNK AT ", i, " = ", hash - merkleizer.combinedChunks[i] = hash - break - else: - let paddingBytes = bytesPerChunk - data.len - - merkleizer.combinedChunks[0].data[0.. 0 and merkleizer.topIndex > 0 - - let proofHeight = merkleizer.topIndex + 1 - result = newSeq[Eth2Digest](chunks.len * proofHeight) - - if chunks.len == 1: - merkleizer.addChunkAndGenMerkleProof(chunks[0], result) - return - - let newTotalChunks = merkleizer.totalChunks + chunks.len.uint64 - - var - # A perfect binary tree will take either `chunks.len * 2` values if the - # number of elements in the base layer is odd and `chunks.len * 2 - 1` - # otherwise. Each row may also need a single extra element at most if - # it must be combined with the existing values in the Merkleizer: - merkleTree = newSeqOfCap[Eth2Digest](chunks.len + merkleizer.topIndex) - inRowIdx = merkleizer.totalChunks - postUpdateInRowIdx = newTotalChunks - zeroMixed = false - - template writeResult(chunkIdx, level: int, chunk: Eth2Digest) = - result[chunkIdx * proofHeight + level] = chunk - - # We'll start by generating the first row of the merkle tree. - var currPairEnd = if inRowIdx.isOdd: - # an odd chunk number means that we must combine the - # hash with the existing pending sibling hash in the - # merkleizer. - writeResult(0, 0, merkleizer.combinedChunks[0]) - merkleTree.add mergeBranches(merkleizer.combinedChunks[0], chunks[0]) - - # TODO: can we immediately write this out? - merkleizer.completeStartedChunk(merkleTree[^1], 1) - 2 - else: - 1 - - if postUpdateInRowIdx.isOdd: - merkleizer.combinedChunks[0] = chunks[^1] - - while currPairEnd < chunks.len: - writeResult(currPairEnd - 1, 0, chunks[currPairEnd]) - writeResult(currPairEnd, 0, chunks[currPairEnd - 1]) - merkleTree.add mergeBranches(chunks[currPairEnd - 1], - chunks[currPairEnd]) - currPairEnd += 2 - - if currPairEnd - 1 < chunks.len: - zeroMixed = true - writeResult(currPairEnd - 1, 0, zeroHashes[0]) - merkleTree.add mergeBranches(chunks[currPairEnd - 1], - zeroHashes[0]) - var - level = 0 - baseChunksPerElement = 1 - treeRowStart = 0 - rowLen = merkleTree.len - - template writeProofs(rowChunkIdx: int, hash: Eth2Digest) = - let - startAbsIdx = (inRowIdx.int + rowChunkIdx) * baseChunksPerElement - endAbsIdx = startAbsIdx + baseChunksPerElement - startResIdx = max(startAbsIdx - merkleizer.totalChunks.int, 0) - endResIdx = min(endAbsIdx - merkleizer.totalChunks.int, chunks.len) - - for resultPos in startResIdx ..< endResIdx: - writeResult(resultPos, level, hash) - - if rowLen > 1: - while level < merkleizer.topIndex: - inc level - baseChunksPerElement *= 2 - inRowIdx = inRowIdx div 2 - postUpdateInRowIdx = postUpdateInRowIdx div 2 - - var currPairEnd = if inRowIdx.isOdd: - # an odd chunk number means that we must combine the - # hash with the existing pending sibling hash in the - # merkleizer. - writeProofs(0, merkleizer.combinedChunks[level]) - merkleTree.add mergeBranches(merkleizer.combinedChunks[level], - merkleTree[treeRowStart]) - - # TODO: can we immediately write this out? - merkleizer.completeStartedChunk(merkleTree[^1], level + 1) - 2 - else: - 1 - - if postUpdateInRowIdx.isOdd: - merkleizer.combinedChunks[level] = merkleTree[treeRowStart + rowLen - - ord(zeroMixed) - 1] - while currPairEnd < rowLen: - writeProofs(currPairEnd - 1, merkleTree[treeRowStart + currPairEnd]) - writeProofs(currPairEnd, merkleTree[treeRowStart + currPairEnd - 1]) - merkleTree.add mergeBranches(merkleTree[treeRowStart + currPairEnd - 1], - merkleTree[treeRowStart + currPairEnd]) - currPairEnd += 2 - - if currPairEnd - 1 < rowLen: - zeroMixed = true - writeProofs(currPairEnd - 1, zeroHashes[level]) - merkleTree.add mergeBranches(merkleTree[treeRowStart + currPairEnd - 1], - zeroHashes[level]) - - treeRowStart += rowLen - rowLen = merkleTree.len - treeRowStart - - if rowLen == 1: - break - - doAssert rowLen == 1 - - if (inRowIdx and 2) != 0: - merkleizer.completeStartedChunk( - mergeBranches(merkleizer.combinedChunks[level + 1], merkleTree[^1]), - level + 2) - - if (not zeroMixed) and (postUpdateInRowIdx and 2) != 0: - merkleizer.combinedChunks[level + 1] = merkleTree[^1] - - while level < merkleizer.topIndex: - inc level - baseChunksPerElement *= 2 - inRowIdx = inRowIdx div 2 - - let hash = if getBitLE(merkleizer.totalChunks, level): - merkleizer.combinedChunks[level] - else: - zeroHashes[level] - - writeProofs(0, hash) - - merkleizer.totalChunks = newTotalChunks - -proc init*(S: type SszMerkleizer): S = - new result.combinedChunks - result.impl = SszMerkleizerImpl( - combinedChunks: cast[ptr UncheckedArray[Eth2Digest]]( - addr result.combinedChunks[][0]), - topIndex: binaryTreeHeight(result.limit) - 1, - totalChunks: 0) - -proc init*(S: type SszMerkleizer, - combinedChunks: openArray[Eth2Digest], - totalChunks: uint64): S = - new result.combinedChunks - result.combinedChunks[][0 ..< combinedChunks.len] = combinedChunks - result.impl = SszMerkleizerImpl( - combinedChunks: cast[ptr UncheckedArray[Eth2Digest]]( - addr result.combinedChunks[][0]), - topIndex: binaryTreeHeight(result.limit) - 1, - totalChunks: totalChunks) - -proc copy*[L: static[Limit]](cloned: SszMerkleizer[L]): SszMerkleizer[L] = - new result.combinedChunks - result.combinedChunks[] = cloned.combinedChunks[] - result.impl = SszMerkleizerImpl( - combinedChunks: cast[ptr UncheckedArray[Eth2Digest]]( - addr result.combinedChunks[][0]), - topIndex: binaryTreeHeight(L) - 1, - totalChunks: cloned.totalChunks) - -template addChunksAndGenMerkleProofs*( - merkleizer: var SszMerkleizer, - chunks: openArray[Eth2Digest]): seq[Eth2Digest] = - addChunksAndGenMerkleProofs(merkleizer.impl, chunks) - -template addChunk*(merkleizer: var SszMerkleizer, data: openArray[byte]) = - addChunk(merkleizer.impl, data) - -template totalChunks*(merkleizer: SszMerkleizer): uint64 = - merkleizer.impl.totalChunks - -template getFinalHash*(merkleizer: SszMerkleizer): Eth2Digest = - merkleizer.impl.getFinalHash - -template createMerkleizer*(totalElements: static Limit): SszMerkleizerImpl = - trs "CREATING A MERKLEIZER FOR ", totalElements - - const treeHeight = binaryTreeHeight totalElements - var combinedChunks {.noInit.}: array[treeHeight, Eth2Digest] - - SszMerkleizerImpl( - combinedChunks: cast[ptr UncheckedArray[Eth2Digest]](addr combinedChunks), - topIndex: treeHeight - 1, - totalChunks: 0) - -func getFinalHash*(merkleizer: SszMerkleizerImpl): Eth2Digest = - if merkleizer.totalChunks == 0: - return zeroHashes[merkleizer.topIndex] - - let - bottomHashIdx = firstOne(merkleizer.totalChunks) - 1 - submittedChunksHeight = bitWidth(merkleizer.totalChunks - 1) - topHashIdx = merkleizer.topIndex - - trs "BOTTOM HASH ", bottomHashIdx - trs "SUBMITTED HEIGHT ", submittedChunksHeight - trs "TOP HASH IDX ", topHashIdx - - if bottomHashIdx != submittedChunksHeight: - # Our tree is not finished. We must complete the work in progress - # branches and then extend the tree to the right height. - result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx], - zeroHashes[bottomHashIdx]) - - for i in bottomHashIdx + 1 ..< topHashIdx: - if getBitLE(merkleizer.totalChunks, i): - result = mergeBranches(merkleizer.combinedChunks[i], result) - trs "COMBINED" - else: - result = mergeBranches(result, zeroHashes[i]) - trs "COMBINED WITH ZERO" - - elif bottomHashIdx == topHashIdx: - # We have a perfect tree (chunks == 2**n) at just the right height! - result = merkleizer.combinedChunks[bottomHashIdx] - else: - # We have a perfect tree of user chunks, but we have more work to - # do - we must extend it to reach the desired height - result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx], - zeroHashes[bottomHashIdx]) - - for i in bottomHashIdx + 1 ..< topHashIdx: - result = mergeBranches(result, zeroHashes[i]) - -func mixInLength*(root: Eth2Digest, length: int): Eth2Digest = - var dataLen: array[32, byte] - dataLen[0..<8] = uint64(length).toBytesLE() - mergeBranches(root, dataLen) - -func hash_tree_root*(x: auto): Eth2Digest {.gcsafe, raises: [Defect].} - -template merkleizeFields(totalElements: static Limit, body: untyped): Eth2Digest = - var merkleizer {.inject.} = createMerkleizer(totalElements) - - template addField(field) = - let hash = hash_tree_root(field) - trs "MERKLEIZING FIELD ", astToStr(field), " = ", hash - addChunk(merkleizer, hash.data) - trs "CHUNK ADDED" - - body - - getFinalHash(merkleizer) - -template writeBytesLE(chunk: var array[bytesPerChunk, byte], atParam: int, - val: UintN) = - let at = atParam - chunk[at ..< at + sizeof(val)] = toBytesLE(val) - -func chunkedHashTreeRootForBasicTypes[T](merkleizer: var SszMerkleizerImpl, - arr: openArray[T]): Eth2Digest = - static: - doAssert T is BasicType - doAssert bytesPerChunk mod sizeof(T) == 0 - - if arr.len == 0: - return getFinalHash(merkleizer) - - when sizeof(T) == 1 or cpuEndian == littleEndian: - var - remainingBytes = when sizeof(T) == 1: arr.len - else: arr.len * sizeof(T) - pos = cast[ptr byte](unsafeAddr arr[0]) - - while remainingBytes >= bytesPerChunk: - merkleizer.addChunk(makeOpenArray(pos, bytesPerChunk)) - pos = offset(pos, bytesPerChunk) - remainingBytes -= bytesPerChunk - - if remainingBytes > 0: - merkleizer.addChunk(makeOpenArray(pos, remainingBytes)) - - else: - const valuesPerChunk = bytesPerChunk div sizeof(T) - - var writtenValues = 0 - - var chunk: array[bytesPerChunk, byte] - while writtenValues < arr.len - valuesPerChunk: - for i in 0 ..< valuesPerChunk: - chunk.writeBytesLE(i * sizeof(T), arr[writtenValues + i]) - merkleizer.addChunk chunk - inc writtenValues, valuesPerChunk - - let remainingValues = arr.len - writtenValues - if remainingValues > 0: - var lastChunk: array[bytesPerChunk, byte] - for i in 0 ..< remainingValues: - lastChunk.writeBytesLE(i * sizeof(T), arr[writtenValues + i]) - merkleizer.addChunk lastChunk - - getFinalHash(merkleizer) - -func bitListHashTreeRoot(merkleizer: var SszMerkleizerImpl, x: BitSeq): Eth2Digest = - # TODO: Switch to a simpler BitList representation and - # replace this with `chunkedHashTreeRoot` - var - totalBytes = bytes(x).len - lastCorrectedByte = bytes(x)[^1] - - if lastCorrectedByte == byte(1): - if totalBytes == 1: - # This is an empty bit list. - # It should be hashed as a tree containing all zeros: - return mergeBranches(zeroHashes[merkleizer.topIndex], - zeroHashes[0]) # this is the mixed length - - totalBytes -= 1 - lastCorrectedByte = bytes(x)[^2] - else: - let markerPos = log2trunc(lastCorrectedByte) - lastCorrectedByte.clearBit(markerPos) - - var - bytesInLastChunk = totalBytes mod bytesPerChunk - fullChunks = totalBytes div bytesPerChunk - - if bytesInLastChunk == 0: - fullChunks -= 1 - bytesInLastChunk = 32 - - for i in 0 ..< fullChunks: - let - chunkStartPos = i * bytesPerChunk - chunkEndPos = chunkStartPos + bytesPerChunk - 1 - - merkleizer.addChunk bytes(x).toOpenArray(chunkStartPos, chunkEndPos) - - var - lastChunk: array[bytesPerChunk, byte] - chunkStartPos = fullChunks * bytesPerChunk - - for i in 0 .. bytesInLastChunk - 2: - lastChunk[i] = bytes(x)[chunkStartPos + i] - - lastChunk[bytesInLastChunk - 1] = lastCorrectedByte - - merkleizer.addChunk lastChunk.toOpenArray(0, bytesInLastChunk - 1) - let contentsHash = merkleizer.getFinalHash - mixInLength contentsHash, x.len - -func maxChunksCount(T: type, maxLen: Limit): Limit = - when T is BitList|BitArray: - (maxLen + bitsPerChunk - 1) div bitsPerChunk - elif T is array|List: - maxChunkIdx(ElemType(T), maxLen) - else: - unsupported T # This should never happen - -func hashTreeRootAux[T](x: T): Eth2Digest = - when T is bool|char: - result.data[0] = byte(x) - elif T is UintN: - when cpuEndian == bigEndian: - result.data[0..= byteLen: - zeroHashes[1] - else: - let - nbytes = min(byteLen - byteIdx, 64) - padding = 64 - nbytes - - digest( - toOpenArray(bytes, int(byteIdx), int(byteIdx + nbytes - 1)), - toOpenArray(zero64, 0, int(padding - 1))) - else: - if chunkIdx + 1 > x.data.len(): - zeroHashes[x.maxDepth] - elif chunkIdx + 1 == x.data.len(): - mergeBranches( - hash_tree_root(x.data[chunkIdx]), - Eth2Digest()) - else: - mergeBranches( - hash_tree_root(x.data[chunkIdx]), - hash_tree_root(x.data[chunkIdx + 1])) - -template mergedHash(x: HashList|HashArray, vIdxParam: int64): Eth2Digest = - # The merged hash of the data at `vIdx` and `vIdx + 1` - let vIdx = vIdxParam - if vIdx >= x.maxChunks: - let dataIdx = vIdx - x.maxChunks - mergedDataHash(x, dataIdx) - else: - mergeBranches( - hashTreeRootCached(x, vIdx), - hashTreeRootCached(x, vIdx + 1)) - -func hashTreeRootCached*(x: HashList, vIdx: int64): Eth2Digest = - doAssert vIdx >= 1, "Only valid for flat merkle tree indices" - - let - layer = layer(vIdx) - idxInLayer = vIdx - (1'i64 shl layer) - layerIdx = idxInlayer + x.indices[layer] - - trs "GETTING ", vIdx, " ", layerIdx, " ", layer, " ", x.indices.len - - doAssert layer < x.maxDepth - if layerIdx >= x.indices[layer + 1]: - trs "ZERO ", x.indices[layer], " ", x.indices[layer + 1] - zeroHashes[x.maxDepth - layer] - else: - if not isCached(x.hashes[layerIdx]): - # TODO oops. so much for maintaining non-mutability. - let px = unsafeAddr x - - trs "REFRESHING ", vIdx, " ", layerIdx, " ", layer - - px[].hashes[layerIdx] = mergedHash(x, vIdx * 2) - else: - trs "CACHED ", layerIdx - - x.hashes[layerIdx] - -func hashTreeRootCached*(x: HashArray, vIdx: int): Eth2Digest = - doAssert vIdx >= 1, "Only valid for flat merkle tree indices" - - if not isCached(x.hashes[vIdx]): - # TODO oops. so much for maintaining non-mutability. - let px = unsafeAddr x - - px[].hashes[vIdx] = mergedHash(x, vIdx * 2) - - return x.hashes[vIdx] - -func hashTreeRootCached*(x: HashArray): Eth2Digest = - hashTreeRootCached(x, 1) # Array does not use idx 0 - -func hashTreeRootCached*(x: HashList): Eth2Digest = - if x.data.len == 0: - mergeBranches( - zeroHashes[x.maxDepth], - zeroHashes[0]) # mixInLength with 0! - else: - if not isCached(x.hashes[0]): - # TODO oops. so much for maintaining non-mutability. - let px = unsafeAddr x - px[].hashes[0] = mixInLength(hashTreeRootCached(x, 1), x.data.len) - - x.hashes[0] - -func hash_tree_root*(x: auto): Eth2Digest = - trs "STARTING HASH TREE ROOT FOR TYPE ", name(type(x)) - mixin toSszType - - result = - when x is HashArray|HashList: - hashTreeRootCached(x) - elif x is List|BitList: - hashTreeRootList(x) - else: - hashTreeRootAux toSszType(x) - - trs "HASH TREE ROOT FOR ", name(type x), " = ", "0x", $result diff --git a/beacon_chain/ssz/navigator.nim b/beacon_chain/ssz/navigator.nim deleted file mode 100644 index e68f92519..000000000 --- a/beacon_chain/ssz/navigator.nim +++ /dev/null @@ -1,143 +0,0 @@ -# beacon_chain -# Copyright (c) 2018-2020 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -{.push raises: [Defect].} -{.pragma: raisesssz, raises: [Defect, MalformedSszError, SszSizeMismatchError].} - -import - stew/[ptrops, objects], stew/ranges/ptr_arith, - ./codec, ./types - -export codec, types - -type - MemRange* = object - startAddr*: ptr byte - length*: int - - SszNavigator*[T] = object - m: MemRange - -func sszMount*(data: openArray[byte], T: type): SszNavigator[T] = - let startAddr = unsafeAddr data[0] - SszNavigator[T](m: MemRange(startAddr: startAddr, length: data.len)) - -func sszMount*(data: openArray[char], T: type): SszNavigator[T] = - let startAddr = cast[ptr byte](unsafeAddr data[0]) - SszNavigator[T](m: MemRange(startAddr: startAddr, length: data.len)) - -template sszMount*(data: MemRange, T: type): SszNavigator[T] = - SszNavigator[T](m: data) - -template getMemRange*(n: SszNavigator): MemRange = - # Please note that this accessor was created intentionally. - # We don't want to expose the `m` field, because the navigated - # type may have a field by that name. We wan't any dot field - # access to be redirected to the navigated type. - # For this reason, this template should always be used with - # the function call syntax `getMemRange(n)`. - n.m - -template checkBounds(m: MemRange, offset: int) = - if offset > m.length: - raise newException(MalformedSszError, "Malformed SSZ") - -template toOpenArray(m: MemRange): auto = - makeOpenArray(m.startAddr, m.length) - -func navigateToField*[T](n: SszNavigator[T], - fieldName: static string, - FieldType: type): SszNavigator[FieldType] {.raisesssz.} = - mixin toSszType - type SszFieldType = type toSszType(declval FieldType) - - const boundingOffsets = getFieldBoundingOffsets(T, fieldName) - checkBounds(n.m, boundingOffsets[1]) - - when isFixedSize(SszFieldType): - SszNavigator[FieldType](m: MemRange( - startAddr: offset(n.m.startAddr, boundingOffsets[0]), - length: boundingOffsets[1] - boundingOffsets[0])) - else: - template readOffset(off): int = - int fromSszBytes(uint32, makeOpenArray(offset(n.m.startAddr, off), - sizeof(uint32))) - let - startOffset = readOffset boundingOffsets[0] - endOffset = when boundingOffsets[1] == -1: n.m.length - else: readOffset boundingOffsets[1] - - if endOffset < startOffset or endOffset > n.m.length: - raise newException(MalformedSszError, "Incorrect offset values") - - SszNavigator[FieldType](m: MemRange( - startAddr: offset(n.m.startAddr, startOffset), - length: endOffset - startOffset)) - -template `.`*[T](n: SszNavigator[T], field: untyped): auto = - type RecType = T - type FieldType = type(default(RecType).field) - navigateToField(n, astToStr(field), FieldType) - -func indexVarSizeList(m: MemRange, idx: int): MemRange {.raisesssz.} = - template readOffset(pos): int = - int fromSszBytes(uint32, makeOpenArray(offset(m.startAddr, pos), offsetSize)) - - let offsetPos = offsetSize * idx - checkBounds(m, offsetPos + offsetSize) - - let firstOffset = readOffset 0 - let listLen = firstOffset div offsetSize - - if idx >= listLen: - # TODO: Use a RangeError here? - # This would require the user to check the `len` upfront - raise newException(MalformedSszError, "Indexing past the end") - - let elemPos = readOffset offsetPos - checkBounds(m, elemPos) - - let endPos = if idx < listLen - 1: - let nextOffsetPos = offsetPos + offsetSize - # TODO. Is there a way to remove this bounds check? - checkBounds(m, nextOffsetPos + offsetSize) - readOffset(offsetPos + nextOffsetPos) - else: - m.length - - MemRange(startAddr: m.startAddr.offset(elemPos), length: endPos - elemPos) - -template indexList(n, idx, T: untyped): untyped = - type R = T - mixin toSszType - type ElemType = type toSszType(declval R) - when isFixedSize(ElemType): - const elemSize = fixedPortionSize(ElemType) - let elemPos = idx * elemSize - checkBounds(n.m, elemPos + elemSize) - SszNavigator[R](m: MemRange(startAddr: offset(n.m.startAddr, elemPos), - length: elemSize)) - else: - SszNavigator[R](m: indexVarSizeList(n.m, idx)) - -template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] = - indexList n, idx, T - -template `[]`*[R, T](n: SszNavigator[array[R, T]], idx: int): SszNavigator[T] = - indexList(n, idx, T) - -func `[]`*[T](n: SszNavigator[T]): T {.raisesssz.} = - mixin toSszType, fromSszBytes - type SszRepr = type toSszType(declval T) - when type(SszRepr) is type(T) or T is List: - readSszValue(toOpenArray(n.m), result) - else: - fromSszBytes(T, toOpenArray(n.m)) - -converter derefNavigator*[T](n: SszNavigator[T]): T {.raisesssz.} = - n[] - diff --git a/beacon_chain/ssz/ssz_serialization.nim b/beacon_chain/ssz/ssz_serialization.nim deleted file mode 100644 index f59e7175d..000000000 --- a/beacon_chain/ssz/ssz_serialization.nim +++ /dev/null @@ -1,250 +0,0 @@ -# beacon_chain -# Copyright (c) 2018-2021 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -{.push raises: [Defect].} -{.pragma: raisesssz, raises: [Defect, MalformedSszError, SszSizeMismatchError].} - -## SSZ serialization for core SSZ types, as specified in: -# https://github.com/ethereum/consensus-specs/blob/v1.0.1/ssz/simple-serialize.md#serialization - -import - std/typetraits, - stew/[endians2, leb128, objects], - serialization, serialization/testing/tracing, - ./codec, ./bitseqs, ./types - -export - serialization, codec, types, bitseqs - -type - SszReader* = object - stream: InputStream - - SszWriter* = object - stream: OutputStream - - SizePrefixed*[T] = distinct T - SszMaxSizeExceeded* = object of SerializationError - - VarSizedWriterCtx = object - fixedParts: WriteCursor - offset: int - - FixedSizedWriterCtx = object - -serializationFormat SSZ - -SSZ.setReader SszReader -SSZ.setWriter SszWriter, PreferredOutput = seq[byte] - -template sizePrefixed*[TT](x: TT): untyped = - type T = TT - SizePrefixed[T](x) - -proc init*(T: type SszReader, - stream: InputStream): T = - T(stream: stream) - -proc writeFixedSized(s: var (OutputStream|WriteCursor), x: auto) {.raises: [Defect, IOError].} = - mixin toSszType - - when x is byte: - s.write x - elif x is bool: - s.write byte(ord(x)) - elif x is UintN: - when cpuEndian == bigEndian: - s.write toBytesLE(x) - else: - s.writeMemCopy x - elif x is array: - when x[0] is byte: - trs "APPENDING FIXED SIZE BYTES", x - s.write x - else: - for elem in x: - trs "WRITING FIXED SIZE ARRAY ELEMENT" - s.writeFixedSized toSszType(elem) - elif x is tuple|object: - enumInstanceSerializedFields(x, fieldName, field): - trs "WRITING FIXED SIZE FIELD", fieldName - s.writeFixedSized toSszType(field) - else: - unsupported x.type - -template writeOffset(cursor: var WriteCursor, offset: int) = - write cursor, toBytesLE(uint32 offset) - -template supports*(_: type SSZ, T: type): bool = - mixin toSszType - anonConst compiles(fixedPortionSize toSszType(declval T)) - -func init*(T: type SszWriter, stream: OutputStream): T = - result.stream = stream - -proc writeVarSizeType(w: var SszWriter, value: auto) {.gcsafe, raises: [Defect, IOError].} - -proc beginRecord*(w: var SszWriter, TT: type): auto = - type T = TT - when isFixedSize(T): - FixedSizedWriterCtx() - else: - const offset = when T is array|HashArray: len(T) * offsetSize - else: fixedPortionSize(T) - VarSizedWriterCtx(offset: offset, - fixedParts: w.stream.delayFixedSizeWrite(offset)) - -template writeField*(w: var SszWriter, - ctx: var auto, - fieldName: string, - field: auto) = - mixin toSszType - when ctx is FixedSizedWriterCtx: - writeFixedSized(w.stream, toSszType(field)) - else: - type FieldType = type toSszType(field) - - when isFixedSize(FieldType): - writeFixedSized(ctx.fixedParts, toSszType(field)) - else: - trs "WRITING OFFSET ", ctx.offset, " FOR ", fieldName - writeOffset(ctx.fixedParts, ctx.offset) - let initPos = w.stream.pos - trs "WRITING VAR SIZE VALUE OF TYPE ", name(FieldType) - when FieldType is BitList: - trs "BIT SEQ ", bytes(field) - writeVarSizeType(w, toSszType(field)) - ctx.offset += w.stream.pos - initPos - -template endRecord*(w: var SszWriter, ctx: var auto) = - when ctx is VarSizedWriterCtx: - finalize ctx.fixedParts - -proc writeSeq[T](w: var SszWriter, value: seq[T]) - {.raises: [Defect, IOError].} = - # Please note that `writeSeq` exists in order to reduce the code bloat - # produced from generic instantiations of the unique `List[N, T]` types. - when isFixedSize(T): - trs "WRITING LIST WITH FIXED SIZE ELEMENTS" - for elem in value: - w.stream.writeFixedSized toSszType(elem) - trs "DONE" - else: - trs "WRITING LIST WITH VAR SIZE ELEMENTS" - var offset = value.len * offsetSize - var cursor = w.stream.delayFixedSizeWrite offset - for elem in value: - cursor.writeFixedSized uint32(offset) - let initPos = w.stream.pos - w.writeVarSizeType toSszType(elem) - offset += w.stream.pos - initPos - finalize cursor - trs "DONE" - -proc writeVarSizeType(w: var SszWriter, value: auto) {.raises: [Defect, IOError].} = - trs "STARTING VAR SIZE TYPE" - - when value is HashArray|HashList: - writeVarSizeType(w, value.data) - elif value is SingleMemberUnion: - doAssert value.selector == 0'u8 - w.writeValue 0'u8 - w.writeValue value.value - elif value is List: - # We reduce code bloat by forwarding all `List` types to a general `seq[T]` proc. - writeSeq(w, asSeq value) - elif value is BitList: - # ATTENTION! We can reuse `writeSeq` only as long as our BitList type is implemented - # to internally match the binary representation of SSZ BitLists in memory. - writeSeq(w, bytes value) - elif value is object|tuple|array: - trs "WRITING OBJECT OR ARRAY" - var ctx = beginRecord(w, type value) - enumerateSubFields(value, field): - writeField w, ctx, astToStr(field), field - endRecord w, ctx - else: - unsupported type(value) - -proc writeValue*(w: var SszWriter, x: auto) {.gcsafe, raises: [Defect, IOError].} = - mixin toSszType - type T = type toSszType(x) - - when isFixedSize(T): - w.stream.writeFixedSized toSszType(x) - else: - w.writeVarSizeType toSszType(x) - -func sszSize*(value: auto): int {.gcsafe, raises: [Defect].} - -func sszSizeForVarSizeList[T](value: openArray[T]): int = - result = len(value) * offsetSize - for elem in value: - result += sszSize(toSszType elem) - -func sszSize*(value: auto): int {.gcsafe, raises: [Defect].} = - mixin toSszType - type T = type toSszType(value) - - when isFixedSize(T): - anonConst fixedPortionSize(T) - - elif T is array|List|HashList|HashArray: - type E = ElemType(T) - when isFixedSize(E): - len(value) * anonConst(fixedPortionSize(E)) - elif T is HashArray: - sszSizeForVarSizeList(value.data) - elif T is array: - sszSizeForVarSizeList(value) - else: - sszSizeForVarSizeList(asSeq value) - - elif T is BitList: - return len(bytes(value)) - - elif T is SingleMemberUnion: - sszSize(toSszType value.value) + 1 - - elif T is object|tuple: - result = anonConst fixedPortionSize(T) - enumInstanceSerializedFields(value, _{.used.}, field): - type FieldType = type toSszType(field) - when not isFixedSize(FieldType): - result += sszSize(toSszType field) - - else: - unsupported T - -proc writeValue*[T](w: var SszWriter, x: SizePrefixed[T]) {.raises: [Defect, IOError].} = - var cursor = w.stream.delayVarSizeWrite(Leb128.maxLen(uint64)) - let initPos = w.stream.pos - w.writeValue T(x) - let length = toBytes(uint64(w.stream.pos - initPos), Leb128) - cursor.finalWrite length.toOpenArray() - -proc readValue*(r: var SszReader, val: var auto) {. - raises: [Defect, MalformedSszError, SszSizeMismatchError, IOError].} = - mixin readSszBytes - type T = type val - when isFixedSize(T): - const minimalSize = fixedPortionSize(T) - if r.stream.readable(minimalSize): - readSszBytes(r.stream.read(minimalSize), val) - else: - raise newException(MalformedSszError, "SSZ input of insufficient size") - else: - # TODO(zah) Read the fixed portion first and precisely measure the - # size of the dynamic portion to consume the right number of bytes. - readSszBytes(r.stream.read(r.stream.len.get), val) - -proc readSszBytes*[T](data: openArray[byte], val: var T) {. - raises: [Defect, MalformedSszError, SszSizeMismatchError].} = - # Overload `readSszBytes` to perform custom operations on T after - # deserialization - mixin readSszValue - readSszValue(data, val) diff --git a/tests/all_tests.nim b/tests/all_tests.nim index f414d3771..f6bc97be2 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -12,12 +12,10 @@ import ./testutil import # Unit test - ./ssz/all_tests as ssz_all_tests, ./test_action_tracker, ./test_attestation_pool, ./test_beacon_chain_db, ./test_beaconstate, - ./test_bitseqs, ./test_block_pool, ./test_datatypes, ./test_discovery, diff --git a/tests/helpers/debug_state.nim b/tests/helpers/debug_state.nim index 7821d2764..34e25b430 100644 --- a/tests/helpers/debug_state.nim +++ b/tests/helpers/debug_state.nim @@ -7,8 +7,8 @@ import macros, - ../../beacon_chain/spec/datatypes/base, - ../../beacon_chain/ssz/types + ssz_serialization/types, + ../../beacon_chain/spec/datatypes/base # digest is necessary for them to be printed as hex export base.`==` diff --git a/tests/ssz/all_tests.nim b/tests/ssz/all_tests.nim deleted file mode 100644 index 65de359f5..000000000 --- a/tests/ssz/all_tests.nim +++ /dev/null @@ -1,5 +0,0 @@ -{.used.} - -import - ./test_ssz_roundtrip, - ./test_ssz_serialization diff --git a/tests/ssz/test_ssz_roundtrip.nim b/tests/ssz/test_ssz_roundtrip.nim deleted file mode 100644 index 78d94759a..000000000 --- a/tests/ssz/test_ssz_roundtrip.nim +++ /dev/null @@ -1,16 +0,0 @@ -# beacon_chain -# Copyright (c) 2018 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -{.used.} - -# this is not part of test_ssz because the roundtrip tests are incompatible -# with unittest2 as of writing -import - serialization/testing/generic_suite, - ../../beacon_chain/ssz/ssz_serialization - -executeRoundTripTests SSZ diff --git a/tests/ssz/test_ssz_serialization.nim b/tests/ssz/test_ssz_serialization.nim deleted file mode 100644 index f65bcfede..000000000 --- a/tests/ssz/test_ssz_serialization.nim +++ /dev/null @@ -1,355 +0,0 @@ -# beacon_chain -# Copyright (c) 2018-2021 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -{.used.} - -import - std/typetraits, - unittest2, - ../../beacon_chain/ssz/[ssz_serialization, merkleization], - ../../beacon_chain/ssz/[navigator, dynamic_navigator] - -type - SomeEnum = enum - A, B, C - - Simple = object - flag: bool - # ignored {.dontSerialize.}: string - data: array[256, bool] - data2: HashArray[256, bool] - - NonFixed = object - data: HashList[uint64, 1024] - -template reject(stmt) = - doAssert(not compiles(stmt)) - -static: - doAssert isFixedSize(bool) == true - - doAssert fixedPortionSize(array[10, bool]) == 10 - doAssert fixedPortionSize(array[SomeEnum, uint64]) == 24 - doAssert fixedPortionSize(array[3..5, List[byte, 256]]) == 12 - - doAssert isFixedSize(array[20, bool]) == true - doAssert isFixedSize(Simple) == true - doAssert isFixedSize(List[bool, 128]) == false - - doAssert isFixedSize(NonFixed) == false - - reject fixedPortionSize(int) - -type - ObjWithFields = object - f0: uint8 - f1: uint32 - f2: array[20, byte] - f3: Eth2Digest - -static: - doAssert fixedPortionSize(ObjWithFields) == - 1 + 4 + sizeof(array[20, byte]) + (256 div 8) - -type - Foo = object - bar: Bar - - BarList = List[uint64, 128] - - Bar = object - b: BarList - baz: Baz - - Baz = object - i: uint64 - -func toDigest[N: static int](x: array[N, byte]): Eth2Digest = - result.data[0 .. N-1] = x - -suite "SSZ navigator": - test "simple object fields": - var foo = Foo(bar: Bar(b: BarList @[1'u64, 2, 3], baz: Baz(i: 10'u64))) - let encoded = SSZ.encode(foo) - - check SSZ.decode(encoded, Foo) == foo - - let mountedFoo = sszMount(encoded, Foo) - check mountedFoo.bar.b[] == BarList @[1'u64, 2, 3] - - let mountedBar = mountedFoo.bar - check mountedBar.baz.i == 10'u64 - - test "lists with max size": - let a = [byte 0x01, 0x02, 0x03].toDigest - let b = [byte 0x04, 0x05, 0x06].toDigest - let c = [byte 0x07, 0x08, 0x09].toDigest - - var xx: List[uint64, 16] - check: - not xx.setLen(17) - xx.setLen(16) - - var leaves = HashList[Eth2Digest, 1'i64 shl 3]() - check: - leaves.add a - leaves.add b - leaves.add c - let root = hash_tree_root(leaves) - check $root == "5248085b588fab1dd1e03f3cd62201602b12e6560665935964f46e805977e8c5" - - while leaves.len < 1 shl 3: - check: - leaves.add c - hash_tree_root(leaves) == hash_tree_root(leaves.data) - - leaves = default(type leaves) - - while leaves.len < (1 shl 3) - 1: - check: - leaves.add c - leaves.add c - hash_tree_root(leaves) == hash_tree_root(leaves.data) - - leaves = default(type leaves) - - while leaves.len < (1 shl 3) - 2: - check: - leaves.add c - leaves.add c - leaves.add c - hash_tree_root(leaves) == hash_tree_root(leaves.data) - - for i in 0 ..< leaves.data.len - 2: - leaves[i] = a - leaves[i + 1] = b - leaves[i + 2] = c - check hash_tree_root(leaves) == hash_tree_root(leaves.data) - - var leaves2 = HashList[Eth2Digest, 1'i64 shl 48]() # Large number! - check: - leaves2.add a - leaves2.add b - leaves2.add c - hash_tree_root(leaves2) == hash_tree_root(leaves2.data) - - var leaves3 = HashList[Eth2Digest, 7]() # Non-power-of-2 - check: - hash_tree_root(leaves3) == hash_tree_root(leaves3.data) - leaves3.add a - leaves3.add b - leaves3.add c - hash_tree_root(leaves3) == hash_tree_root(leaves3.data) - - test "basictype": - var leaves = HashList[uint64, 1'i64 shl 3]() - while leaves.len < leaves.maxLen: - check: - leaves.add leaves.len.uint64 - hash_tree_root(leaves) == hash_tree_root(leaves.data) - -suite "SSZ dynamic navigator": - test "navigating fields": - var fooOrig = Foo(bar: Bar(b: BarList @[1'u64, 2, 3], baz: Baz(i: 10'u64))) - let fooEncoded = SSZ.encode(fooOrig) - - var navFoo = DynamicSszNavigator.init(fooEncoded, Foo) - - var navBar = navFoo.navigate("bar") - check navBar.toJson(pretty = false) == """{"b":[1,2,3],"baz":{"i":10}}""" - - var navB = navBar.navigate("b") - check navB.toJson(pretty = false) == "[1,2,3]" - - var navBaz = navBar.navigate("baz") - var navI = navBaz.navigate("i") - check navI.toJson == "10" - - expect KeyError: - discard navBar.navigate("biz") - -type - Obj = object - arr: array[8, Eth2Digest] - - li: List[Eth2Digest, 8] - - HashObj = object - arr: HashArray[8, Eth2Digest] - - li: HashList[Eth2Digest, 8] - -suite "hash": - test "HashArray": - var - o = Obj() - ho = HashObj() - - template both(body) = - block: - template it: auto {.inject.} = o - body - block: - template it: auto {.inject.} = ho - body - - let htro = hash_tree_root(o) - let htrho = hash_tree_root(ho) - - check: - o.arr == ho.arr.data - o.li == ho.li.data - htro == htrho - - both: it.arr[0].data[0] = byte 1 - - both: check: it.li.add Eth2Digest() - - var y: HashArray[32, uint64] - check: hash_tree_root(y) == hash_tree_root(y.data) - for i in 0..