From 7e846a0bce507bc993896226368e449c235f58c5 Mon Sep 17 00:00:00 2001 From: Zahary Karadjov Date: Mon, 11 May 2020 19:27:44 +0300 Subject: [PATCH] Implement sszSize --- beacon_chain/ssz.nim | 31 +++++++++++++++++-- tests/official/fixtures_utils.nim | 16 ++++++++++ .../test_fixture_ssz_consensus_objects.nim | 19 ++++++++---- .../test_fixture_ssz_generic_types.nim | 14 +++------ vendor/nim-serialization | 2 +- 5 files changed, 62 insertions(+), 20 deletions(-) diff --git a/beacon_chain/ssz.nim b/beacon_chain/ssz.nim index a70884cfe..62466dcf5 100644 --- a/beacon_chain/ssz.nim +++ b/beacon_chain/ssz.nim @@ -206,8 +206,7 @@ proc writeVarSizeType(w: var SszWriter, value: auto) {.raises: [Defect, IOError] when T is seq|string|openarray: type E = ElemType(T) - const isFixed = when E is Option: false - else: isFixedSize(E) + const isFixed = isFixedSize(E) when isFixed: trs "WRITING LIST WITH FIXED SIZE ELEMENTS" for elem in value: @@ -243,6 +242,32 @@ proc writeValue*(w: var SszWriter, x: auto) {.gcsafe, raises: [Defect, IOError]. else: unsupported type(x) +func sszSize*(value: auto): int = + mixin toSszType + type T = type toSszType(value) + + when isFixedSize(T): + anonConst fixedPortionSize(T) + + elif T is seq|string|array|openarray: + type E = ElemType(T) + when isFixedSize(E): + len(value) * anonConst(fixedPortionSize(E)) + else: + result = len(value) * offsetSize + for elem in value: + result += sszSize(toSszType elem) + + elif T is object|tuple: + result = anonConst fixedPortionSize(T) + enumInstanceSerializedFields(value, _, field): + type FieldType = type toSszType(field) + when not isFixedSize(FieldType): + result += sszSize(toSszType field) + + else: + unsupported T + proc writeValue*[T](w: var SszWriter, x: SizePrefixed[T]) {.raises: [Defect, IOError].} = var cursor = w.stream.delayVarSizeWrite(10) let initPos = w.stream.pos @@ -465,7 +490,7 @@ func merkleizeSerializedChunks(merkleizer: SszChunksMerkleizer, # We assume there are no side-effects here, because the # SszHashingStream is keeping all of its output in memory. hashingStream.writeFixedSized obj - hashingStream.flush + close hashingStream merkleizer.getFinalHash except IOError as e: # Hashing shouldn't raise in theory but because of abstraction diff --git a/tests/official/fixtures_utils.nim b/tests/official/fixtures_utils.nim index 8fa440d7b..0d74f2708 100644 --- a/tests/official/fixtures_utils.nim +++ b/tests/official/fixtures_utils.nim @@ -34,6 +34,10 @@ proc readValue*(r: var JsonReader, a: var seq[byte]) {.inline.} = # ####################### # Test helpers +type + UnconsumedInput* = object of CatchableError + TestSizeError* = object of ValueError + const FixturesDir* = currentSourcePath.rsplit(DirSep, 1)[0] / ".." / ".." / "vendor" / "nim-eth2-scenarios" SszTestsDir* = FixturesDir/"tests-v0.11.1" @@ -47,3 +51,15 @@ proc parseTest*(path: string, Format: typedesc[Json or SSZ], T: typedesc): T = stderr.write $Format & " load issue for file \"", path, "\"\n" stderr.write err.formatMsg(path), "\n" quit 1 + +template readFileBytes*(path: string): seq[byte] = + cast[seq[byte]](readFile(path)) + +proc sszDecodeEntireInput*(input: openarray[byte], Decoded: type): Decoded = + var stream = unsafeMemoryInput(input) + var reader = init(SszReader, stream) + result = reader.readValue(Decoded) + + if stream.readable: + raise newException(UnconsumedInput, "Remaining bytes in the input") + diff --git a/tests/official/test_fixture_ssz_consensus_objects.nim b/tests/official/test_fixture_ssz_consensus_objects.nim index ecf763752..5bd466c30 100644 --- a/tests/official/test_fixture_ssz_consensus_objects.nim +++ b/tests/official/test_fixture_ssz_consensus_objects.nim @@ -41,10 +41,10 @@ setDefaultValue(SSZHashTreeRoot, signing_root, "") # Note this only tracks HashTreeRoot # Checking the values against the yaml file is TODO (require more flexible Yaml parser) - proc checkSSZ(T: type SignedBeaconBlock, dir: string, expectedHash: SSZHashTreeRoot) = # Deserialize into a ref object to not fill Nim stack - var deserialized = newClone(SSZ.loadFile(dir/"serialized.ssz", T)) + let encoded = readFileBytes(dir/"serialized.ssz") + var deserialized = newClone sszDecodeEntireInput(encoded, T) # SignedBeaconBlocks usually not hashed because they're identified by # htr(BeaconBlock), so do it manually @@ -52,15 +52,22 @@ proc checkSSZ(T: type SignedBeaconBlock, dir: string, expectedHash: SSZHashTreeR [hash_tree_root(deserialized.message), hash_tree_root(deserialized.signature)])) - # TODO check the value + check SSZ.encode(deserialized[]) == encoded + check sszSize(deserialized[]) == encoded.len -proc checkSSZ(T: typedesc, dir: string, expectedHash: SSZHashTreeRoot) = + # TODO check the value (requires YAML loader) + +proc checkSSZ(T: type, dir: string, expectedHash: SSZHashTreeRoot) = # Deserialize into a ref object to not fill Nim stack - var deserialized = newClone(SSZ.loadFile(dir/"serialized.ssz", T)) + let encoded = readFileBytes(dir/"serialized.ssz") + var deserialized = newClone sszDecodeEntireInput(encoded, T) check: expectedHash.root == "0x" & toLowerASCII($hash_tree_root(deserialized[])) - # TODO check the value + check SSZ.encode(deserialized[]) == encoded + check sszSize(deserialized[]) == encoded.len + + # TODO check the value (requires YAML loader) proc loadExpectedHashTreeRoot(dir: string): SSZHashTreeRoot = var s = openFileStream(dir/"roots.yaml") diff --git a/tests/official/test_fixture_ssz_generic_types.nim b/tests/official/test_fixture_ssz_generic_types.nim index 087281763..a4226619a 100644 --- a/tests/official/test_fixture_ssz_generic_types.nim +++ b/tests/official/test_fixture_ssz_generic_types.nim @@ -33,9 +33,6 @@ type # Containers have a root (thankfully) and signing_root field signing_root: string - UnconsumedInput* = object of CatchableError - TestSizeError* = object of ValueError - # Make signing root optional setDefaultValue(SSZHashTreeRoot, signing_root, "") @@ -79,18 +76,15 @@ type proc checkBasic(T: typedesc, dir: string, expectedHash: SSZHashTreeRoot) = - var fileContents = readFile(dir/"serialized.ssz") - var stream = unsafeMemoryInput(fileContents) - var reader = init(SszReader, stream) - var deserialized = reader.readValue(T) - - if stream.readable: - raise newException(UnconsumedInput, "Remaining bytes in the input") + var fileContents = readFileBytes(dir/"serialized.ssz") + var deserialized = sszDecodeEntireInput(fileContents, T) let expectedHash = expectedHash.root actualHash = "0x" & toLowerASCII($deserialized.hashTreeRoot()) + check expectedHash == actualHash + check sszSize(deserialized) == fileContents.len # TODO check the value diff --git a/vendor/nim-serialization b/vendor/nim-serialization index 1c0e0adf4..7b0d0f364 160000 --- a/vendor/nim-serialization +++ b/vendor/nim-serialization @@ -1 +1 @@ -Subproject commit 1c0e0adf459932e45b2a7483f704086cee7c906a +Subproject commit 7b0d0f364be07b0949d648b6a8b0d3614eb107a8