From f53b55cbe0cc7615b4f40c93be165283b2e568a1 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 27 May 2020 13:36:02 +0200 Subject: [PATCH] SSZ cleanup * be stricter about SSZ length prefix * compute zeroHash list at compile time * remove SSZ schema stuff * move SSZ navigation to ncli * cleanup a few leftover openArray uses --- Makefile | 1 + beacon_chain/beacon_node.nim | 24 +--- beacon_chain/conf.nim | 21 ---- beacon_chain/spec/state_transition_epoch.nim | 4 +- beacon_chain/ssz.nim | 71 ++++++----- beacon_chain/ssz/bytes_reader.nim | 17 ++- beacon_chain/ssz/types.nim | 117 +++---------------- ncli/ncli_query.nim | 56 +++++++++ tests/official/fixtures_utils.nim | 3 +- 9 files changed, 120 insertions(+), 194 deletions(-) create mode 100644 ncli/ncli_query.nim diff --git a/Makefile b/Makefile index ce18f7a91..f88c497a3 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ TOOLS := \ deposit_contract \ ncli_hash_tree_root \ ncli_pretty \ + ncli_query \ ncli_transition \ process_dashboard \ stack_sizes \ diff --git a/beacon_chain/beacon_node.nim b/beacon_chain/beacon_node.nim index d8a5b7ff1..0b9dca2cf 100644 --- a/beacon_chain/beacon_node.nim +++ b/beacon_chain/beacon_node.nim @@ -24,7 +24,7 @@ import attestation_pool, block_pool, eth2_network, eth2_discovery, beacon_node_common, beacon_node_types, nimbus_binary_common, - mainchain_monitor, version, ssz, ssz/dynamic_navigator, + mainchain_monitor, version, ssz, sync_protocol, request_manager, validator_keygen, interop, statusbar, sync_manager, state_transition, validator_duties, validator_api @@ -1012,25 +1012,3 @@ programMain: config.depositContractAddress, config.depositPrivateKey, delayGenerator) - - of query: - case config.queryCmd - of QueryCmd.nimQuery: - # TODO: This will handle a simple subset of Nim using - # dot syntax and `[]` indexing. - echo "nim query: ", config.nimQueryExpression - - of QueryCmd.get: - let pathFragments = config.getQueryPath.split('/', maxsplit = 1) - let bytes = - case pathFragments[0] - of "genesis_state": - readFile(config.dataDir/genesisFile).string.toBytes() - else: - stderr.write config.getQueryPath & " is not a valid path" - quit 1 - - let navigator = DynamicSszNavigator.init(bytes, BeaconState) - - echo navigator.navigatePath(pathFragments[1 .. ^1]).toJson - diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index bd833fe5c..15ce7ae07 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -18,15 +18,10 @@ type importValidator createTestnet makeDeposits - query VCStartUpCmd* = enum VCNoCommand - QueryCmd* = enum - nimQuery - get - Eth1Network* = enum custom mainnet @@ -270,22 +265,6 @@ type desc: "Maximum possible delay between making two deposits (in seconds)" name: "max-delay" }: float - of query: - case queryCmd* {. - defaultValue: nimQuery - command - desc: "Query the beacon node database and print the result" }: QueryCmd - - of nimQuery: - nimQueryExpression* {. - argument - desc: "Nim expression to evaluate (using limited syntax)" }: string - - of get: - getQueryPath* {. - argument - desc: "REST API path to evaluate" }: string - ValidatorClientConf* = object logLevel* {. defaultValue: "DEBUG" diff --git a/beacon_chain/spec/state_transition_epoch.nim b/beacon_chain/spec/state_transition_epoch.nim index 9e0f4b278..d74ff3340 100644 --- a/beacon_chain/spec/state_transition_epoch.nim +++ b/beacon_chain/spec/state_transition_epoch.nim @@ -384,7 +384,7 @@ func process_final_updates*(state: var BeaconState) {.nbench.}= # Reset eth1 data votes if next_epoch mod EPOCHS_PER_ETH1_VOTING_PERIOD == 0: - state.eth1_data_votes = type(state.eth1_data_votes) @[] + state.eth1_data_votes = default(type state.eth1_data_votes) # Update effective balances with hysteresis for index, validator in state.validators: @@ -420,7 +420,7 @@ func process_final_updates*(state: var BeaconState) {.nbench.}= # Rotate current/previous epoch attestations state.previous_epoch_attestations = state.current_epoch_attestations - state.current_epoch_attestations = typeof(state.current_epoch_attestations) @[] + state.current_epoch_attestations = default(type state.current_epoch_attestations) # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#epoch-processing proc process_epoch*(state: var BeaconState, updateFlags: UpdateFlags) diff --git a/beacon_chain/ssz.nim b/beacon_chain/ssz.nim index 5dfc5e23b..9562da1a1 100644 --- a/beacon_chain/ssz.nim +++ b/beacon_chain/ssz.nim @@ -29,12 +29,11 @@ export when defined(serialization_tracing): import - typetraits, stew/ranges/ptr_arith + typetraits const bytesPerChunk = 32 bitsPerChunk = bytesPerChunk * 8 - defaultMaxObjectSize = 1 * 1024 * 1024 type SszReader* = object @@ -63,6 +62,17 @@ serializationFormat SSZ, Writer = SszWriter, PreferedOutput = seq[byte] +template decode*(Format: type SSZ, + input: openarray[byte], + RecordType: distinct type): auto = + serialization.decode(SSZ, input, RecordType, maxObjectSize = input.len) + +template loadFile*(Format: type SSZ, + file: string, + RecordType: distinct type): auto = + let bytes = readFile(file) + decode(SSZ, toOpenArrayByte(string bytes, 0, bytes.high), RecordType) + template bytes(x: BitSeq): untyped = seq[byte](x) @@ -72,16 +82,12 @@ template sizePrefixed*[TT](x: TT): untyped = proc init*(T: type SszReader, stream: InputStream, - maxObjectSize = defaultMaxObjectSize): T {.raises: [Defect].} = + maxObjectSize: int): T {.raises: [Defect].} = T(stream: stream, maxObjectSize: maxObjectSize) -proc mount*(F: type SSZ, stream: InputStream, T: type): T {.raises: [Defect].} = - mixin readValue - var reader = init(SszReader, stream) - reader.readValue(T) - -method formatMsg*(err: ref SszSizeMismatchError, filename: string): string {.gcsafe, raises: [Defect].} = - # TODO: implement proper error string +method formatMsg*( + err: ref SszSizeMismatchError, + filename: string): string {.gcsafe, raises: [Defect].} = try: &"SSZ size mismatch, element {err.elementSize}, actual {err.actualSszSize}, type {err.deserializedType}, file {filename}" except CatchableError: @@ -96,7 +102,7 @@ template toSszType*(x: auto): auto = when x is Slot|Epoch|ValidatorIndex|enum: uint64(x) elif x is Eth2Digest: x.data elif x is BlsCurveType: toRaw(x) - elif x is ForkDigest|Version: array[4, byte](x) + elif x is ForkDigest|Version: distinctBase(x) else: x proc writeFixedSized(s: var (OutputStream|WriteCursor), x: auto) {.raises: [Defect, IOError].} = @@ -140,7 +146,7 @@ template enumerateSubFields(holder, fieldVar, body: untyped) = when holder is array: for fieldVar in holder: body else: - enumInstanceSerializedFields(holder, _, fieldVar): body + enumInstanceSerializedFields(holder, _{.used.}, fieldVar): body proc writeVarSizeType(w: var SszWriter, value: auto) {.gcsafe.} @@ -306,8 +312,8 @@ const func hash(a, b: openArray[byte]): Eth2Digest = result = withEth2Hash: trs "MERGING BRANCHES " - trs a - trs b + trs toHex(a) + trs toHex(b) h.update a h.update b @@ -316,8 +322,8 @@ func hash(a, b: openArray[byte]): Eth2Digest = func mergeBranches(existing: Eth2Digest, newData: openarray[byte]): Eth2Digest = result = withEth2Hash: trs "MERGING BRANCHES OPEN ARRAY" - trs existing.data - trs newData + trs toHex(existing.data) + trs toHex(newData) h.update existing.data h.update newData @@ -331,19 +337,12 @@ func mergeBranches(existing: Eth2Digest, newData: openarray[byte]): Eth2Digest = template mergeBranches(a, b: Eth2Digest): Eth2Digest = hash(a.data, b.data) -func computeZeroHashes: array[100, Eth2Digest] = +func computeZeroHashes: array[64, Eth2Digest] = result[0] = Eth2Digest(data: zeroChunk) for i in 1 .. result.high: result[i] = mergeBranches(result[i - 1], result[i - 1]) -let zeroHashes = computeZeroHashes() - -template getZeroHashWithoutSideEffect(idx: int): Eth2Digest = - # TODO this is a work-around for the somewhat broken side - # effects analysis of Nim - reading from global let variables - # is considered a side-effect. - {.noSideEffect.}: - zeroHashes[idx] +const zeroHashes = computeZeroHashes() func addChunk(merkleizer: var SszChunksMerkleizer, data: openarray[byte]) = doAssert data.len > 0 and data.len <= bytesPerChunk @@ -381,7 +380,7 @@ template createMerkleizer(totalElements: static Limit): SszChunksMerkleizer = func getFinalHash(merkleizer: var SszChunksMerkleizer): Eth2Digest = if merkleizer.totalChunks == 0: - return getZeroHashWithoutSideEffect(merkleizer.topIndex) + return zeroHashes[merkleizer.topIndex] let bottomHashIdx = firstOne(merkleizer.totalChunks) - 1 @@ -396,14 +395,14 @@ func getFinalHash(merkleizer: var SszChunksMerkleizer): Eth2Digest = # Our tree is not finished. We must complete the work in progress # branches and then extend the tree to the right height. result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx], - getZeroHashWithoutSideEffect(bottomHashIdx)) + zeroHashes[bottomHashIdx]) for i in bottomHashIdx + 1 ..< topHashIdx: if getBitLE(merkleizer.totalChunks, i): result = mergeBranches(merkleizer.combinedChunks[i], result) trs "COMBINED" else: - result = mergeBranches(result, getZeroHashWithoutSideEffect(i)) + result = mergeBranches(result, zeroHashes[i]) trs "COMBINED WITH ZERO" elif bottomHashIdx == topHashIdx: @@ -413,10 +412,10 @@ func getFinalHash(merkleizer: var SszChunksMerkleizer): Eth2Digest = # We have a perfect tree of user chunks, but we have more work to # do - we must extend it to reach the desired height result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx], - getZeroHashWithoutSideEffect(bottomHashIdx)) + zeroHashes[bottomHashIdx]) for i in bottomHashIdx + 1 ..< topHashIdx: - result = mergeBranches(result, getZeroHashWithoutSideEffect(i)) + result = mergeBranches(result, zeroHashes[i]) func mixInLength(root: Eth2Digest, length: int): Eth2Digest = var dataLen: array[32, byte] @@ -505,8 +504,8 @@ func bitListHashTreeRoot(merkleizer: var SszChunksMerkleizer, x: BitSeq): Eth2Di if totalBytes == 1: # This is an empty bit list. # It should be hashed as a tree containing all zeros: - return mergeBranches(getZeroHashWithoutSideEffect(merkleizer.topIndex), - getZeroHashWithoutSideEffect(0)) # this is the mixed length + return mergeBranches(zeroHashes[merkleizer.topIndex], + zeroHashes[0]) # this is the mixed length totalBytes -= 1 lastCorrectedByte = bytes(x)[^2] @@ -595,14 +594,14 @@ func hashTreeRootAux[T](x: T): Eth2Digest = func hash_tree_root*(x: auto): Eth2Digest {.raises: [Defect], nbench.} = trs "STARTING HASH TREE ROOT FOR TYPE ", name(type(x)) mixin toSszType - when x is List|BitList: + result = when x is List|BitList: const maxLen = static(x.maxLen) type T = type(x) const limit = maxChunksCount(T, maxLen) var merkleizer = createMerkleizer(limit) when x is BitList: - result = merkleizer.bitListHashTreeRoot(BitSeq x) + merkleizer.bitListHashTreeRoot(BitSeq x) else: type E = ElemType(T) let contentsHash = when E is BasicType: @@ -612,9 +611,9 @@ func hash_tree_root*(x: auto): Eth2Digest {.raises: [Defect], nbench.} = let elemHash = hash_tree_root(elem) merkleizer.addChunk(elemHash.data) merkleizer.getFinalHash() - result = mixInLength(contentsHash, x.len) + mixInLength(contentsHash, x.len) else: - result = hashTreeRootAux toSszType(x) + hashTreeRootAux toSszType(x) trs "HASH TREE ROOT FOR ", name(type x), " = ", "0x", $result diff --git a/beacon_chain/ssz/bytes_reader.nim b/beacon_chain/ssz/bytes_reader.nim index 1ec5645e7..6a08bad88 100644 --- a/beacon_chain/ssz/bytes_reader.nim +++ b/beacon_chain/ssz/bytes_reader.nim @@ -31,8 +31,7 @@ func fromSszBytes*(T: type UintN, data: openarray[byte]): T {.raisesssz.} = T.fromBytesLE(data) func fromSszBytes*(T: type bool, data: openarray[byte]): T {.raisesssz.} = - # TODO: spec doesn't say what to do if the value is >1 - we'll use the C - # definition for now, but maybe this should be a parse error instead? + # Strict: only allow 0 or 1 if data.len != 1 or byte(data[0]) > byte(1): raise newException(MalformedSszError, "invalid boolean value") data[0] == 1 @@ -123,14 +122,14 @@ func readSszValue*(input: openarray[byte], T: type): T {.raisesssz.} = checkForForbiddenBits(T, input, result.maxLen + 1) elif result is List|array: - type ElemType = type result[0] - when ElemType is byte: + type E = type result[0] + when E is byte: result.setOutputSize input.len if input.len > 0: copyMem(addr result[0], unsafeAddr input[0], input.len) - elif isFixedSize(ElemType): - const elemSize = fixedPortionSize(ElemType) + elif isFixedSize(E): + const elemSize = fixedPortionSize(E) if input.len mod elemSize != 0: var ex = new SszSizeMismatchError ex.deserializedType = cstring typetraits.name(T) @@ -142,7 +141,7 @@ func readSszValue*(input: openarray[byte], T: type): T {.raisesssz.} = for i in 0 ..< result.len: trs "TRYING TO READ LIST ELEM ", i let offset = i * elemSize - result[i] = readSszValue(input.toOpenArray(offset, offset + elemSize - 1), ElemType) + result[i] = readSszValue(input.toOpenArray(offset, offset + elemSize - 1), E) trs "LIST READING COMPLETE" else: @@ -171,10 +170,10 @@ func readSszValue*(input: openarray[byte], T: type): T {.raisesssz.} = if nextOffset <= offset: raise newException(MalformedSszError, "SSZ list element offsets are not monotonically increasing") else: - result[i - 1] = readSszValue(input.toOpenArray(offset, nextOffset - 1), ElemType) + result[i - 1] = readSszValue(input.toOpenArray(offset, nextOffset - 1), E) offset = nextOffset - result[resultLen - 1] = readSszValue(input.toOpenArray(offset, input.len - 1), ElemType) + result[resultLen - 1] = readSszValue(input.toOpenArray(offset, input.len - 1), E) # TODO: Should be possible to remove BitArray from here elif result is UintN|bool|enum: diff --git a/beacon_chain/ssz/types.nim b/beacon_chain/ssz/types.nim index 42bc51667..03d94c812 100644 --- a/beacon_chain/ssz/types.nim +++ b/beacon_chain/ssz/types.nim @@ -30,48 +30,6 @@ type actualSszSize*: int elementSize*: int - SszChunksLimitExceeded* = object of SszError - - SszSchema* = ref object - nodes*: seq[SszNode] - - SszTypeKind* = enum - sszNull - sszUInt - sszBool - sszList - sszVector - sszBitList - sszBitVector - sszRecord - - SszType* = ref object - case kind*: SszTypeKind - of sszUInt, sszBitVector: - bits*: int - of sszBool, sszNull, sszBitList: - discard - of sszVector: - size*: int - vectorElemType*: SszType - of sszList: - listElemType*: SszType - of sszRecord: - schema*: SszSchema - - SszNodeKind* = enum - Field - Union - - SszNode* = ref object - name*: string - typ*: SszType - case kind: SszNodeKind - of Union: - variants*: seq[SszSchema] - of Field: - discard - template asSeq*(x: List): auto = distinctBase(x) template init*[T](L: type List, x: seq[T], N: static Limit): auto = @@ -81,21 +39,21 @@ template init*[T, N](L: type List[T, N], x: seq[T]): auto = List[T, N](x) template `$`*(x: List): auto = $(distinctBase x) -template add*(x: List, val: auto) = add(distinctBase x, val) +template add*(x: var List, val: auto) = add(distinctBase x, val) template len*(x: List): auto = len(distinctBase x) -template setLen*(x: List, val: auto) = setLen(distinctBase x, val) +template setLen*(x: var List, val: auto) = setLen(distinctBase x, val) template low*(x: List): auto = low(distinctBase x) template high*(x: List): auto = high(distinctBase x) template `[]`*(x: List, idx: auto): untyped = distinctBase(x)[idx] -template `[]=`*(x: List, idx: auto, val: auto) = distinctBase(x)[idx] = val +template `[]=`*(x: var List, idx: auto, val: auto) = distinctBase(x)[idx] = val template `==`*(a, b: List): bool = asSeq(a) == distinctBase(b) template `&`*(a, b: List): auto = (type(a)(distinctBase(a) & distinctBase(b))) template items* (x: List): untyped = items(distinctBase x) template pairs* (x: List): untyped = pairs(distinctBase x) -template mitems*(x: List): untyped = mitems(distinctBase x) -template mpairs*(x: List): untyped = mpairs(distinctBase x) +template mitems*(x: var List): untyped = mitems(distinctBase x) +template mpairs*(x: var List): untyped = mpairs(distinctBase x) template init*(L: type BitList, x: seq[byte], N: static Limit): auto = BitList[N](data: x) @@ -130,32 +88,27 @@ macro unsupported*(T: typed): untyped = template ElemType*(T: type[array]): untyped = type(default(T)[low(T)]) -template ElemType*[T](A: type[openarray[T]]): untyped = - T - template ElemType*(T: type[seq|List]): untyped = type(default(T)[0]) func isFixedSize*(T0: type): bool {.compileTime.} = mixin toSszType, enumAllSerializedFields - when T0 is openarray: - return false - else: - type T = type toSszType(declval T0) + type T = type toSszType(declval T0) - when T is BasicType: - return true - elif T is array: - return isFixedSize(ElemType(T)) - elif T is object|tuple: - enumAllSerializedFields(T): - when not isFixedSize(FieldType): - return false - return true + when T is BasicType: + return true + elif T is array: + return isFixedSize(ElemType(T)) + elif T is object|tuple: + enumAllSerializedFields(T): + when not isFixedSize(FieldType): + return false + return true func fixedPortionSize*(T0: type): int {.compileTime.} = mixin enumAllSerializedFields, toSszType + type T = type toSszType(declval T0) when T is BasicType: sizeof(T) @@ -163,7 +116,6 @@ func fixedPortionSize*(T0: type): int {.compileTime.} = type E = ElemType(T) when isFixedSize(E): len(T) * fixedPortionSize(E) else: len(T) * offsetSize - elif T is seq|openarray: offsetSize elif T is object|tuple: enumAllSerializedFields(T): when isFixedSize(FieldType): @@ -173,43 +125,6 @@ func fixedPortionSize*(T0: type): int {.compileTime.} = else: unsupported T0 -func sszSchemaType*(T0: type): SszType {.compileTime.} = - mixin toSszType, enumAllSerializedFields - type T = type toSszType(declval T0) - - when T is bool: - SszType(kind: sszBool) - elif T is uint8|char: - SszType(kind: sszUInt, bits: 8) - elif T is uint16: - SszType(kind: sszUInt, bits: 16) - elif T is uint32: - SszType(kind: sszUInt, bits: 32) - elif T is uint64: - SszType(kind: sszUInt, bits: 64) - elif T is seq: - SszType(kind: sszList, listElemType: sszSchemaType(ElemType(T))) - elif T is array: - SszType(kind: sszVector, vectorElemType: sszSchemaType(ElemType(T))) - elif T is BitArray: - SszType(kind: sszBitVector, bits: T.bits) - elif T is BitSeq: - SszType(kind: sszBitList) - elif T is object|tuple: - var recordSchema = SszSchema() - var caseBranches = initTable[string, SszSchema]() - caseBranches[""] = recordSchema - # TODO case objects are still not supported here. - # `recordFields` has to be refactored to properly - # report nested discriminator fields. - enumAllSerializedFields(T): - recordSchema.nodes.add SszNode( - name: fieldName, - typ: sszSchemaType(FieldType), - kind: Field) - else: - unsupported T0 - # TODO This should have been an iterator, but the VM can't compile the # code due to "too many registers required". proc fieldInfos*(RecordType: type): seq[tuple[name: string, diff --git a/ncli/ncli_query.nim b/ncli/ncli_query.nim new file mode 100644 index 000000000..46281e0cd --- /dev/null +++ b/ncli/ncli_query.nim @@ -0,0 +1,56 @@ +import + confutils, os, strutils, chronicles, json_serialization, + stew/byteutils, + ../beacon_chain/spec/[crypto, datatypes, digest], + ../beacon_chain/[ssz], + ../beacon_chain/ssz/dynamic_navigator + +type + QueryCmd* = enum + nimQuery + get + + QueryConf = object + file* {. + defaultValue: "" + desc: "BeaconState ssz file" + name: "file" }: InputFile + + case queryCmd* {. + defaultValue: nimQuery + command + desc: "Query the beacon node database and print the result" }: QueryCmd + + of nimQuery: + nimQueryExpression* {. + argument + desc: "Nim expression to evaluate (using limited syntax)" }: string + + of get: + getQueryPath* {. + argument + desc: "REST API path to evaluate" }: string + + +let + config = QueryConf.load() + +case config.queryCmd +of QueryCmd.nimQuery: + # TODO: This will handle a simple subset of Nim using + # dot syntax and `[]` indexing. + echo "nim query: ", config.nimQueryExpression + +of QueryCmd.get: + let pathFragments = config.getQueryPath.split('/', maxsplit = 1) + let bytes = + case pathFragments[0] + of "genesis_state": + readFile(config.file.string).string.toBytes() + else: + stderr.write config.getQueryPath & " is not a valid path" + quit 1 + + let navigator = DynamicSszNavigator.init(bytes, BeaconState) + + echo navigator.navigatePath(pathFragments[1 .. ^1]).toJson diff --git a/tests/official/fixtures_utils.nim b/tests/official/fixtures_utils.nim index 110aecafe..661d96b42 100644 --- a/tests/official/fixtures_utils.nim +++ b/tests/official/fixtures_utils.nim @@ -57,9 +57,8 @@ template readFileBytes*(path: string): seq[byte] = proc sszDecodeEntireInput*(input: openarray[byte], Decoded: type): Decoded = var stream = unsafeMemoryInput(input) - var reader = init(SszReader, stream) + var reader = init(SszReader, stream, input.len) result = reader.readValue(Decoded) if stream.readable: raise newException(UnconsumedInput, "Remaining bytes in the input") -