SSZ cleanup

* be stricter about SSZ length prefix
* compute zeroHash list at compile time
* remove SSZ schema stuff
* move SSZ navigation to ncli
* cleanup a few leftover openArray uses

Makefile

@@ -22,6 +22,7 @@ TOOLS := \
 	deposit_contract \
 	ncli_hash_tree_root \
 	ncli_pretty \
+	ncli_query \
 	ncli_transition \
 	process_dashboard \
 	stack_sizes \

beacon_chain/beacon_node.nim

@@ -24,7 +24,7 @@ import
   attestation_pool, block_pool, eth2_network, eth2_discovery,
   beacon_node_common, beacon_node_types,
   nimbus_binary_common,
-  mainchain_monitor, version, ssz, ssz/dynamic_navigator,
+  mainchain_monitor, version, ssz,
   sync_protocol, request_manager, validator_keygen, interop, statusbar,
   sync_manager, state_transition,
   validator_duties, validator_api
@@ -1012,25 +1012,3 @@ programMain:
         config.depositContractAddress,
         config.depositPrivateKey,
         delayGenerator)
-
-  of query:
-    case config.queryCmd
-    of QueryCmd.nimQuery:
-      # TODO: This will handle a simple subset of Nim using
-      #       dot syntax and `[]` indexing.
-      echo "nim query: ", config.nimQueryExpression
-
-    of QueryCmd.get:
-      let pathFragments = config.getQueryPath.split('/', maxsplit = 1)
-      let bytes =
-        case pathFragments[0]
-        of "genesis_state":
-          readFile(config.dataDir/genesisFile).string.toBytes()
-        else:
-          stderr.write config.getQueryPath & " is not a valid path"
-          quit 1
-
-      let navigator = DynamicSszNavigator.init(bytes, BeaconState)
-
-      echo navigator.navigatePath(pathFragments[1 .. ^1]).toJson
-

beacon_chain/conf.nim

@@ -18,15 +18,10 @@ type
     importValidator
     createTestnet
     makeDeposits
-    query
 
   VCStartUpCmd* = enum
     VCNoCommand
 
-  QueryCmd* = enum
-    nimQuery
-    get
-
   Eth1Network* = enum
     custom
     mainnet
@@ -270,22 +265,6 @@ type
         desc: "Maximum possible delay between making two deposits (in seconds)"
         name: "max-delay" }: float
 
-    of query:
-      case queryCmd* {.
-        defaultValue: nimQuery
-        command
-        desc: "Query the beacon node database and print the result" }: QueryCmd
-
-      of nimQuery:
-        nimQueryExpression* {.
-          argument
-          desc: "Nim expression to evaluate (using limited syntax)" }: string
-
-      of get:
-        getQueryPath* {.
-          argument
-          desc: "REST API path to evaluate" }: string
-
   ValidatorClientConf* = object
     logLevel* {.
       defaultValue: "DEBUG"

beacon_chain/spec/state_transition_epoch.nim

@@ -384,7 +384,7 @@ func process_final_updates*(state: var BeaconState) {.nbench.}=
 
   # Reset eth1 data votes
   if next_epoch mod EPOCHS_PER_ETH1_VOTING_PERIOD == 0:
-    state.eth1_data_votes = type(state.eth1_data_votes) @[]
+    state.eth1_data_votes = default(type state.eth1_data_votes)
 
   # Update effective balances with hysteresis
   for index, validator in state.validators:
@@ -420,7 +420,7 @@ func process_final_updates*(state: var BeaconState) {.nbench.}=
 
   # Rotate current/previous epoch attestations
   state.previous_epoch_attestations = state.current_epoch_attestations
-  state.current_epoch_attestations = typeof(state.current_epoch_attestations) @[]
+  state.current_epoch_attestations = default(type state.current_epoch_attestations)
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/beacon-chain.md#epoch-processing
 proc process_epoch*(state: var BeaconState, updateFlags: UpdateFlags)

beacon_chain/ssz.nim

@@ -29,12 +29,11 @@ export
 
 when defined(serialization_tracing):
   import
-    typetraits, stew/ranges/ptr_arith
+    typetraits
 
 const
   bytesPerChunk = 32
   bitsPerChunk = bytesPerChunk * 8
-  defaultMaxObjectSize = 1 * 1024 * 1024
 
 type
   SszReader* = object
@@ -63,6 +62,17 @@ serializationFormat SSZ,
                     Writer = SszWriter,
                     PreferedOutput = seq[byte]
 
+template decode*(Format: type SSZ,
+                 input: openarray[byte],
+                 RecordType: distinct type): auto =
+  serialization.decode(SSZ, input, RecordType, maxObjectSize = input.len)
+
+template loadFile*(Format: type SSZ,
+                   file: string,
+                   RecordType: distinct type): auto =
+  let bytes = readFile(file)
+  decode(SSZ, toOpenArrayByte(string bytes, 0, bytes.high), RecordType)
+
 template bytes(x: BitSeq): untyped =
   seq[byte](x)
 
@@ -72,16 +82,12 @@ template sizePrefixed*[TT](x: TT): untyped =
 
 proc init*(T: type SszReader,
            stream: InputStream,
-           maxObjectSize = defaultMaxObjectSize): T {.raises: [Defect].} =
+           maxObjectSize: int): T {.raises: [Defect].} =
   T(stream: stream, maxObjectSize: maxObjectSize)
 
-proc mount*(F: type SSZ, stream: InputStream, T: type): T {.raises: [Defect].} =
+method formatMsg*(
-  mixin readValue
+  err: ref SszSizeMismatchError,
-  var reader = init(SszReader, stream)
+  filename: string): string {.gcsafe, raises: [Defect].} =
-  reader.readValue(T)
-
-method formatMsg*(err: ref SszSizeMismatchError, filename: string): string {.gcsafe, raises: [Defect].} =
-  # TODO: implement proper error string
   try:
     &"SSZ size mismatch, element {err.elementSize}, actual {err.actualSszSize}, type {err.deserializedType}, file {filename}"
   except CatchableError:
@@ -96,7 +102,7 @@ template toSszType*(x: auto): auto =
   when x is Slot|Epoch|ValidatorIndex|enum: uint64(x)
   elif x is Eth2Digest: x.data
   elif x is BlsCurveType: toRaw(x)
-  elif x is ForkDigest|Version: array[4, byte](x)
+  elif x is ForkDigest|Version: distinctBase(x)
   else: x
 
 proc writeFixedSized(s: var (OutputStream|WriteCursor), x: auto) {.raises: [Defect, IOError].} =
@@ -140,7 +146,7 @@ template enumerateSubFields(holder, fieldVar, body: untyped) =
   when holder is array:
     for fieldVar in holder: body
   else:
-    enumInstanceSerializedFields(holder, _, fieldVar): body
+    enumInstanceSerializedFields(holder, _{.used.}, fieldVar): body
 
 proc writeVarSizeType(w: var SszWriter, value: auto) {.gcsafe.}
 
@@ -306,8 +312,8 @@ const
 func hash(a, b: openArray[byte]): Eth2Digest =
   result = withEth2Hash:
     trs "MERGING BRANCHES "
-    trs a
+    trs toHex(a)
-    trs b
+    trs toHex(b)
 
     h.update a
     h.update b
@@ -316,8 +322,8 @@ func hash(a, b: openArray[byte]): Eth2Digest =
 func mergeBranches(existing: Eth2Digest, newData: openarray[byte]): Eth2Digest =
   result = withEth2Hash:
     trs "MERGING BRANCHES OPEN ARRAY"
-    trs existing.data
+    trs toHex(existing.data)
-    trs newData
+    trs toHex(newData)
 
     h.update existing.data
     h.update newData
@@ -331,19 +337,12 @@ func mergeBranches(existing: Eth2Digest, newData: openarray[byte]): Eth2Digest =
 template mergeBranches(a, b: Eth2Digest): Eth2Digest =
   hash(a.data, b.data)
 
-func computeZeroHashes: array[100, Eth2Digest] =
+func computeZeroHashes: array[64, Eth2Digest] =
   result[0] = Eth2Digest(data: zeroChunk)
   for i in 1 .. result.high:
     result[i] = mergeBranches(result[i - 1], result[i - 1])
 
-let zeroHashes = computeZeroHashes()
+const zeroHashes = computeZeroHashes()
-
-template getZeroHashWithoutSideEffect(idx: int): Eth2Digest =
-  # TODO this is a work-around for the somewhat broken side
-  # effects analysis of Nim - reading from global let variables
-  # is considered a side-effect.
-  {.noSideEffect.}:
-    zeroHashes[idx]
 
 func addChunk(merkleizer: var SszChunksMerkleizer, data: openarray[byte]) =
   doAssert data.len > 0 and data.len <= bytesPerChunk
@@ -381,7 +380,7 @@ template createMerkleizer(totalElements: static Limit): SszChunksMerkleizer =
 
 func getFinalHash(merkleizer: var SszChunksMerkleizer): Eth2Digest =
   if merkleizer.totalChunks == 0:
-    return getZeroHashWithoutSideEffect(merkleizer.topIndex)
+    return zeroHashes[merkleizer.topIndex]
 
   let
     bottomHashIdx = firstOne(merkleizer.totalChunks) - 1
@@ -396,14 +395,14 @@ func getFinalHash(merkleizer: var SszChunksMerkleizer): Eth2Digest =
     # Our tree is not finished. We must complete the work in progress
     # branches and then extend the tree to the right height.
     result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx],
-                           getZeroHashWithoutSideEffect(bottomHashIdx))
+                           zeroHashes[bottomHashIdx])
 
     for i in bottomHashIdx + 1 ..< topHashIdx:
       if getBitLE(merkleizer.totalChunks, i):
         result = mergeBranches(merkleizer.combinedChunks[i], result)
         trs "COMBINED"
       else:
-        result = mergeBranches(result, getZeroHashWithoutSideEffect(i))
+        result = mergeBranches(result, zeroHashes[i])
         trs "COMBINED WITH ZERO"
 
   elif bottomHashIdx == topHashIdx:
@@ -413,10 +412,10 @@ func getFinalHash(merkleizer: var SszChunksMerkleizer): Eth2Digest =
     # We have a perfect tree of user chunks, but we have more work to
     # do - we must extend it to reach the desired height
     result = mergeBranches(merkleizer.combinedChunks[bottomHashIdx],
-                           getZeroHashWithoutSideEffect(bottomHashIdx))
+                           zeroHashes[bottomHashIdx])
 
     for i in bottomHashIdx + 1 ..< topHashIdx:
-      result = mergeBranches(result, getZeroHashWithoutSideEffect(i))
+      result = mergeBranches(result, zeroHashes[i])
 
 func mixInLength(root: Eth2Digest, length: int): Eth2Digest =
   var dataLen: array[32, byte]
@@ -505,8 +504,8 @@ func bitListHashTreeRoot(merkleizer: var SszChunksMerkleizer, x: BitSeq): Eth2Di
     if totalBytes == 1:
       # This is an empty bit list.
       # It should be hashed as a tree containing all zeros:
-      return mergeBranches(getZeroHashWithoutSideEffect(merkleizer.topIndex),
+      return mergeBranches(zeroHashes[merkleizer.topIndex],
-                           getZeroHashWithoutSideEffect(0)) # this is the mixed length
+                           zeroHashes[0]) # this is the mixed length
 
     totalBytes -= 1
     lastCorrectedByte = bytes(x)[^2]
@@ -595,14 +594,14 @@ func hashTreeRootAux[T](x: T): Eth2Digest =
 func hash_tree_root*(x: auto): Eth2Digest {.raises: [Defect], nbench.} =
   trs "STARTING HASH TREE ROOT FOR TYPE ", name(type(x))
   mixin toSszType
-  when x is List|BitList:
+  result = when x is List|BitList:
     const maxLen = static(x.maxLen)
     type T = type(x)
     const limit = maxChunksCount(T, maxLen)
     var merkleizer = createMerkleizer(limit)
 
     when x is BitList:
-      result = merkleizer.bitListHashTreeRoot(BitSeq x)
+      merkleizer.bitListHashTreeRoot(BitSeq x)
     else:
       type E = ElemType(T)
       let contentsHash = when E is BasicType:
@@ -612,9 +611,9 @@ func hash_tree_root*(x: auto): Eth2Digest {.raises: [Defect], nbench.} =
           let elemHash = hash_tree_root(elem)
           merkleizer.addChunk(elemHash.data)
         merkleizer.getFinalHash()
-      result = mixInLength(contentsHash, x.len)
+      mixInLength(contentsHash, x.len)
   else:
-    result = hashTreeRootAux toSszType(x)
+    hashTreeRootAux toSszType(x)
 
   trs "HASH TREE ROOT FOR ", name(type x), " = ", "0x", $result
 

beacon_chain/ssz/bytes_reader.nim

@@ -31,8 +31,7 @@ func fromSszBytes*(T: type UintN, data: openarray[byte]): T {.raisesssz.} =
   T.fromBytesLE(data)
 
 func fromSszBytes*(T: type bool, data: openarray[byte]): T {.raisesssz.} =
-  # TODO: spec doesn't say what to do if the value is >1 - we'll use the C
+  # Strict: only allow 0 or 1
-  #       definition for now, but maybe this should be a parse error instead?
   if data.len != 1 or byte(data[0]) > byte(1):
     raise newException(MalformedSszError, "invalid boolean value")
   data[0] == 1
@@ -123,14 +122,14 @@ func readSszValue*(input: openarray[byte], T: type): T {.raisesssz.} =
       checkForForbiddenBits(T, input, result.maxLen + 1)
 
   elif result is List|array:
-    type ElemType = type result[0]
+    type E = type result[0]
-    when ElemType is byte:
+    when E is byte:
       result.setOutputSize input.len
       if input.len > 0:
         copyMem(addr result[0], unsafeAddr input[0], input.len)
 
-    elif isFixedSize(ElemType):
+    elif isFixedSize(E):
-      const elemSize = fixedPortionSize(ElemType)
+      const elemSize = fixedPortionSize(E)
       if input.len mod elemSize != 0:
         var ex = new SszSizeMismatchError
         ex.deserializedType = cstring typetraits.name(T)
@@ -142,7 +141,7 @@ func readSszValue*(input: openarray[byte], T: type): T {.raisesssz.} =
       for i in 0 ..< result.len:
         trs "TRYING TO READ LIST ELEM ", i
         let offset = i * elemSize
-        result[i] = readSszValue(input.toOpenArray(offset, offset + elemSize - 1), ElemType)
+        result[i] = readSszValue(input.toOpenArray(offset, offset + elemSize - 1), E)
       trs "LIST READING COMPLETE"
 
     else:
@@ -171,10 +170,10 @@ func readSszValue*(input: openarray[byte], T: type): T {.raisesssz.} =
         if nextOffset <= offset:
           raise newException(MalformedSszError, "SSZ list element offsets are not monotonically increasing")
         else:
-          result[i - 1] = readSszValue(input.toOpenArray(offset, nextOffset - 1), ElemType)
+          result[i - 1] = readSszValue(input.toOpenArray(offset, nextOffset - 1), E)
         offset = nextOffset
 
-      result[resultLen - 1] = readSszValue(input.toOpenArray(offset, input.len - 1), ElemType)
+      result[resultLen - 1] = readSszValue(input.toOpenArray(offset, input.len - 1), E)
 
   # TODO: Should be possible to remove BitArray from here
   elif result is UintN|bool|enum:

beacon_chain/ssz/types.nim

@@ -30,48 +30,6 @@ type
     actualSszSize*: int
     elementSize*: int
 
-  SszChunksLimitExceeded* = object of SszError
-
-  SszSchema* = ref object
-    nodes*: seq[SszNode]
-
-  SszTypeKind* = enum
-    sszNull
-    sszUInt
-    sszBool
-    sszList
-    sszVector
-    sszBitList
-    sszBitVector
-    sszRecord
-
-  SszType* = ref object
-    case kind*: SszTypeKind
-    of sszUInt, sszBitVector:
-      bits*: int
-    of sszBool, sszNull, sszBitList:
-      discard
-    of sszVector:
-      size*: int
-      vectorElemType*: SszType
-    of sszList:
-      listElemType*: SszType
-    of sszRecord:
-      schema*: SszSchema
-
-  SszNodeKind* = enum
-    Field
-    Union
-
-  SszNode* = ref object
-    name*: string
-    typ*: SszType
-    case kind: SszNodeKind
-    of Union:
-      variants*: seq[SszSchema]
-    of Field:
-      discard
-
 template asSeq*(x: List): auto = distinctBase(x)
 
 template init*[T](L: type List, x: seq[T], N: static Limit): auto =
@@ -81,21 +39,21 @@ template init*[T, N](L: type List[T, N], x: seq[T]): auto =
   List[T, N](x)
 
 template `$`*(x: List): auto = $(distinctBase x)
-template add*(x: List, val: auto) = add(distinctBase x, val)
+template add*(x: var List, val: auto) = add(distinctBase x, val)
 template len*(x: List): auto = len(distinctBase x)
-template setLen*(x: List, val: auto) = setLen(distinctBase x, val)
+template setLen*(x: var List, val: auto) = setLen(distinctBase x, val)
 template low*(x: List): auto = low(distinctBase x)
 template high*(x: List): auto = high(distinctBase x)
 template `[]`*(x: List, idx: auto): untyped = distinctBase(x)[idx]
-template `[]=`*(x: List, idx: auto, val: auto) = distinctBase(x)[idx] = val
+template `[]=`*(x: var List, idx: auto, val: auto) = distinctBase(x)[idx] = val
 template `==`*(a, b: List): bool = asSeq(a) == distinctBase(b)
 
 template `&`*(a, b: List): auto = (type(a)(distinctBase(a) & distinctBase(b)))
 
 template items* (x: List): untyped = items(distinctBase x)
 template pairs* (x: List): untyped = pairs(distinctBase x)
-template mitems*(x: List): untyped = mitems(distinctBase x)
+template mitems*(x: var List): untyped = mitems(distinctBase x)
-template mpairs*(x: List): untyped = mpairs(distinctBase x)
+template mpairs*(x: var List): untyped = mpairs(distinctBase x)
 
 template init*(L: type BitList, x: seq[byte], N: static Limit): auto =
   BitList[N](data: x)
@@ -130,32 +88,27 @@ macro unsupported*(T: typed): untyped =
 template ElemType*(T: type[array]): untyped =
   type(default(T)[low(T)])
 
-template ElemType*[T](A: type[openarray[T]]): untyped =
-  T
-
 template ElemType*(T: type[seq|List]): untyped =
   type(default(T)[0])
 
 func isFixedSize*(T0: type): bool {.compileTime.} =
   mixin toSszType, enumAllSerializedFields
 
-  when T0 is openarray:
+  type T = type toSszType(declval T0)
-    return false
-  else:
-    type T = type toSszType(declval T0)
 
-    when T is BasicType:
+  when T is BasicType:
-      return true
+    return true
-    elif T is array:
+  elif T is array:
-      return isFixedSize(ElemType(T))
+    return isFixedSize(ElemType(T))
-    elif T is object|tuple:
+  elif T is object|tuple:
-      enumAllSerializedFields(T):
+    enumAllSerializedFields(T):
-        when not isFixedSize(FieldType):
+      when not isFixedSize(FieldType):
-          return false
+        return false
-      return true
+    return true
 
 func fixedPortionSize*(T0: type): int {.compileTime.} =
   mixin enumAllSerializedFields, toSszType
+
   type T = type toSszType(declval T0)
 
   when T is BasicType: sizeof(T)
@@ -163,7 +116,6 @@ func fixedPortionSize*(T0: type): int {.compileTime.} =
     type E = ElemType(T)
     when isFixedSize(E): len(T) * fixedPortionSize(E)
     else: len(T) * offsetSize
-  elif T is seq|openarray: offsetSize
   elif T is object|tuple:
     enumAllSerializedFields(T):
       when isFixedSize(FieldType):
@@ -173,43 +125,6 @@ func fixedPortionSize*(T0: type): int {.compileTime.} =
   else:
     unsupported T0
 
-func sszSchemaType*(T0: type): SszType {.compileTime.} =
-  mixin toSszType, enumAllSerializedFields
-  type T = type toSszType(declval T0)
-
-  when T is bool:
-    SszType(kind: sszBool)
-  elif T is uint8|char:
-    SszType(kind: sszUInt, bits: 8)
-  elif T is uint16:
-    SszType(kind: sszUInt, bits: 16)
-  elif T is uint32:
-    SszType(kind: sszUInt, bits: 32)
-  elif T is uint64:
-    SszType(kind: sszUInt, bits: 64)
-  elif T is seq:
-    SszType(kind: sszList, listElemType: sszSchemaType(ElemType(T)))
-  elif T is array:
-    SszType(kind: sszVector, vectorElemType: sszSchemaType(ElemType(T)))
-  elif T is BitArray:
-    SszType(kind: sszBitVector, bits: T.bits)
-  elif T is BitSeq:
-    SszType(kind: sszBitList)
-  elif T is object|tuple:
-    var recordSchema = SszSchema()
-    var caseBranches = initTable[string, SszSchema]()
-    caseBranches[""] = recordSchema
-    # TODO case objects are still not supported here.
-    # `recordFields` has to be refactored to properly
-    # report nested discriminator fields.
-    enumAllSerializedFields(T):
-      recordSchema.nodes.add SszNode(
-        name: fieldName,
-        typ: sszSchemaType(FieldType),
-        kind: Field)
-  else:
-    unsupported T0
-
 # TODO This should have been an iterator, but the VM can't compile the
 # code due to "too many registers required".
 proc fieldInfos*(RecordType: type): seq[tuple[name: string,

ncli/ncli_query.nim

@@ -0,0 +1,56 @@
+import
+  confutils, os, strutils, chronicles, json_serialization,
+  stew/byteutils,
+  ../beacon_chain/spec/[crypto, datatypes, digest],
+  ../beacon_chain/[ssz],
+  ../beacon_chain/ssz/dynamic_navigator
+
+type
+  QueryCmd* = enum
+    nimQuery
+    get
+
+  QueryConf = object
+    file* {.
+        defaultValue: ""
+        desc: "BeaconState ssz file"
+        name: "file" }: InputFile
+
+    case queryCmd* {.
+      defaultValue: nimQuery
+      command
+      desc: "Query the beacon node database and print the result" }: QueryCmd
+
+    of nimQuery:
+      nimQueryExpression* {.
+        argument
+        desc: "Nim expression to evaluate (using limited syntax)" }: string
+
+    of get:
+      getQueryPath* {.
+        argument
+        desc: "REST API path to evaluate" }: string
+
+
+let
+  config = QueryConf.load()
+
+case config.queryCmd
+of QueryCmd.nimQuery:
+  # TODO: This will handle a simple subset of Nim using
+  #       dot syntax and `[]` indexing.
+  echo "nim query: ", config.nimQueryExpression
+
+of QueryCmd.get:
+  let pathFragments = config.getQueryPath.split('/', maxsplit = 1)
+  let bytes =
+    case pathFragments[0]
+    of "genesis_state":
+      readFile(config.file.string).string.toBytes()
+    else:
+      stderr.write config.getQueryPath & " is not a valid path"
+      quit 1
+
+  let navigator = DynamicSszNavigator.init(bytes, BeaconState)
+
+  echo navigator.navigatePath(pathFragments[1 .. ^1]).toJson

tests/official/fixtures_utils.nim

@@ -57,9 +57,8 @@ template readFileBytes*(path: string): seq[byte] =
 
 proc sszDecodeEntireInput*(input: openarray[byte], Decoded: type): Decoded =
   var stream = unsafeMemoryInput(input)
-  var reader = init(SszReader, stream)
+  var reader = init(SszReader, stream, input.len)
   result = reader.readValue(Decoded)
 
   if stream.readable:
     raise newException(UnconsumedInput, "Remaining bytes in the input")
-