Dynamic SSZ navigator

beacon_chain/ssz/dynamic_navigator.nim

@@ -0,0 +1,126 @@
+import
+  parseutils,
+  faststreams/output_stream, json_serialization/writer,
+  types, bytes_reader, navigator
+
+type
+  ObjKind = enum
+    Record
+    Indexable
+    LeafValue
+
+  FieldInfo = ref object
+    name: string
+    fieldType: TypeInfo
+    navigator: proc (m: MemRange): MemRange {. gcsafe
+                                               noSideEffect
+                                               raises: [Defect,
+                                                        MalformedSszError] }
+  TypeInfo = ref object
+    case kind: ObjKind
+    of Record:
+      fields: seq[FieldInfo]
+    of Indexable:
+      elemType: TypeInfo
+      navigator: proc (m: MemRange, idx: int): MemRange {. gcsafe
+                                                           noSideEffect
+                                                           raises: [Defect,
+                                                                    MalformedSszError] }
+    else:
+      discard
+
+    jsonPrinter: proc (m: MemRange,
+                       outStream: OutputStreamVar,
+                       pretty: bool) {.gcsafe, noSideEffect.}
+
+  DynamicSszNavigator* = object
+    m: MemRange
+    typ: TypeInfo
+
+func jsonPrinterImpl[T](m: MemRange, outStream: OutputStreamVar, pretty: bool) =
+  var typedNavigator = sszMount(m, T)
+  var jsonWriter = init(JsonWriter, outStream, pretty)
+  # TODO: it should be possible to serialize the navigator object
+  #       without dereferencing it (to avoid the intermediate value).
+  writeValue(jsonWriter, typedNavigator[])
+
+func findField(fields: seq[FieldInfo], name: string): FieldInfo =
+  # TODO: Replace this with a binary search?
+  #       Will it buy us anything when there are only few fields?
+  for field in fields:
+    if field.name == name:
+      return field
+
+func indexableNavigatorImpl[T](m: MemRange, idx: int): MemRange =
+  var typedNavigator = sszMount(m, T)
+  getMemRange(typedNavigator[idx])
+
+func fieldNavigatorImpl[RecordType; FieldType;
+                        fieldName: static string](m: MemRange): MemRange {.raises: [Defect, MalformedSszError].} =
+  # TODO: Make sure this doesn't fail with a Defect when
+  #       navigating to an inactive field in a case object.
+  var typedNavigator = sszMount(m, RecordType)
+  getMemRange navigateToField(typedNavigator, fieldName, FieldType)
+
+func genTypeInfo(T: type): TypeInfo {.gcsafe.}
+
+proc typeInfo*(T: type): TypeInfo =
+  let res {.global.} = genTypeInfo(T)
+
+  # TODO This will be safer if the RTTI object use only manually
+  #      managed memory, but the `fields` sequence right now make
+  #      things harder. We'll need to switch to a different seq type.
+  {.gcsafe, noSideEffect.}: res
+
+func genTypeInfo(T: type): TypeInfo =
+  mixin enumAllSerializedFields
+
+  result = when T is object:
+    var fields: seq[FieldInfo]
+    enumAllSerializedFields(T):
+      fields.add FieldInfo(name: fieldName,
+                           fieldType: typeInfo(FieldType),
+                           navigator: fieldNavigatorImpl[T, FieldType, fieldName])
+    TypeInfo(kind: Record, fields: fields)
+  elif T is seq|array:
+    TypeInfo(kind: Indexable,
+             elemType: typeInfo(ElemType(T)),
+             navigator: indexableNavigatorImpl[T])
+  else:
+    TypeInfo(kind: LeafValue)
+
+  result.jsonPrinter = jsonPrinterImpl[T]
+
+func `[]`*(n: DynamicSszNavigator, idx: int): DynamicSszNavigator =
+  doAssert n.typ.kind == Indexable
+  DynamicSszNavigator(m: n.typ.navigator(n.m, idx), typ: n.typ.elemType)
+
+func navigate*(n: DynamicSszNavigator, path: string): DynamicSszNavigator {.
+               raises: [Defect, ValueError, MalformedSszError] .} =
+  case n.typ.kind
+  of Record:
+    let fieldInfo = n.typ.fields.findField(path)
+    if fieldInfo == nil:
+      raise newException(KeyError, "Unrecogned field name: " & path)
+    return DynamicSszNavigator(m: fieldInfo.navigator(n.m),
+                               typ: fieldInfo.fieldType)
+  of Indexable:
+    var idx: int
+    let consumed = parseInt(path, idx)
+    if consumed == 0 or idx < 0:
+      raise newException(ValueError, "Indexing should be done with natural numbers")
+    return n[idx]
+  else:
+    doAssert false, "Navigation should be terminated once you reach a leaf value"
+
+func init*(T: type DynamicSszNavigator, bytes: openarray[byte], typ: TypeInfo): T =
+  T(m: MemRange(startAddr: unsafeAddr bytes[0], length: bytes.len), typ: typ)
+
+func writeJson*(n: DynamicSszNavigator, outStream: OutputStreamVar, pretty = true) =
+  n.typ.jsonPrinter(n.m, outStream, pretty)
+
+func toJson*(n: DynamicSszNavigator, pretty = true): string =
+  var outStream = init OutputStream
+  writeJson(n, outStream, pretty)
+  outStream.getOutput(string)
+

beacon_chain/ssz/navigator.nim

@@ -3,9 +3,9 @@ import
   ./types, ./bytes_reader
 
 type
-  MemRange = object
-    startAddr: ptr byte
-    length: int
+  MemRange* = object
+    startAddr*: ptr byte
+    length*: int
 
   SszNavigator*[T] = object
     m: MemRange
@@ -14,6 +14,18 @@ func sszMount*(data: openarray[byte], T: type): SszNavigator[T] =
   let startAddr = unsafeAddr data[0]
   SszNavigator[T](m: MemRange(startAddr: startAddr, length: data.len))
 
+template sszMount*(data: MemRange, T: type): SszNavigator[T] =
+  SszNavigator[T](m: data)
+
+template getMemRange*(n: SszNavigator): MemRange =
+  # Please note that this accessor was created intentionally.
+  # We don't want to expose the `m` field, because the navigated
+  # type may have a field by that name. We wan't any dot field
+  # access to be redirected to the navigated type.
+  # For this reason, this template should always be used with
+  # the function call syntax `getMemRange(n)`.
+  n.m
+
 template checkBounds(m: MemRange, offset: int) =
   if offset > m.length:
     raise newException(MalformedSszError, "Malformed SSZ")
@@ -21,7 +33,7 @@ template checkBounds(m: MemRange, offset: int) =
 template toOpenArray(m: MemRange): auto =
   makeOpenArray(m.startAddr, m.length)
 
-func navigateToField[T](n: SszNavigator[T],
+func navigateToField*[T](n: SszNavigator[T],
                          fieldName: static string,
                          FieldType: type): SszNavigator[FieldType] =
   mixin toSszType
@@ -55,6 +67,47 @@ template `.`*[T](n: SszNavigator[T], field: untyped): auto =
   type FieldType = type(default(RecType).field)
   navigateToField(n, astToStr(field), FieldType)
 
+func indexVarSizeList(m: MemRange, idx: int): MemRange =
+  template readOffset(pos): int =
+    int fromSszBytes(uint32, makeOpenArray(shift(m.startAddr, pos), offsetSize))
+
+  let offsetPos = offsetSize * idx
+  checkBounds(m, offsetPos + offsetSize)
+
+  let firstOffset = readOffset 0
+  let listLen = firstOffset div offsetSize
+
+  if idx >= listLen:
+    # TODO: Use a RangeError here?
+    # This would require the user to check the `len` upfront
+    raise newException(MalformedSszError, "Indexing past the end")
+
+  let elemPos = readOffset offsetPos
+  checkBounds(m, elemPos)
+
+  let endPos = if idx < listLen - 1:
+    let nextOffsetPos = offsetPos + offsetSize
+    # TODO. Is there a way to remove this bounds check?
+    checkBounds(m, nextOffsetPos + offsetSize)
+    readOffset(offsetPos + nextOffsetPos)
+  else:
+    m.length
+
+  MemRange(startAddr: m.startAddr.shift(elemPos), length: endPos - elemPos)
+
+template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] =
+  type R = T
+  mixin toSszType
+  type ElemType = type toSszType(default T)
+  when isFixedSize(ElemType):
+    const elemSize = fixedPortionSize(ElemType)
+    let elemPos = idx * elemSize
+    checkBounds(n.m, elemPos + elemSize)
+    SszNavigator[R](m: MemRange(startAddr: shift(n.m.startAddr, elemPos),
+                                length: elemSize))
+  else:
+    SszNavigator[R](m: indexVarSizeList(n.m, idx))
+
 func `[]`*[T](n: SszNavigator[T]): T =
   readSszValue(toOpenArray(n.m), T)
 

tests/test_ssz.nim

@@ -9,7 +9,7 @@ import
   unittest, sequtils, options,
   stint, nimcrypto, eth/common, blscurve, serialization/testing/generic_suite,
   ../beacon_chain/spec/[datatypes, digest],
-  ../beacon_chain/ssz, ../beacon_chain/ssz/navigator
+  ../beacon_chain/ssz, ../beacon_chain/ssz/[navigator, dynamic_navigator]
 
 type
   SomeEnum = enum
@@ -71,7 +71,7 @@ type
 proc toDigest[N: static int](x: array[N, byte]): Eth2Digest =
   result.data[0 .. N-1] = x
 
-suite "SSZ Navigation":
+suite "SSZ navigator":
   test "simple object fields":
     var foo = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64)))
     let encoded = SSZ.encode(foo)
@@ -96,3 +96,24 @@ suite "SSZ Navigation":
     let leaves2 = sszList(@[a, b, c], int64(1 shl 10))
     let root2 = hash_tree_root(leaves2)
     check $root2 == "9FB7D518368DC14E8CC588FB3FD2749BEEF9F493FEF70AE34AF5721543C67173"
+
+suite "SSZ dynamic navigator":
+  test "navigating fields":
+    var fooOrig = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64)))
+    let fooEncoded = SSZ.encode(fooOrig)
+
+    var navFoo = DynamicSszNavigator.init(fooEncoded, typeInfo(Foo))
+
+    var navBar = navFoo.navigate("bar")
+    check navBar.toJson(pretty = false) == """{"b":"bar","baz":{"i":10}}"""
+
+    var navB = navBar.navigate("b")
+    check navB.toJson == "\"bar\""
+
+    var navBaz = navBar.navigate("baz")
+    var navI = navBaz.navigate("i")
+    check navI.toJson == "10"
+
+    expect KeyError:
+      discard navBar.navigate("biz")
+