Dynamic SSZ navigator

This commit is contained in:
Zahary Karadjov 2019-11-01 17:14:34 +02:00 committed by zah
parent fb180a1313
commit a644839b79
3 changed files with 208 additions and 8 deletions

View File

@ -0,0 +1,126 @@
import
parseutils,
faststreams/output_stream, json_serialization/writer,
types, bytes_reader, navigator
type
ObjKind = enum
Record
Indexable
LeafValue
FieldInfo = ref object
name: string
fieldType: TypeInfo
navigator: proc (m: MemRange): MemRange {. gcsafe
noSideEffect
raises: [Defect,
MalformedSszError] }
TypeInfo = ref object
case kind: ObjKind
of Record:
fields: seq[FieldInfo]
of Indexable:
elemType: TypeInfo
navigator: proc (m: MemRange, idx: int): MemRange {. gcsafe
noSideEffect
raises: [Defect,
MalformedSszError] }
else:
discard
jsonPrinter: proc (m: MemRange,
outStream: OutputStreamVar,
pretty: bool) {.gcsafe, noSideEffect.}
DynamicSszNavigator* = object
m: MemRange
typ: TypeInfo
func jsonPrinterImpl[T](m: MemRange, outStream: OutputStreamVar, pretty: bool) =
var typedNavigator = sszMount(m, T)
var jsonWriter = init(JsonWriter, outStream, pretty)
# TODO: it should be possible to serialize the navigator object
# without dereferencing it (to avoid the intermediate value).
writeValue(jsonWriter, typedNavigator[])
func findField(fields: seq[FieldInfo], name: string): FieldInfo =
# TODO: Replace this with a binary search?
# Will it buy us anything when there are only few fields?
for field in fields:
if field.name == name:
return field
func indexableNavigatorImpl[T](m: MemRange, idx: int): MemRange =
var typedNavigator = sszMount(m, T)
getMemRange(typedNavigator[idx])
func fieldNavigatorImpl[RecordType; FieldType;
fieldName: static string](m: MemRange): MemRange {.raises: [Defect, MalformedSszError].} =
# TODO: Make sure this doesn't fail with a Defect when
# navigating to an inactive field in a case object.
var typedNavigator = sszMount(m, RecordType)
getMemRange navigateToField(typedNavigator, fieldName, FieldType)
func genTypeInfo(T: type): TypeInfo {.gcsafe.}
proc typeInfo*(T: type): TypeInfo =
let res {.global.} = genTypeInfo(T)
# TODO This will be safer if the RTTI object use only manually
# managed memory, but the `fields` sequence right now make
# things harder. We'll need to switch to a different seq type.
{.gcsafe, noSideEffect.}: res
func genTypeInfo(T: type): TypeInfo =
mixin enumAllSerializedFields
result = when T is object:
var fields: seq[FieldInfo]
enumAllSerializedFields(T):
fields.add FieldInfo(name: fieldName,
fieldType: typeInfo(FieldType),
navigator: fieldNavigatorImpl[T, FieldType, fieldName])
TypeInfo(kind: Record, fields: fields)
elif T is seq|array:
TypeInfo(kind: Indexable,
elemType: typeInfo(ElemType(T)),
navigator: indexableNavigatorImpl[T])
else:
TypeInfo(kind: LeafValue)
result.jsonPrinter = jsonPrinterImpl[T]
func `[]`*(n: DynamicSszNavigator, idx: int): DynamicSszNavigator =
doAssert n.typ.kind == Indexable
DynamicSszNavigator(m: n.typ.navigator(n.m, idx), typ: n.typ.elemType)
func navigate*(n: DynamicSszNavigator, path: string): DynamicSszNavigator {.
raises: [Defect, ValueError, MalformedSszError] .} =
case n.typ.kind
of Record:
let fieldInfo = n.typ.fields.findField(path)
if fieldInfo == nil:
raise newException(KeyError, "Unrecogned field name: " & path)
return DynamicSszNavigator(m: fieldInfo.navigator(n.m),
typ: fieldInfo.fieldType)
of Indexable:
var idx: int
let consumed = parseInt(path, idx)
if consumed == 0 or idx < 0:
raise newException(ValueError, "Indexing should be done with natural numbers")
return n[idx]
else:
doAssert false, "Navigation should be terminated once you reach a leaf value"
func init*(T: type DynamicSszNavigator, bytes: openarray[byte], typ: TypeInfo): T =
T(m: MemRange(startAddr: unsafeAddr bytes[0], length: bytes.len), typ: typ)
func writeJson*(n: DynamicSszNavigator, outStream: OutputStreamVar, pretty = true) =
n.typ.jsonPrinter(n.m, outStream, pretty)
func toJson*(n: DynamicSszNavigator, pretty = true): string =
var outStream = init OutputStream
writeJson(n, outStream, pretty)
outStream.getOutput(string)

View File

@ -3,9 +3,9 @@ import
./types, ./bytes_reader ./types, ./bytes_reader
type type
MemRange = object MemRange* = object
startAddr: ptr byte startAddr*: ptr byte
length: int length*: int
SszNavigator*[T] = object SszNavigator*[T] = object
m: MemRange m: MemRange
@ -14,6 +14,18 @@ func sszMount*(data: openarray[byte], T: type): SszNavigator[T] =
let startAddr = unsafeAddr data[0] let startAddr = unsafeAddr data[0]
SszNavigator[T](m: MemRange(startAddr: startAddr, length: data.len)) SszNavigator[T](m: MemRange(startAddr: startAddr, length: data.len))
template sszMount*(data: MemRange, T: type): SszNavigator[T] =
SszNavigator[T](m: data)
template getMemRange*(n: SszNavigator): MemRange =
# Please note that this accessor was created intentionally.
# We don't want to expose the `m` field, because the navigated
# type may have a field by that name. We wan't any dot field
# access to be redirected to the navigated type.
# For this reason, this template should always be used with
# the function call syntax `getMemRange(n)`.
n.m
template checkBounds(m: MemRange, offset: int) = template checkBounds(m: MemRange, offset: int) =
if offset > m.length: if offset > m.length:
raise newException(MalformedSszError, "Malformed SSZ") raise newException(MalformedSszError, "Malformed SSZ")
@ -21,9 +33,9 @@ template checkBounds(m: MemRange, offset: int) =
template toOpenArray(m: MemRange): auto = template toOpenArray(m: MemRange): auto =
makeOpenArray(m.startAddr, m.length) makeOpenArray(m.startAddr, m.length)
func navigateToField[T](n: SszNavigator[T], func navigateToField*[T](n: SszNavigator[T],
fieldName: static string, fieldName: static string,
FieldType: type): SszNavigator[FieldType] = FieldType: type): SszNavigator[FieldType] =
mixin toSszType mixin toSszType
type SszFieldType = type toSszType(default FieldType) type SszFieldType = type toSszType(default FieldType)
@ -55,6 +67,47 @@ template `.`*[T](n: SszNavigator[T], field: untyped): auto =
type FieldType = type(default(RecType).field) type FieldType = type(default(RecType).field)
navigateToField(n, astToStr(field), FieldType) navigateToField(n, astToStr(field), FieldType)
func indexVarSizeList(m: MemRange, idx: int): MemRange =
template readOffset(pos): int =
int fromSszBytes(uint32, makeOpenArray(shift(m.startAddr, pos), offsetSize))
let offsetPos = offsetSize * idx
checkBounds(m, offsetPos + offsetSize)
let firstOffset = readOffset 0
let listLen = firstOffset div offsetSize
if idx >= listLen:
# TODO: Use a RangeError here?
# This would require the user to check the `len` upfront
raise newException(MalformedSszError, "Indexing past the end")
let elemPos = readOffset offsetPos
checkBounds(m, elemPos)
let endPos = if idx < listLen - 1:
let nextOffsetPos = offsetPos + offsetSize
# TODO. Is there a way to remove this bounds check?
checkBounds(m, nextOffsetPos + offsetSize)
readOffset(offsetPos + nextOffsetPos)
else:
m.length
MemRange(startAddr: m.startAddr.shift(elemPos), length: endPos - elemPos)
template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] =
type R = T
mixin toSszType
type ElemType = type toSszType(default T)
when isFixedSize(ElemType):
const elemSize = fixedPortionSize(ElemType)
let elemPos = idx * elemSize
checkBounds(n.m, elemPos + elemSize)
SszNavigator[R](m: MemRange(startAddr: shift(n.m.startAddr, elemPos),
length: elemSize))
else:
SszNavigator[R](m: indexVarSizeList(n.m, idx))
func `[]`*[T](n: SszNavigator[T]): T = func `[]`*[T](n: SszNavigator[T]): T =
readSszValue(toOpenArray(n.m), T) readSszValue(toOpenArray(n.m), T)

View File

@ -9,7 +9,7 @@ import
unittest, sequtils, options, unittest, sequtils, options,
stint, nimcrypto, eth/common, blscurve, serialization/testing/generic_suite, stint, nimcrypto, eth/common, blscurve, serialization/testing/generic_suite,
../beacon_chain/spec/[datatypes, digest], ../beacon_chain/spec/[datatypes, digest],
../beacon_chain/ssz, ../beacon_chain/ssz/navigator ../beacon_chain/ssz, ../beacon_chain/ssz/[navigator, dynamic_navigator]
type type
SomeEnum = enum SomeEnum = enum
@ -71,7 +71,7 @@ type
proc toDigest[N: static int](x: array[N, byte]): Eth2Digest = proc toDigest[N: static int](x: array[N, byte]): Eth2Digest =
result.data[0 .. N-1] = x result.data[0 .. N-1] = x
suite "SSZ Navigation": suite "SSZ navigator":
test "simple object fields": test "simple object fields":
var foo = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64))) var foo = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64)))
let encoded = SSZ.encode(foo) let encoded = SSZ.encode(foo)
@ -96,3 +96,24 @@ suite "SSZ Navigation":
let leaves2 = sszList(@[a, b, c], int64(1 shl 10)) let leaves2 = sszList(@[a, b, c], int64(1 shl 10))
let root2 = hash_tree_root(leaves2) let root2 = hash_tree_root(leaves2)
check $root2 == "9FB7D518368DC14E8CC588FB3FD2749BEEF9F493FEF70AE34AF5721543C67173" check $root2 == "9FB7D518368DC14E8CC588FB3FD2749BEEF9F493FEF70AE34AF5721543C67173"
suite "SSZ dynamic navigator":
test "navigating fields":
var fooOrig = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64)))
let fooEncoded = SSZ.encode(fooOrig)
var navFoo = DynamicSszNavigator.init(fooEncoded, typeInfo(Foo))
var navBar = navFoo.navigate("bar")
check navBar.toJson(pretty = false) == """{"b":"bar","baz":{"i":10}}"""
var navB = navBar.navigate("b")
check navB.toJson == "\"bar\""
var navBaz = navBar.navigate("baz")
var navI = navBaz.navigate("i")
check navI.toJson == "10"
expect KeyError:
discard navBar.navigate("biz")