Dynamic SSZ navigator
This commit is contained in:
parent
fb180a1313
commit
a644839b79
|
@ -0,0 +1,126 @@
|
||||||
|
import
|
||||||
|
parseutils,
|
||||||
|
faststreams/output_stream, json_serialization/writer,
|
||||||
|
types, bytes_reader, navigator
|
||||||
|
|
||||||
|
type
|
||||||
|
ObjKind = enum
|
||||||
|
Record
|
||||||
|
Indexable
|
||||||
|
LeafValue
|
||||||
|
|
||||||
|
FieldInfo = ref object
|
||||||
|
name: string
|
||||||
|
fieldType: TypeInfo
|
||||||
|
navigator: proc (m: MemRange): MemRange {. gcsafe
|
||||||
|
noSideEffect
|
||||||
|
raises: [Defect,
|
||||||
|
MalformedSszError] }
|
||||||
|
TypeInfo = ref object
|
||||||
|
case kind: ObjKind
|
||||||
|
of Record:
|
||||||
|
fields: seq[FieldInfo]
|
||||||
|
of Indexable:
|
||||||
|
elemType: TypeInfo
|
||||||
|
navigator: proc (m: MemRange, idx: int): MemRange {. gcsafe
|
||||||
|
noSideEffect
|
||||||
|
raises: [Defect,
|
||||||
|
MalformedSszError] }
|
||||||
|
else:
|
||||||
|
discard
|
||||||
|
|
||||||
|
jsonPrinter: proc (m: MemRange,
|
||||||
|
outStream: OutputStreamVar,
|
||||||
|
pretty: bool) {.gcsafe, noSideEffect.}
|
||||||
|
|
||||||
|
DynamicSszNavigator* = object
|
||||||
|
m: MemRange
|
||||||
|
typ: TypeInfo
|
||||||
|
|
||||||
|
func jsonPrinterImpl[T](m: MemRange, outStream: OutputStreamVar, pretty: bool) =
|
||||||
|
var typedNavigator = sszMount(m, T)
|
||||||
|
var jsonWriter = init(JsonWriter, outStream, pretty)
|
||||||
|
# TODO: it should be possible to serialize the navigator object
|
||||||
|
# without dereferencing it (to avoid the intermediate value).
|
||||||
|
writeValue(jsonWriter, typedNavigator[])
|
||||||
|
|
||||||
|
func findField(fields: seq[FieldInfo], name: string): FieldInfo =
|
||||||
|
# TODO: Replace this with a binary search?
|
||||||
|
# Will it buy us anything when there are only few fields?
|
||||||
|
for field in fields:
|
||||||
|
if field.name == name:
|
||||||
|
return field
|
||||||
|
|
||||||
|
func indexableNavigatorImpl[T](m: MemRange, idx: int): MemRange =
|
||||||
|
var typedNavigator = sszMount(m, T)
|
||||||
|
getMemRange(typedNavigator[idx])
|
||||||
|
|
||||||
|
func fieldNavigatorImpl[RecordType; FieldType;
|
||||||
|
fieldName: static string](m: MemRange): MemRange {.raises: [Defect, MalformedSszError].} =
|
||||||
|
# TODO: Make sure this doesn't fail with a Defect when
|
||||||
|
# navigating to an inactive field in a case object.
|
||||||
|
var typedNavigator = sszMount(m, RecordType)
|
||||||
|
getMemRange navigateToField(typedNavigator, fieldName, FieldType)
|
||||||
|
|
||||||
|
func genTypeInfo(T: type): TypeInfo {.gcsafe.}
|
||||||
|
|
||||||
|
proc typeInfo*(T: type): TypeInfo =
|
||||||
|
let res {.global.} = genTypeInfo(T)
|
||||||
|
|
||||||
|
# TODO This will be safer if the RTTI object use only manually
|
||||||
|
# managed memory, but the `fields` sequence right now make
|
||||||
|
# things harder. We'll need to switch to a different seq type.
|
||||||
|
{.gcsafe, noSideEffect.}: res
|
||||||
|
|
||||||
|
func genTypeInfo(T: type): TypeInfo =
|
||||||
|
mixin enumAllSerializedFields
|
||||||
|
|
||||||
|
result = when T is object:
|
||||||
|
var fields: seq[FieldInfo]
|
||||||
|
enumAllSerializedFields(T):
|
||||||
|
fields.add FieldInfo(name: fieldName,
|
||||||
|
fieldType: typeInfo(FieldType),
|
||||||
|
navigator: fieldNavigatorImpl[T, FieldType, fieldName])
|
||||||
|
TypeInfo(kind: Record, fields: fields)
|
||||||
|
elif T is seq|array:
|
||||||
|
TypeInfo(kind: Indexable,
|
||||||
|
elemType: typeInfo(ElemType(T)),
|
||||||
|
navigator: indexableNavigatorImpl[T])
|
||||||
|
else:
|
||||||
|
TypeInfo(kind: LeafValue)
|
||||||
|
|
||||||
|
result.jsonPrinter = jsonPrinterImpl[T]
|
||||||
|
|
||||||
|
func `[]`*(n: DynamicSszNavigator, idx: int): DynamicSszNavigator =
|
||||||
|
doAssert n.typ.kind == Indexable
|
||||||
|
DynamicSszNavigator(m: n.typ.navigator(n.m, idx), typ: n.typ.elemType)
|
||||||
|
|
||||||
|
func navigate*(n: DynamicSszNavigator, path: string): DynamicSszNavigator {.
|
||||||
|
raises: [Defect, ValueError, MalformedSszError] .} =
|
||||||
|
case n.typ.kind
|
||||||
|
of Record:
|
||||||
|
let fieldInfo = n.typ.fields.findField(path)
|
||||||
|
if fieldInfo == nil:
|
||||||
|
raise newException(KeyError, "Unrecogned field name: " & path)
|
||||||
|
return DynamicSszNavigator(m: fieldInfo.navigator(n.m),
|
||||||
|
typ: fieldInfo.fieldType)
|
||||||
|
of Indexable:
|
||||||
|
var idx: int
|
||||||
|
let consumed = parseInt(path, idx)
|
||||||
|
if consumed == 0 or idx < 0:
|
||||||
|
raise newException(ValueError, "Indexing should be done with natural numbers")
|
||||||
|
return n[idx]
|
||||||
|
else:
|
||||||
|
doAssert false, "Navigation should be terminated once you reach a leaf value"
|
||||||
|
|
||||||
|
func init*(T: type DynamicSszNavigator, bytes: openarray[byte], typ: TypeInfo): T =
|
||||||
|
T(m: MemRange(startAddr: unsafeAddr bytes[0], length: bytes.len), typ: typ)
|
||||||
|
|
||||||
|
func writeJson*(n: DynamicSszNavigator, outStream: OutputStreamVar, pretty = true) =
|
||||||
|
n.typ.jsonPrinter(n.m, outStream, pretty)
|
||||||
|
|
||||||
|
func toJson*(n: DynamicSszNavigator, pretty = true): string =
|
||||||
|
var outStream = init OutputStream
|
||||||
|
writeJson(n, outStream, pretty)
|
||||||
|
outStream.getOutput(string)
|
||||||
|
|
|
@ -3,9 +3,9 @@ import
|
||||||
./types, ./bytes_reader
|
./types, ./bytes_reader
|
||||||
|
|
||||||
type
|
type
|
||||||
MemRange = object
|
MemRange* = object
|
||||||
startAddr: ptr byte
|
startAddr*: ptr byte
|
||||||
length: int
|
length*: int
|
||||||
|
|
||||||
SszNavigator*[T] = object
|
SszNavigator*[T] = object
|
||||||
m: MemRange
|
m: MemRange
|
||||||
|
@ -14,6 +14,18 @@ func sszMount*(data: openarray[byte], T: type): SszNavigator[T] =
|
||||||
let startAddr = unsafeAddr data[0]
|
let startAddr = unsafeAddr data[0]
|
||||||
SszNavigator[T](m: MemRange(startAddr: startAddr, length: data.len))
|
SszNavigator[T](m: MemRange(startAddr: startAddr, length: data.len))
|
||||||
|
|
||||||
|
template sszMount*(data: MemRange, T: type): SszNavigator[T] =
|
||||||
|
SszNavigator[T](m: data)
|
||||||
|
|
||||||
|
template getMemRange*(n: SszNavigator): MemRange =
|
||||||
|
# Please note that this accessor was created intentionally.
|
||||||
|
# We don't want to expose the `m` field, because the navigated
|
||||||
|
# type may have a field by that name. We wan't any dot field
|
||||||
|
# access to be redirected to the navigated type.
|
||||||
|
# For this reason, this template should always be used with
|
||||||
|
# the function call syntax `getMemRange(n)`.
|
||||||
|
n.m
|
||||||
|
|
||||||
template checkBounds(m: MemRange, offset: int) =
|
template checkBounds(m: MemRange, offset: int) =
|
||||||
if offset > m.length:
|
if offset > m.length:
|
||||||
raise newException(MalformedSszError, "Malformed SSZ")
|
raise newException(MalformedSszError, "Malformed SSZ")
|
||||||
|
@ -21,9 +33,9 @@ template checkBounds(m: MemRange, offset: int) =
|
||||||
template toOpenArray(m: MemRange): auto =
|
template toOpenArray(m: MemRange): auto =
|
||||||
makeOpenArray(m.startAddr, m.length)
|
makeOpenArray(m.startAddr, m.length)
|
||||||
|
|
||||||
func navigateToField[T](n: SszNavigator[T],
|
func navigateToField*[T](n: SszNavigator[T],
|
||||||
fieldName: static string,
|
fieldName: static string,
|
||||||
FieldType: type): SszNavigator[FieldType] =
|
FieldType: type): SszNavigator[FieldType] =
|
||||||
mixin toSszType
|
mixin toSszType
|
||||||
type SszFieldType = type toSszType(default FieldType)
|
type SszFieldType = type toSszType(default FieldType)
|
||||||
|
|
||||||
|
@ -55,6 +67,47 @@ template `.`*[T](n: SszNavigator[T], field: untyped): auto =
|
||||||
type FieldType = type(default(RecType).field)
|
type FieldType = type(default(RecType).field)
|
||||||
navigateToField(n, astToStr(field), FieldType)
|
navigateToField(n, astToStr(field), FieldType)
|
||||||
|
|
||||||
|
func indexVarSizeList(m: MemRange, idx: int): MemRange =
|
||||||
|
template readOffset(pos): int =
|
||||||
|
int fromSszBytes(uint32, makeOpenArray(shift(m.startAddr, pos), offsetSize))
|
||||||
|
|
||||||
|
let offsetPos = offsetSize * idx
|
||||||
|
checkBounds(m, offsetPos + offsetSize)
|
||||||
|
|
||||||
|
let firstOffset = readOffset 0
|
||||||
|
let listLen = firstOffset div offsetSize
|
||||||
|
|
||||||
|
if idx >= listLen:
|
||||||
|
# TODO: Use a RangeError here?
|
||||||
|
# This would require the user to check the `len` upfront
|
||||||
|
raise newException(MalformedSszError, "Indexing past the end")
|
||||||
|
|
||||||
|
let elemPos = readOffset offsetPos
|
||||||
|
checkBounds(m, elemPos)
|
||||||
|
|
||||||
|
let endPos = if idx < listLen - 1:
|
||||||
|
let nextOffsetPos = offsetPos + offsetSize
|
||||||
|
# TODO. Is there a way to remove this bounds check?
|
||||||
|
checkBounds(m, nextOffsetPos + offsetSize)
|
||||||
|
readOffset(offsetPos + nextOffsetPos)
|
||||||
|
else:
|
||||||
|
m.length
|
||||||
|
|
||||||
|
MemRange(startAddr: m.startAddr.shift(elemPos), length: endPos - elemPos)
|
||||||
|
|
||||||
|
template `[]`*[T](n: SszNavigator[seq[T]], idx: int): SszNavigator[T] =
|
||||||
|
type R = T
|
||||||
|
mixin toSszType
|
||||||
|
type ElemType = type toSszType(default T)
|
||||||
|
when isFixedSize(ElemType):
|
||||||
|
const elemSize = fixedPortionSize(ElemType)
|
||||||
|
let elemPos = idx * elemSize
|
||||||
|
checkBounds(n.m, elemPos + elemSize)
|
||||||
|
SszNavigator[R](m: MemRange(startAddr: shift(n.m.startAddr, elemPos),
|
||||||
|
length: elemSize))
|
||||||
|
else:
|
||||||
|
SszNavigator[R](m: indexVarSizeList(n.m, idx))
|
||||||
|
|
||||||
func `[]`*[T](n: SszNavigator[T]): T =
|
func `[]`*[T](n: SszNavigator[T]): T =
|
||||||
readSszValue(toOpenArray(n.m), T)
|
readSszValue(toOpenArray(n.m), T)
|
||||||
|
|
||||||
|
|
|
@ -9,7 +9,7 @@ import
|
||||||
unittest, sequtils, options,
|
unittest, sequtils, options,
|
||||||
stint, nimcrypto, eth/common, blscurve, serialization/testing/generic_suite,
|
stint, nimcrypto, eth/common, blscurve, serialization/testing/generic_suite,
|
||||||
../beacon_chain/spec/[datatypes, digest],
|
../beacon_chain/spec/[datatypes, digest],
|
||||||
../beacon_chain/ssz, ../beacon_chain/ssz/navigator
|
../beacon_chain/ssz, ../beacon_chain/ssz/[navigator, dynamic_navigator]
|
||||||
|
|
||||||
type
|
type
|
||||||
SomeEnum = enum
|
SomeEnum = enum
|
||||||
|
@ -71,7 +71,7 @@ type
|
||||||
proc toDigest[N: static int](x: array[N, byte]): Eth2Digest =
|
proc toDigest[N: static int](x: array[N, byte]): Eth2Digest =
|
||||||
result.data[0 .. N-1] = x
|
result.data[0 .. N-1] = x
|
||||||
|
|
||||||
suite "SSZ Navigation":
|
suite "SSZ navigator":
|
||||||
test "simple object fields":
|
test "simple object fields":
|
||||||
var foo = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64)))
|
var foo = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64)))
|
||||||
let encoded = SSZ.encode(foo)
|
let encoded = SSZ.encode(foo)
|
||||||
|
@ -96,3 +96,24 @@ suite "SSZ Navigation":
|
||||||
let leaves2 = sszList(@[a, b, c], int64(1 shl 10))
|
let leaves2 = sszList(@[a, b, c], int64(1 shl 10))
|
||||||
let root2 = hash_tree_root(leaves2)
|
let root2 = hash_tree_root(leaves2)
|
||||||
check $root2 == "9FB7D518368DC14E8CC588FB3FD2749BEEF9F493FEF70AE34AF5721543C67173"
|
check $root2 == "9FB7D518368DC14E8CC588FB3FD2749BEEF9F493FEF70AE34AF5721543C67173"
|
||||||
|
|
||||||
|
suite "SSZ dynamic navigator":
|
||||||
|
test "navigating fields":
|
||||||
|
var fooOrig = Foo(bar: Bar(b: "bar", baz: Baz(i: 10'u64)))
|
||||||
|
let fooEncoded = SSZ.encode(fooOrig)
|
||||||
|
|
||||||
|
var navFoo = DynamicSszNavigator.init(fooEncoded, typeInfo(Foo))
|
||||||
|
|
||||||
|
var navBar = navFoo.navigate("bar")
|
||||||
|
check navBar.toJson(pretty = false) == """{"b":"bar","baz":{"i":10}}"""
|
||||||
|
|
||||||
|
var navB = navBar.navigate("b")
|
||||||
|
check navB.toJson == "\"bar\""
|
||||||
|
|
||||||
|
var navBaz = navBar.navigate("baz")
|
||||||
|
var navI = navBaz.navigate("i")
|
||||||
|
check navI.toJson == "10"
|
||||||
|
|
||||||
|
expect KeyError:
|
||||||
|
discard navBar.navigate("biz")
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue