nimbus-eth2/beacon_chain/ssz/dynamic_navigator.nim

165 lines
6.2 KiB
Nim
Raw Normal View History

# beacon_chain
# Copyright (c) 2018-2021 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [Defect].}
{.pragma: raisesssz, raises: [Defect, IOError, MalformedSszError, SszSizeMismatchError].}
2019-11-01 15:14:34 +00:00
import
std/[strutils, parseutils],
stew/objects, faststreams/outputs, json_serialization/writer,
../spec/datatypes/base,
disentangle eth2 types from the ssz library (#2785) * reorganize ssz dependencies This PR continues the work in https://github.com/status-im/nimbus-eth2/pull/2646, https://github.com/status-im/nimbus-eth2/pull/2779 as well as past issues with serialization and type, to disentangle SSZ from eth2 and at the same time simplify imports and exports with a structured approach. The principal idea here is that when a library wants to introduce SSZ support, they do so via 3 files: * `ssz_codecs` which imports and reexports `codecs` - this covers the basic byte conversions and ensures no overloads get lost * `xxx_merkleization` imports and exports `merkleization` to specialize and get access to `hash_tree_root` and friends * `xxx_ssz_serialization` imports and exports `ssz_serialization` to specialize ssz for a specific library Those that need to interact with SSZ always import the `xxx_` versions of the modules and never `ssz` itself so as to keep imports simple and safe. This is similar to how the REST / JSON-RPC serializers are structured in that someone wanting to serialize spec types to REST-JSON will import `eth2_rest_serialization` and nothing else. * split up ssz into a core library that is independendent of eth2 types * rename `bytes_reader` to `codec` to highlight that it contains coding and decoding of bytes and native ssz types * remove tricky List init overload that causes compile issues * get rid of top-level ssz import * reenable merkleization tests * move some "standard" json serializers to spec * remove `ValidatorIndex` serialization for now * remove test_ssz_merkleization * add tests for over/underlong byte sequences * fix broken seq[byte] test - seq[byte] is not an SSZ type There are a few things this PR doesn't solve: * like #2646 this PR is weak on how to handle root and other dontSerialize fields that "sometimes" should be computed - the same problem appears in REST / JSON-RPC etc * Fix a build problem on macOS * Another way to fix the macOS builds Co-authored-by: Zahary Karadjov <zahary@gmail.com>
2021-08-18 18:57:58 +00:00
./codec, ./types, ./navigator
2019-11-01 15:14:34 +00:00
export
disentangle eth2 types from the ssz library (#2785) * reorganize ssz dependencies This PR continues the work in https://github.com/status-im/nimbus-eth2/pull/2646, https://github.com/status-im/nimbus-eth2/pull/2779 as well as past issues with serialization and type, to disentangle SSZ from eth2 and at the same time simplify imports and exports with a structured approach. The principal idea here is that when a library wants to introduce SSZ support, they do so via 3 files: * `ssz_codecs` which imports and reexports `codecs` - this covers the basic byte conversions and ensures no overloads get lost * `xxx_merkleization` imports and exports `merkleization` to specialize and get access to `hash_tree_root` and friends * `xxx_ssz_serialization` imports and exports `ssz_serialization` to specialize ssz for a specific library Those that need to interact with SSZ always import the `xxx_` versions of the modules and never `ssz` itself so as to keep imports simple and safe. This is similar to how the REST / JSON-RPC serializers are structured in that someone wanting to serialize spec types to REST-JSON will import `eth2_rest_serialization` and nothing else. * split up ssz into a core library that is independendent of eth2 types * rename `bytes_reader` to `codec` to highlight that it contains coding and decoding of bytes and native ssz types * remove tricky List init overload that causes compile issues * get rid of top-level ssz import * reenable merkleization tests * move some "standard" json serializers to spec * remove `ValidatorIndex` serialization for now * remove test_ssz_merkleization * add tests for over/underlong byte sequences * fix broken seq[byte] test - seq[byte] is not an SSZ type There are a few things this PR doesn't solve: * like #2646 this PR is weak on how to handle root and other dontSerialize fields that "sometimes" should be computed - the same problem appears in REST / JSON-RPC etc * Fix a build problem on macOS * Another way to fix the macOS builds Co-authored-by: Zahary Karadjov <zahary@gmail.com>
2021-08-18 18:57:58 +00:00
codec, navigator, types
2019-11-01 15:14:34 +00:00
type
ObjKind = enum
Record
Indexable
LeafValue
FieldInfo = ref object
name: string
fieldType: TypeInfo
navigator: proc (m: MemRange): MemRange {. gcsafe
noSideEffect
raisesssz }
2019-11-01 15:14:34 +00:00
TypeInfo = ref object
case kind: ObjKind
of Record:
fields: seq[FieldInfo]
of Indexable:
elemType: TypeInfo
navigator: proc (m: MemRange, idx: int): MemRange {. gcsafe
noSideEffect
raisesssz }
2019-11-01 15:14:34 +00:00
else:
discard
jsonPrinter: proc (m: MemRange,
outStream: OutputStream,
pretty: bool) {.gcsafe, raisesssz.}
2019-11-01 15:14:34 +00:00
DynamicSszNavigator* = object
m: MemRange
typ: TypeInfo
proc jsonPrinterImpl[T](m: MemRange, outStream: OutputStream, pretty: bool) {.raisesssz.} =
2019-11-01 15:14:34 +00:00
var typedNavigator = sszMount(m, T)
var jsonWriter = Json.Writer.init(outStream, pretty)
2019-11-01 15:14:34 +00:00
# TODO: it should be possible to serialize the navigator object
# without dereferencing it (to avoid the intermediate value).
writeValue(jsonWriter, typedNavigator[])
func findField(fields: seq[FieldInfo], name: string): FieldInfo =
# TODO: Replace this with a binary search?
# Will it buy us anything when there are only few fields?
for field in fields:
if field.name == name:
return field
func indexableNavigatorImpl[T](m: MemRange, idx: int): MemRange {.raisesssz.} =
2019-11-01 15:14:34 +00:00
var typedNavigator = sszMount(m, T)
getMemRange(typedNavigator[idx])
func fieldNavigatorImpl[RecordType; FieldType;
fieldName: static string](m: MemRange): MemRange {.raisesssz.} =
2019-11-01 15:14:34 +00:00
# TODO: Make sure this doesn't fail with a Defect when
# navigating to an inactive field in a case object.
var typedNavigator = sszMount(m, RecordType)
getMemRange navigateToField(typedNavigator, fieldName, FieldType)
func genTypeInfo(T: type): TypeInfo {.gcsafe.}
proc typeInfo*(T: type): TypeInfo =
let res {.global.} = genTypeInfo(T)
# TODO This will be safer if the RTTI object use only manually
# managed memory, but the `fields` sequence right now make
# things harder. We'll need to switch to a different seq type.
{.gcsafe, noSideEffect.}: res
func genTypeInfo(T: type): TypeInfo =
mixin toSszType, enumAllSerializedFields
type SszType = type toSszType(declval T)
result = when type(SszType) isnot T:
TypeInfo(kind: LeafValue)
elif T is object:
2019-11-01 15:14:34 +00:00
var fields: seq[FieldInfo]
enumAllSerializedFields(T):
fields.add FieldInfo(name: fieldName,
fieldType: typeInfo(FieldType),
navigator: fieldNavigatorImpl[T, FieldType, fieldName])
TypeInfo(kind: Record, fields: fields)
elif T is seq|array:
TypeInfo(kind: Indexable,
elemType: typeInfo(ElemType(T)),
navigator: indexableNavigatorImpl[T])
else:
TypeInfo(kind: LeafValue)
result.jsonPrinter = jsonPrinterImpl[T]
func `[]`*(n: DynamicSszNavigator, idx: int): DynamicSszNavigator {.raisesssz.} =
2019-11-01 15:14:34 +00:00
doAssert n.typ.kind == Indexable
DynamicSszNavigator(m: n.typ.navigator(n.m, idx), typ: n.typ.elemType)
func navigate*(n: DynamicSszNavigator, path: string): DynamicSszNavigator {.
raises: [Defect, KeyError, IOError, MalformedSszError, SszSizeMismatchError, ValueError] .} =
2019-11-01 15:14:34 +00:00
case n.typ.kind
of Record:
let fieldInfo = n.typ.fields.findField(path)
if fieldInfo == nil:
raise newException(KeyError, "Unrecogned field name: " & path)
return DynamicSszNavigator(m: fieldInfo.navigator(n.m),
typ: fieldInfo.fieldType)
of Indexable:
var idx: int
let consumed = parseInt(path, idx)
if consumed == 0 or idx < 0:
raise newException(KeyError, "Indexing should be done with natural numbers")
2019-11-01 15:14:34 +00:00
return n[idx]
else:
doAssert false, "Navigation should be terminated once you reach a leaf value"
template navigatePathImpl(nav, iterabalePathFragments: untyped) =
result = nav
for pathFragment in iterabalePathFragments:
if pathFragment.len == 0:
continue
result = result.navigate(pathFragment)
if result.typ.kind == LeafValue:
return
func navigatePath*(n: DynamicSszNavigator, path: string): DynamicSszNavigator {.
raises: [Defect, IOError, ValueError, MalformedSszError, SszSizeMismatchError] .} =
navigatePathImpl n, split(path, '/')
2020-10-28 18:35:31 +00:00
func navigatePath*(n: DynamicSszNavigator, path: openArray[string]): DynamicSszNavigator {.
raises: [Defect, IOError, ValueError, MalformedSszError, SszSizeMismatchError] .} =
navigatePathImpl n, path
func init*(T: type DynamicSszNavigator,
2020-10-28 18:35:31 +00:00
bytes: openArray[byte], Navigated: type): T =
T(m: MemRange(startAddr: unsafeAddr bytes[0], length: bytes.len),
typ: typeInfo(Navigated))
2019-11-01 15:14:34 +00:00
proc writeJson*(n: DynamicSszNavigator, outStream: OutputStream, pretty = true) {.raisesssz.} =
2019-11-01 15:14:34 +00:00
n.typ.jsonPrinter(n.m, outStream, pretty)
func toJson*(n: DynamicSszNavigator, pretty = true): string {.raisesssz.} =
var outStream = memoryOutput()
{.noSideEffect.}:
# We are assuming that there are no side-effects here
# because we are using a `memoryOutput`. The computed
# side-effects are coming from the fact that the dynamic
# dispatch mechanisms used in faststreams may be reading
# from a file or a network device.
writeJson(n, outStream, pretty)
2019-11-01 15:14:34 +00:00
outStream.getOutput(string)