mirror of https://github.com/status-im/NimYAML.git
Use variant object types for heterogeneous data
* Made variant object types work (really this time) * Added `markAsImplicit` * Implemented implicit variant object types * Added documentation
This commit is contained in:
parent
e81da97a17
commit
48aeff20c0
|
@ -343,6 +343,72 @@ Loading Nim objects from JSON
|
|||
Processing a Sequence of Heterogeneous Items
|
||||
--------------------------------------------
|
||||
|
||||
… With variant objects
|
||||
......................
|
||||
|
||||
.. raw:: html
|
||||
<table class="quickstart-example"><thead><tr><th>code.nim</th>
|
||||
<th>in.yaml</th></tr></thead><tbody><tr><td>
|
||||
|
||||
.. code-block:: nim
|
||||
import yaml
|
||||
type
|
||||
Person = object
|
||||
name: string
|
||||
|
||||
ContainerKind = enum
|
||||
ckString, ckInt, ckBool, ckPerson, ckNone
|
||||
|
||||
Container = object
|
||||
case kind: ContainerKind
|
||||
of ckString:
|
||||
strVal: string
|
||||
of ckInt:
|
||||
intVal: int
|
||||
of ckBool:
|
||||
boolVal: bool
|
||||
of ckPerson:
|
||||
personVal: Person
|
||||
of ckNone:
|
||||
discard
|
||||
|
||||
setTagUri(Person, "!nim:demo:Person")
|
||||
|
||||
# tell NimYAML to use Container as implicit type.
|
||||
# only possible with variant object types where
|
||||
# each branch contains at most one object.
|
||||
markAsImplicit(Container)
|
||||
|
||||
var list: seq[Container]
|
||||
|
||||
var s = newFileStream("in.yaml")
|
||||
load(s, list)
|
||||
s.close()
|
||||
|
||||
assert(list[0].kind == ckString)
|
||||
assert(list[0].strVal == "this is a string")
|
||||
# and so on
|
||||
|
||||
.. raw:: html
|
||||
</td>
|
||||
<td>
|
||||
|
||||
.. code-block:: yaml
|
||||
%YAML 1.2
|
||||
---
|
||||
- this is a string
|
||||
- 42
|
||||
- false
|
||||
- !!str 23
|
||||
- !nim:demo:Person {name: Trillian}
|
||||
- !!null
|
||||
|
||||
.. raw:: html
|
||||
</td></tr></tbody></table>
|
||||
|
||||
… With the Sequential API
|
||||
.........................
|
||||
|
||||
.. raw:: html
|
||||
<table class="quickstart-example"><thead><tr><th>code.nim</th>
|
||||
<th>in.yaml</th></tr></thead><tbody><tr><td>
|
||||
|
|
|
@ -86,7 +86,8 @@ cannot be loaded to Nim collection types. For example, this sequence:
|
|||
- !!string foo
|
||||
|
||||
Cannot be loaded to a Nim ``seq``. For this reason, you cannot load YAML's
|
||||
native ``!!map`` and ``!!seq`` types directly into Nim types.
|
||||
native ``!!map`` and ``!!seq`` types directly into Nim types. However, you can
|
||||
use variant object types to process heterogeneous value lists, see below.
|
||||
|
||||
Nim ``seq`` types may be ``nil``. This is handled by serializing them to an
|
||||
empty scalar with the tag ``!nim:nil:seq``.
|
||||
|
@ -115,12 +116,81 @@ conditions must be met:
|
|||
- All fields of an object type must be accessible from the code position where
|
||||
you call NimYAML. If an object has non-public member fields, it can only be
|
||||
processed in the module where it is defined.
|
||||
- The object may not contain a ``case`` clause.
|
||||
- The object may not have a generic parameter
|
||||
|
||||
NimYAML will present enum types as YAML scalars, and tuple and object types as
|
||||
YAML maps. Some of the conditions above may be loosened in future releases.
|
||||
|
||||
Variant Object Types
|
||||
....................
|
||||
|
||||
A *variant object type* is an object type that contains one or more ``case``
|
||||
clauses. NimYAML currently supports variant object types. However, this feature
|
||||
is **highly experimental**. Only the currently accessible fields of a variant
|
||||
object type are dumped, and only those may be present when loading. The
|
||||
discriminator field(s) are treated like all other fields. The value of a
|
||||
discriminator field must occur before any value of a field that depends on it.
|
||||
This violates the YAML specification and therefore will be changed in the
|
||||
future.
|
||||
|
||||
While dumping variant object types directly is currently not production ready,
|
||||
you can use them for processing heterogeneous data sets. For example, if you
|
||||
have a YAML document which contains differently typed values in the same list
|
||||
like this:
|
||||
|
||||
.. code-block:: yaml
|
||||
%YAML 1.2
|
||||
---
|
||||
- 42
|
||||
- this is a string
|
||||
- !!null
|
||||
|
||||
You can define a variant object type that can hold all types that occur in this
|
||||
list in order to load it:
|
||||
|
||||
.. code-block:: nim
|
||||
import yaml
|
||||
|
||||
type
|
||||
ContainerKind = enum
|
||||
ckInt, ckString, ckNone
|
||||
Container = object
|
||||
case kind: ContainerKind
|
||||
of ckInt:
|
||||
intVal: int
|
||||
of ckString:
|
||||
strVal: string
|
||||
of ckNone:
|
||||
discard
|
||||
|
||||
markAsImplicit(Container)
|
||||
|
||||
var
|
||||
list: seq[Container]
|
||||
s = newFileStream("in.yaml")
|
||||
load(s, list)
|
||||
|
||||
``markAsImplicit`` tells NimYAML that you want to use the type ``Container``
|
||||
implicitly, i.e. its fields are not visible in YAML, and are set dependent on
|
||||
the value type that gets loaded into it. The type ``Container`` must fullfil the
|
||||
following requirements:
|
||||
|
||||
- It must contain exactly one ``case`` clause, and nothing else.
|
||||
- Each branch of the ``case`` clause must contain exactly one field, with one
|
||||
exception: There may be at most one branch that contains no field at all.
|
||||
- It must not be a derived object type (this is currently not enforced)
|
||||
|
||||
When loading the sequence, NimYAML writes the value into the first field that
|
||||
can hold the value's type. All complex values (i.e. non-scalar values) *must*
|
||||
have a tag in the YAML source, because NimYAML would otherwise be unable to
|
||||
determine their type. The type of scalar values will be guessed if no tag is
|
||||
available, but be aware that ``42`` can fit in both ``int8`` and ``int16``, so
|
||||
in the case you have fields for both types, you should annotate the value.
|
||||
|
||||
When dumping the sequence, NimYAML will always annotate a tag to each value it
|
||||
outputs. This is to avoid possible ambiguity when loading. If a branch without
|
||||
a field exists, it is represented as a ``!!null`` value.
|
||||
|
||||
Tags
|
||||
====
|
||||
|
||||
|
|
|
@ -387,14 +387,6 @@ proc representObject*[K, V](value: OrderedTable[K, V], ts: TagStyle,
|
|||
yield endMapEvent()
|
||||
yield endSeqEvent()
|
||||
|
||||
proc isVariant(t: typedesc): bool {.compileTime.} =
|
||||
let typeDesc = getType(t)
|
||||
if typeDesc.len > 1:
|
||||
for child in typeDesc[1].children:
|
||||
if child.kind == nnkRecCase:
|
||||
return true
|
||||
return false
|
||||
|
||||
proc yamlTag*(T: typedesc[object|enum]):
|
||||
TagId {.inline, raises: [].} =
|
||||
var uri = "!nim:custom:" & (typetraits.name(type(T)))
|
||||
|
@ -415,27 +407,60 @@ proc yamlTag*(T: typedesc[tuple]):
|
|||
try: serializationTagLibrary.tags[uri]
|
||||
except KeyError: serializationTagLibrary.registerUri(uri)
|
||||
|
||||
macro constructFieldValue(t: typedesc, stream: expr, context: expr,
|
||||
name: expr, o: expr): stmt =
|
||||
let tDesc = getType(getType(t)[1])
|
||||
result = newNimNode(nnkCaseStmt).add(name)
|
||||
for child in tDesc[1].children:
|
||||
if child.kind == nnkRecCase:
|
||||
let
|
||||
discriminant = newDotExpr(o, newIdentNode($child[0]))
|
||||
discType = newCall("type", discriminant)
|
||||
var disOb = newNimNode(nnkOfBranch).add(newStrLitNode($child[0]))
|
||||
disOb.add(newStmtList(
|
||||
newNimNode(nnkVarSection).add(
|
||||
newNimNode(nnkIdentDefs).add(
|
||||
newIdentNode("value"), discType, newEmptyNode())),
|
||||
newCall("constructChild", stream, context, newIdentNode("value")),
|
||||
newCall("reset", o),
|
||||
newAssignment(discriminant, newIdentNode("value"))))
|
||||
result.add(disOb)
|
||||
for bIndex in 1 .. len(child) - 1:
|
||||
let discTest = infix(discriminant, "==", child[bIndex][0])
|
||||
for item in child[bIndex][1].children:
|
||||
assert item.kind == nnkSym
|
||||
var ob = newNimNode(nnkOfBranch).add(newStrLitNode($item))
|
||||
let field = newDotExpr(o, newIdentNode($item))
|
||||
var ifStmt = newIfStmt((cond: discTest, body: newStmtList(
|
||||
newCall("constructChild", stream, context, field))))
|
||||
ifStmt.add(newNimNode(nnkElse).add(newNimNode(nnkDiscardStmt).add(
|
||||
newEmptyNode()))) # todo: raise exception here
|
||||
ob.add(newStmtList(ifStmt))
|
||||
result.add(ob)
|
||||
else:
|
||||
assert child.kind == nnkSym
|
||||
var ob = newNimNode(nnkOfBranch).add(newStrLitNode($child))
|
||||
let field = newDotExpr(o, newIdentNode($child))
|
||||
ob.add(newStmtList(newCall("constructChild", stream, context, field)))
|
||||
result.add(ob)
|
||||
|
||||
proc constructObject*[O: object|tuple](
|
||||
s: var YamlStream, c: ConstructionContext, result: var O)
|
||||
{.raises: [YamlConstructionError, YamlStreamError].} =
|
||||
## constructs a Nim object or tuple from a YAML mapping
|
||||
let e = s.next()
|
||||
if e.kind != yamlStartMap:
|
||||
raise newException(YamlConstructionError, "Expected map start, got " &
|
||||
$e.kind)
|
||||
raise newException(YamlConstructionError, "While constructing " &
|
||||
typetraits.name(O) & ": Expected map start, got " & $e.kind)
|
||||
while s.peek.kind != yamlEndMap:
|
||||
# todo: check for duplicates in input and raise appropriate exception
|
||||
# also todo: check for missing items and raise appropriate exception
|
||||
let e = s.next()
|
||||
if e.kind != yamlScalar:
|
||||
raise newException(YamlConstructionError,
|
||||
"Expected field name, got " & $e.kind)
|
||||
let name = e.scalarContent
|
||||
when compiles(implicitVariantObject(O)):
|
||||
discard
|
||||
else:
|
||||
for fname, value in fieldPairs(result):
|
||||
if fname == name:
|
||||
constructChild(s, c, value)
|
||||
break
|
||||
constructFieldValue(O, s, c, name, result)
|
||||
discard s.next()
|
||||
|
||||
proc representObject*[O: object|tuple](value: O, ts: TagStyle,
|
||||
|
@ -478,30 +503,110 @@ proc representObject*[O: enum](value: O, ts: TagStyle,
|
|||
|
||||
proc yamlTag*[O](T: typedesc[ref O]): TagId {.inline, raises: [].} = yamlTag(O)
|
||||
|
||||
macro constructImplicitVariantObject(s, c, r, possibleTagIds: expr,
|
||||
t: typedesc): stmt =
|
||||
let tDesc = getType(getType(t)[1])
|
||||
assert tDesc.kind == nnkObjectTy
|
||||
let recCase = tDesc[1][0]
|
||||
assert recCase.kind == nnkRecCase
|
||||
let
|
||||
discriminant = newDotExpr(r, newIdentNode($recCase[0]))
|
||||
discType = newCall("type", discriminant)
|
||||
var ifStmt = newNimNode(nnkIfStmt)
|
||||
for i in 1 .. recCase.len - 1:
|
||||
assert recCase[i].kind == nnkOfBranch
|
||||
var branch = newNimNode(nnkElifBranch)
|
||||
var branchContent = newStmtList(newAssignment(discriminant, recCase[i][0]))
|
||||
case recCase[i][1].len
|
||||
of 0:
|
||||
branch.add(infix(newIdentNode("yTagNull"), "in", possibleTagIds))
|
||||
branchContent.add(newNimNode(nnkDiscardStmt).add(newCall("next", s)))
|
||||
of 1:
|
||||
let field = newDotExpr(r, newIdentNode($recCase[i][1][0]))
|
||||
branch.add(infix(
|
||||
newCall("yamlTag", newCall("type", field)), "in", possibleTagIds))
|
||||
branchContent.add(newCall("constructChild", s, c, field))
|
||||
else: assert false
|
||||
branch.add(branchContent)
|
||||
ifStmt.add(branch)
|
||||
let raiseStmt = newNimNode(nnkRaiseStmt).add(
|
||||
newCall("newException", newIdentNode("YamlConstructionError"),
|
||||
infix(newStrLitNode("This value type does not map to any field in " &
|
||||
typetraits.name(t) & ": "), "&",
|
||||
newCall("uri", newIdentNode("serializationTagLibrary"),
|
||||
newNimNode(nnkBracketExpr).add(possibleTagIds, newIntLitNode(0)))
|
||||
)
|
||||
))
|
||||
ifStmt.add(newNimNode(nnkElse).add(newNimNode(nnkTryStmt).add(
|
||||
newStmtList(raiseStmt), newNimNode(nnkExceptBranch).add(
|
||||
newIdentNode("KeyError"), newStmtList(newCall("assert", newLit(false)))
|
||||
))))
|
||||
result = newStmtList(newCall("reset", r), ifStmt)
|
||||
|
||||
proc constructChild*[T](s: var YamlStream, c: ConstructionContext,
|
||||
result: var T) =
|
||||
let item = s.peek()
|
||||
case item.kind
|
||||
of yamlScalar:
|
||||
if item.scalarTag notin [yTagQuestionMark, yTagExclamationMark, yamlTag(T)]:
|
||||
raise newException(YamlConstructionError, "Wrong tag for " &
|
||||
typetraits.name(T))
|
||||
elif item.scalarAnchor != yAnchorNone:
|
||||
raise newException(YamlConstructionError, "Anchor on non-ref type")
|
||||
of yamlStartMap:
|
||||
if item.mapTag notin [yTagQuestionMark, yamlTag(T)]:
|
||||
raise newException(YamlConstructionError, "Wrong tag for " &
|
||||
typetraits.name(T))
|
||||
elif item.mapAnchor != yAnchorNone:
|
||||
raise newException(YamlConstructionError, "Anchor on non-ref type")
|
||||
of yamlStartSeq:
|
||||
if item.seqTag notin [yTagQuestionMark, yamlTag(T)]:
|
||||
raise newException(YamlConstructionError, "Wrong tag for " &
|
||||
typetraits.name(T))
|
||||
elif item.seqAnchor != yAnchorNone:
|
||||
raise newException(YamlConstructionError, "Anchor on non-ref type")
|
||||
else: assert false
|
||||
constructObject(s, c, result)
|
||||
when compiles(implicitVariantObject(result)):
|
||||
var possibleTagIds = newSeq[TagId]()
|
||||
case item.kind
|
||||
of yamlScalar:
|
||||
case item.scalarTag
|
||||
of yTagQuestionMark:
|
||||
case guessType(item.scalarContent)
|
||||
of yTypeInteger:
|
||||
possibleTagIds.add([yamlTag(int), yamlTag(int8), yamlTag(int16),
|
||||
yamlTag(int32), yamlTag(int64)])
|
||||
if item.scalarContent[0] != '-':
|
||||
possibleTagIds.add([yamlTag(uint), yamlTag(uint8), yamlTag(uint16),
|
||||
yamlTag(uint32), yamlTag(uint64)])
|
||||
of yTypeFloat, yTypeFloatInf, yTypeFloatNaN:
|
||||
possibleTagIds.add([yamlTag(float), yamlTag(float32),
|
||||
yamlTag(float64)])
|
||||
of yTypeBoolTrue, yTypeBoolFalse:
|
||||
possibleTagIds.add(yamlTag(bool))
|
||||
of yTypeNull:
|
||||
raise newException(YamlConstructionError, "not implemented!")
|
||||
of yTypeUnknown:
|
||||
possibleTagIds.add(yamlTag(string))
|
||||
of yTagExclamationMark:
|
||||
possibleTagIds.add(yamlTag(string))
|
||||
else:
|
||||
possibleTagIds.add(item.scalarTag)
|
||||
of yamlStartMap:
|
||||
if item.mapTag in [yTagQuestionMark, yTagExclamationMark]:
|
||||
raise newException(YamlConstructionError,
|
||||
"Complex value of implicit variant object type must have a tag.")
|
||||
possibleTagIds.add(item.mapTag)
|
||||
of yamlStartSeq:
|
||||
if item.seqTag in [yTagQuestionMark, yTagExclamationMark]:
|
||||
raise newException(YamlConstructionError,
|
||||
"Complex value of implicit variant object type must have a tag.")
|
||||
possibleTagIds.add(item.seqTag)
|
||||
else: assert false
|
||||
constructImplicitVariantObject(s, c, result, possibleTagIds, T)
|
||||
else:
|
||||
case item.kind
|
||||
of yamlScalar:
|
||||
if item.scalarTag notin [yTagQuestionMark, yTagExclamationMark,
|
||||
yamlTag(T)]:
|
||||
raise newException(YamlConstructionError, "Wrong tag for " &
|
||||
typetraits.name(T))
|
||||
elif item.scalarAnchor != yAnchorNone:
|
||||
raise newException(YamlConstructionError, "Anchor on non-ref type")
|
||||
of yamlStartMap:
|
||||
if item.mapTag notin [yTagQuestionMark, yamlTag(T)]:
|
||||
raise newException(YamlConstructionError, "Wrong tag for " &
|
||||
typetraits.name(T))
|
||||
elif item.mapAnchor != yAnchorNone:
|
||||
raise newException(YamlConstructionError, "Anchor on non-ref type")
|
||||
of yamlStartSeq:
|
||||
if item.seqTag notin [yTagQuestionMark, yamlTag(T)]:
|
||||
raise newException(YamlConstructionError, "Wrong tag for " &
|
||||
typetraits.name(T))
|
||||
elif item.seqAnchor != yAnchorNone:
|
||||
raise newException(YamlConstructionError, "Anchor on non-ref type")
|
||||
else: assert false
|
||||
constructObject(s, c, result)
|
||||
|
||||
proc constructChild*(s: var YamlStream, c: ConstructionContext,
|
||||
result: var string) =
|
||||
|
@ -636,8 +741,20 @@ proc representChild*[O](value: ref O, ts: TagStyle, c: SerializationContext):
|
|||
|
||||
proc representChild*[O](value: O, ts: TagStyle,
|
||||
c: SerializationContext): RawYamlStream =
|
||||
result = representObject(value, ts, c, if ts == tsNone:
|
||||
yTagQuestionMark else: yamlTag(O))
|
||||
when compiles(implicitVariantObject(value)):
|
||||
# todo: this would probably be nicer if constructed with a macro
|
||||
var count = 0
|
||||
for name, field in fieldPairs(value):
|
||||
if count > 0:
|
||||
result =
|
||||
representChild(field, if ts == tsAll: tsAll else: tsRootOnly, c)
|
||||
inc(count)
|
||||
if count == 1:
|
||||
result = iterator(): YamlStreamEvent =
|
||||
yield scalarEvent("~", yTagNull)
|
||||
else:
|
||||
result = representObject(value, ts, c, if ts == tsNone:
|
||||
yTagQuestionMark else: yamlTag(O))
|
||||
|
||||
proc construct*[T](s: var YamlStream, target: var T) =
|
||||
var context = newConstructionContext()
|
||||
|
@ -675,7 +792,7 @@ proc load*[K](input: Stream, target: var K) =
|
|||
let e = (ref YamlStreamError)(getCurrentException())
|
||||
if e.parent of IOError: raise (ref IOError)(e.parent)
|
||||
elif e.parent of YamlParserError: raise (ref YamlParserError)(e.parent)
|
||||
else: assert(false)
|
||||
else: assert false
|
||||
|
||||
proc setAnchor(a: var AnchorId, q: var Table[pointer, AnchorId])
|
||||
{.inline.} =
|
||||
|
|
26
yaml.nim
26
yaml.nim
|
@ -693,6 +693,32 @@ template setTagUri*(t: typedesc, uri: string, idName: expr): stmt =
|
|||
proc yamlTag*(T: typedesc[t]): TagId {.inline, raises: [].} = idName
|
||||
## autogenerated
|
||||
|
||||
proc canBeImplicit(t: typedesc): bool {.compileTime.} =
|
||||
let tDesc = getType(t)
|
||||
if tDesc.kind != nnkObjectTy: return false
|
||||
if tDesc[1].len != 1: return false
|
||||
if tDesc[1][0].kind != nnkRecCase: return false
|
||||
var foundEmptyBranch = false
|
||||
for i in 1.. tDesc[1][0].len - 1:
|
||||
case tDesc[1][0][i][1].len # branch contents
|
||||
of 0:
|
||||
if foundEmptyBranch: return false
|
||||
else: foundEmptyBranch = true
|
||||
of 1: discard
|
||||
else: return false
|
||||
return true
|
||||
|
||||
template markAsImplicit*(t: typedesc): stmt =
|
||||
## Mark a variant object type as implicit. This requires the type to consist
|
||||
## of nothing but a case expression and each branch of the case expression
|
||||
## containing exactly one field - with the exception that one branch may
|
||||
## contain zero fields.
|
||||
when canBeImplicit(t):
|
||||
# this will be checked by means of compiles(implicitVariantObject(...))
|
||||
proc implicitVariantObject*(unused: t) = discard
|
||||
else:
|
||||
{. fatal: "This type cannot be marked as implicit" .}
|
||||
|
||||
static:
|
||||
# standard YAML tags used by serialization
|
||||
registeredUris.add("!")
|
||||
|
|
Loading…
Reference in New Issue