status-im
/
NimYAML
mirror of https://github.com/status-im/NimYAML.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'devel'

This commit is contained in:
Felix Krause 2016-11-08 21:30:33 +01:00
parent 721632322b 2ba08370eb
commit 52298298e5
55 changed files with 1813 additions and 813 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@ -4,7 +4,9 @@ test/tlex
test/tdom
test/tserialization
test/tjson
test/tparser
test/yamlTestSuite
test/tquickstart
test/*.exe
test/*.pdb
test/*.ilk
@ -17,4 +19,8 @@ libyaml.dylib
libyaml.so
bench/json
docout
doc/rstPreproc
doc/tmp.rst
doc/**/code
yaml-dev-kit
nimsuggest.log

8
.travis.yml
View File

@ -37,8 +37,12 @@ install:
before_script:
- export PATH="nim-$BRANCH/bin${PATH:+:$PATH}"
script:
- nim tests --verbosity:0
- nim yamlTestSuite --verbosity:0
- nim lexerTests
- nim parserTests
- nim jsonTests
- nim domTests
- nim serializationTests
- nim quickstartTests
cache:
directories:
- nim-master

22
CHANGELOG.md
View File

@ -1,3 +1,25 @@
### 0.8.0
Features:
* NimYAML now has a global tag URI prefix for Nim types,
`tag:nimyaml.org,2016:`. This prefix is denoted by the custom tag handle
`!n!`.
* Support arbitrary tag handles.
* Added ability to mark object and tuple fields as transient.
* Added ability to set a default value for object fields.
* Added ability to ignore key-value pairs in the input when loading object
values.
* Support `!!timestamp` by parsing it to `Time` from module `times`.
Bugfixes:
* Fixed a bug concerning duplicate TagIds for different tags in the
`serializationTagLibrary`
* Convert commas in tag URIs to semicolons when using a tag URI as generic
parameter to another one, because commas after the tag handle are interpreted
as flow separators.
### 0.7.0
Features:

7
README.md
View File

@ -25,15 +25,16 @@ available as tags in this repository and can be fetched via nimble:
## Developers
```bash
nim tests # runs unit tests (serialization, dom, json)
# for parser tests, see yamlTestSuite
nim tests # runs all tests
nim lexerTests # run lexer tests
nim parserTests # run parser tests (git-clones yaml-dev-kit)
nim serializationTests # runs serialization tests
nim quickstartTests # run tests for quickstart snippets from documentation
nim documentation # builds documentation to folder docout
nim server # builds the REST server used for the testing ground
nim bench # runs benchmarks, requires libyaml
nim clean # guess
nim build # build a library
nim yamlTestSuite # execute YAML test suite (git-clones yaml-dev-kit)
```
NimYAML needs at least Nim 0.15.0 in order to compile.

42
config.nims
View File

@ -8,29 +8,55 @@ task tests, "Run all tests":
--verbosity:0
setCommand "c", "test/tests"
task yamlTestSuite, "Run YAML 1.2 test suite":
task lexerTests, "Run lexer tests":
--r
--verbosity:0
setCommand "c", "test/yamlTestSuite"
setCommand "c", "test/tlex"
task parserTests, "Run parser tests":
--r
--verbosity:0
setCommand "c", "test/tparser"
task jsonTests, "Run JSON tests":
--r
--verbosity:0
setCommand "c", "test/tjson"
task domTests, "Run DOM tests":
--r
--verbosity:0
setCommand "c", "test/tdom"
task serializationTests, "Run serialization tests":
--r
--verbosity:0
setCommand "c", "test/serializing"
setCommand "c", "test/tserialization"
task quickstartTests, "Run quickstart tests":
--r
--verbosity:0
setCommand "c", "test/tquickstart"
task documentation, "Generate documentation":
exec "mkdir -p docout"
withDir "doc":
exec r"nim c rstPreproc"
exec r"./rstPreproc -o:tmp.rst index.txt"
exec r"nim rst2html -o:../docout/index.html tmp.rst"
exec r"./rstPreproc -o:tmp.rst api.txt"
exec r"nim rst2html -o:../docout/api.html tmp.rst"
exec r"./rstPreproc -o:tmp.rst serialization.txt"
exec r"nim rst2html -o:../docout/serialization.html tmp.rst"
exec r"nim rst2html -o:../docout/testing.html testing.rst"
exec r"nim rst2html -o:../docout/schema.html schema.rst"
exec "cp docutils.css style.css processing.svg ../docout"
exec r"nim doc2 -o:docout/yaml.html --docSeeSrcUrl:https://github.com/flyx/NimYAML/blob/`git log -n 1 --format=%H` yaml"
# bash! ah-ah \\ savior of the universe
for file in listFiles("yaml"):
let packageName = file[5..^5]
exec r"nim doc2 -o:docout/yaml." & packageName &
".html --docSeeSrcUrl:https://github.com/flyx/NimYAML/blob/yaml/`git log -n 1 --format=%H` " &
file
exec r"nim rst2html -o:docout/index.html doc/index.txt"
exec r"nim rst2html -o:docout/api.html doc/api.txt"
exec r"nim rst2html -o:docout/serialization.html doc/serialization.txt"
exec "cp doc/docutils.css doc/style.css doc/testing.html doc/processing.svg docout"
setCommand "nop"
task bench, "Benchmarking":

466
doc/index.txt
View File

@ -16,468 +16,4 @@ install it with `Nimble <https://github.com/nim-lang/nimble>`_:
.. code-block:: bash
nimble install yaml
Quickstart
==========
Dumping Nim objects as YAML
--------------------------
.. raw:: html
<table class="quickstart-example"><thead><tr><th>code.nim</th>
<th>out.yaml</th></tr></thead><tbody><tr><td>
.. code-block:: nim
import yaml
type Person = object
name : string
age : int32
var personList = newSeq[Person]()
personList.add(Person(name: "Karl Koch", age: 23))
personList.add(Person(name: "Peter Pan", age: 12))
var s = newFileStream("out.yaml", fmWrite)
dump(personList, s)
s.close()
.. raw:: html
</td>
<td>
.. code-block:: yaml
%YAML 1.2
--- !nim:system:seq(nim:custom:Person)
-
name: Karl Koch
age: 23
-
name: Peter Pan
age: 12
.. raw:: html
</td></tr></tbody></table>
Loading Nim objects from YAML
----------------------------
.. raw:: html
<table class="quickstart-example"><thead><tr><th>code.nim</th>
<th>in.yaml</th></tr></thead><tbody><tr><td>
.. code-block:: nim
import yaml
type Person = object
name : string
age : int32
var personList: seq[Person]
var s = newFileStream("in.yaml")
load(s, personList)
s.close()
.. raw:: html
</td>
<td>
.. code-block:: yaml
%YAML 1.2
---
- { name: Karl Koch, age: 23 }
- { name: Peter Pan, age: 12 }
.. raw:: html
</td></tr></tbody></table>
Customizing output style
----------------------
.. raw:: html
<table class="quickstart-example"><thead><tr><th>code.nim</th>
<th>out.yaml</th></tr></thead><tbody><tr><td>
.. code-block:: nim
import yaml
type Person = object
name: string
age: int32
var personList: seq[Person]
personList.add(Person(name: "Karl Koch", age: 23))
personList.add(Person(name: "Peter Pan", age: 12))
var s = newFileStream("out.yaml")
dump(personList, s, options = defineOptions(
style = psCanonical,
indentationStep = 3,
newlines = nlLF,
outputVersion = ov1_1))
s.close()
.. raw:: html
</td><td>
.. code-block:: yaml
%YAML 1.1
--- !nim:system:seq(nim:custom:Person)
[
!nim:custom:Person {
? !!str "name"
: !!str "Karl Koch",
? !!str "age"
: !nim:system:int32 "23"
},
!nim:custom:Person {
? !!str "name"
: !!str "Peter Pan",
? !!str "age"
: !nim:system:int32 "12"
}
]
.. raw:: html
</td></tr></tbody></table>
Dumping reference types and cyclic structures
---------------------------------------------
.. raw:: html
<table class="quickstart-example"><thead><tr><th>code.nim</th>
<th>out.yaml</th></tr></thead><tbody><tr><td>
.. code-block:: nim
import yaml
type
Node = ref NodeObj
NodeObj = object
name: string
left, right: Node
var node1, node2, node3: Node
new(node1); new(node2); new(node3)
node1.name = "Node 1"
node2.name = "Node 2"
node3.name = "Node 3"
node1.left = node2
node1.right = node3
node2.right = node3
node3.left = node1
var s = newFileStream("out.yaml", fmWrite)
dump(node1, s)
s.close()
.. raw:: html
</td><td>
.. code-block:: yaml
%YAML 1.2
--- !nim:custom:NodeObj &a
name: Node 1
left:
name: Node 2
left: !!null ~
right: &b
name: Node 3
left: *a
right: !!null ~
right: *b
.. raw:: html
</td></tr></tbody></table>
Loading reference types and cyclic structures
---------------------------------------------
.. raw:: html
<table class="quickstart-example"><thead><tr><th>code.nim</th>
<th>in.yaml</th></tr></thead><tbody><tr><td>
.. code-block:: nim
import yaml
type
Node = ref NodeObj
NodeObj = object
name: string
left, right: Node
var node1: Node
var s = newFileStream("in.yaml")
load(s, node1)
s.close()
.. raw:: html
</td><td>
.. code-block:: yaml
%YAML 1.2
--- !nim:custom:NodeObj &a
name: Node 1
left:
name: Node 2
left: ~
right: &b
name: Node 3
left: *a
right: ~
right: *b
.. raw:: html
</td></tr></tbody></table>
Defining a custom tag uri for a type
------------------------------------
.. raw:: html
<table class="quickstart-example"><thead><tr><th>code.nim</th>
<th>out.yaml</th></tr></thead><tbody><tr><td>
.. code-block:: nim
import yaml
type Mob = object
level, experience: int32
drops: seq[string]
setTagUri(Mob, "!Mob")
setTagUri(seq[string], "!Drops")
var mob = Mob(level: 42, experience: 1800, drops:
@["Sword of Mob Slaying"])
var s = newFileStream("out.yaml", fmWrite)
dump(mob, s,
options = defineOptions(tagStyle = tsAll))
s.close()
.. raw:: html
</td><td>
.. code-block:: yaml
YAML 1.2
--- !Mob
!nim:field level: !nim:system:int32 42
!nim:field experience: !nim:system:int32 1800
!nim:field drops: !Drops [!!str Sword of Mob Slaying]
.. raw:: html
</td></tr></tbody></table>
Dumping Nim objects as JSON
---------------------------
.. raw:: html
<table class="quickstart-example"><thead><tr><th>code.nim</th>
<th>out.yaml</th></tr></thead><tbody><tr><td>
.. code-block:: nim
import yaml
type Person = object
name : string
age : int32
var personList = newSeq[Person]()
personList.add(Person(name: "Karl Koch", age: 23))
personList.add(Person(name: "Peter Pan", age: 12))
var s = newFileStream("out.yaml", fmWrite)
dump(personList, s,
options = defineOptions(style = psJson))
s.close()
.. raw:: html
</td>
<td>
.. code-block:: yaml
[
{
"name": "Karl Koch",
"age": 23
},
{
"name": "Peter Pan",
"age": 12
}
]
.. raw:: html
</td></tr></tbody></table>
Loading Nim objects from JSON
-----------------------------
.. raw:: html
<table class="quickstart-example"><thead><tr><th>code.nim</th>
<th>in.yaml</th></tr></thead><tbody><tr><td>
.. code-block:: nim
import yaml
type Person = object
name : string
age : int32
var personList: seq[Person]
var s = newFileStream("in.yaml")
load(s, personList)
s.close()
.. raw:: html
</td>
<td>
.. code-block:: yaml
[
{
"name": "Karl Koch",
"age": 23
},
{
"name": "Peter Pan",
"age": 12
}
]
.. raw:: html
</td></tr></tbody></table>
Processing a Sequence of Heterogeneous Items
--------------------------------------------
… With variant objects
......................
.. raw:: html
<table class="quickstart-example"><thead><tr><th>code.nim</th>
<th>in.yaml</th></tr></thead><tbody><tr><td>
.. code-block:: nim
import yaml
type
Person = object
name: string
ContainerKind = enum
ckString, ckInt, ckBool, ckPerson, ckNone
Container = object
case kind: ContainerKind
of ckString:
strVal: string
of ckInt:
intVal: int
of ckBool:
boolVal: bool
of ckPerson:
personVal: Person
of ckNone:
discard
setTagUri(Person, "!nim:demo:Person")
# tell NimYAML to use Container as implicit type.
# only possible with variant object types where
# each branch contains at most one object.
markAsImplicit(Container)
var list: seq[Container]
var s = newFileStream("in.yaml")
load(s, list)
s.close()
assert(list[0].kind == ckString)
assert(list[0].strVal == "this is a string")
# and so on
.. raw:: html
</td>
<td>
.. code-block:: yaml
%YAML 1.2
---
- this is a string
- 42
- false
- !!str 23
- !nim:demo:Person {name: Trillian}
- !!null
.. raw:: html
</td></tr></tbody></table>
… With the Sequential API
.........................
.. raw:: html
<table class="quickstart-example"><thead><tr><th>code.nim</th>
<th>in.yaml</th></tr></thead><tbody><tr><td>
.. code-block:: nim
import yaml
type Person = object
name: string
setTagUri(Person, "!nim:demo:Person", yTagPerson)
var
s = newFileStream("in.yaml", fmRead)
context = newConstructionContext()
parser = newYamlParser(serializationTagLibrary)
events = parser.parse(s)
assert events.next().kind == yamlStartDoc
assert events.next().kind == yamlStartSeq
var nextEvent = events.peek()
while nextEvent.kind != yamlEndSeq:
var curTag = nextEvent.tag()
if curTag == yTagQuestionMark:
# we only support implicitly tagged scalars
assert nextEvent.kind == yamlScalar
case guessType(nextEvent.scalarContent)
of yTypeInteger: curTag = yTagInteger
of yTypeBoolTrue, yTypeBoolFalse:
curTag = yTagBoolean
of yTypeUnknown: curTag = yTagString
else: assert false, "Type not supported!"
elif curTag == yTagExclamationMark:
curTag = yTagString
case curTag
of yTagString:
var s: string
events.constructChild(context, s)
echo "got string: ", s
of yTagInteger:
var i: int32
events.constructChild(context, i)
echo "got integer: ", i
of yTagBoolean:
var b: bool
events.constructChild(context, b)
echo "got boolean: ", b
of yTagPerson:
var p: Person
events.constructChild(context, p)
echo "got Person with name: ", p.name
else: assert false, "unsupported tag: " & $curTag
nextEvent = events.peek()
assert events.next().kind == yamlEndSeq
assert events.next().kind == yamlEndDoc
assert events.finished()
s.close()
.. raw:: html
</td>
<td>
.. code-block:: yaml
%YAML 1.2
--- !!seq
- this is a string
- 42
- false
- !!str 23
- !nim:demo:Person {name: Trillian}
.. raw:: html
</td></tr></tbody></table>
%quickstart%0

134
doc/rstPreproc.nim Normal file
View File

@ -0,0 +1,134 @@
## This is a tool for preprocessing rst files. Lines starting with ``%`` will
## get substituted by nicely layouted nim and yaml code included from file in
## the snippets tree.
##
## The syntax of substituted lines is ``'%' path '%' level``. *path* shall be
## a path relative to the *snippets* directory. *level* shall be the level depth
## of the first title that should be produced.
##
## Usage:
##
## rstPreproc -o:path <infile>
##
## *path* is the output path. If omitted, it will be equal to infile with its
## suffix substituted by ``.rst``. *infile* is the source rst file.
##
## The reason for this complex approach is to have all snippets used in the docs
## available as source files for automatic testing. This way, we can make sure
## that the code in the docs actually works.
import parseopt2, streams, tables, strutils, os
var
infile = ""
path: string = nil
for kind, key, val in getopt():
case kind
of cmdArgument:
if infile == "":
if key == "":
echo "invalid input file with empty name!"
quit 1
infile = key
else:
echo "Only one input file is supported!"
quit 1
of cmdLongOption, cmdShortOption:
case key
of "out", "o":
if isNil(path): path = val
else:
echo "Duplicate output path!"
quit 1
else:
echo "Unknown option: ", key
quit 1
of cmdEnd: assert(false) # cannot happen
if infile == "":
echo "Missing input file!"
quit 1
if isNil(path):
for i in countdown(infile.len - 1, 0):
if infile[i] == '.':
if infile[i..^1] == ".rst": path = infile & ".rst"
else: path = infile[0..i] & "rst"
break
if isNil(path): path = infile & ".rst"
var tmpOut = newFileStream(path, fmWrite)
proc append(s: string) =
tmpOut.writeLine(s)
const headingChars = ['=', '-', '`', ':', '\'']
proc outputExamples(curPath: string, level: int = 0) =
let titlePath = curPath / "title"
if fileExists(titlePath):
let titleFile = open(titlePath, fmRead)
defer: titleFile.close()
var title = ""
if titleFile.readLine(title):
let headingChar = if level >= headingChars.len: headingChars[^1] else:
headingChars[level]
append(title)
append(repeat(headingChar, title.len) & '\l')
# process content files under this directory
var codeFiles = newSeq[string]()
for kind, filePath in walkDir(curPath, true):
if kind == pcFile:
if filePath != "title": codeFiles.add(filePath)
case codeFiles.len
of 0: discard
of 1:
let (_, _, extension) = codeFiles[0].splitFile()
append(".. code:: " & extension[1..^1])
append(" :file: " & (curPath / codeFiles[0]) & '\l')
of 2:
append(".. raw:: html")
append(" <table class=\"quickstart-example\"><thead><tr>")
for codeFile in codeFiles:
append(" <th>" & codeFile[3..^1] & "</th>")
append(" </th></tr></thead><tbody><tr><td>\n")
var first = true
for codeFile in codeFiles:
if first: first = false
else: append(".. raw:: html\n </td>\n <td>\n")
let (_, _, extension) = codeFile.splitFile()
append(".. code:: " & extension[1..^1])
append(" :file: " & (curPath / codeFile) & '\l')
append(".. raw:: html")
append(" </td></tr></tbody></table>\n")
else:
echo "Unexpected number of files in ", curPath, ": ", codeFiles.len
# process child directories
for kind, dirPath in walkDir(curPath):
if kind == pcDir:
outputExamples(dirPath, level + 1)
var lineNum = 0
for line in infile.lines():
if line.len > 0 and line[0] == '%':
var
srcPath: string = nil
level = 0
for i in 1..<line.len:
if line[i] == '%':
srcPath = line[1 .. i - 1]
level = parseInt(line[i + 1 .. ^1])
break
if isNil(srcPath):
echo "Second % missing in line " & $lineNum & "! content:\n"
echo line
quit 1
outputExamples("snippets" / srcPath, level)
else:
append(line)

140
doc/schema.rst Normal file
View File

@ -0,0 +1,140 @@
====================
Serialization Schema
====================
This document details the existing mappings in NimYAML from Nim types to YAML
tags. Throughout this document, there are two *tag shorthands* being used:
========= =========================
Shorthand Expansion
========= =========================
``!!`` ``tag:yaml.org,2002:``
``!n!`` ``tag:nimyaml.org,2016:``
========= =========================
The first one is defined by the YAML specification and is used for types from
the YAML failsafe, JSON or core schema. The second one is defined by NimYAML and
is used for types from the Nim standard library.
The YAML tag system has no understanding of generics. This means that NimYAML
must map every generic type instance to a YAML tag that describes that exact
type instance. For example, a ``seq[string]`` is mapped to the tag
``!n!system:seq(tag:yaml.org;2002:string)``.
As you can see, the expanded tag handle of the generic type parameter is added
to the tag of the generic type. To be compliant with the YAML spec, the
following modifications are made:
* Any exclamation marks are removed from the expanded tag. An exclamation mark
may only occur at the beginning of the tag as defined by the YAML spec.
* Any commas are replaces by semicolons, because they may not occur in a tag
apart from within the tag handle expansion.
If a type takes multiple generic parameters, the tag handles are separated by
semicolons within the parentheses. Note that this does not guarantee unique tag
handles for every type, but it is currently seen as good enough.
Note that user-defined generic types are currently not officially supported by
NimYAML. Only the generic collection types explicitly listed here use this
mechanism for crafting YAML tags.
Scalar Types
============
The following table defines all non-composed, atomar types that are mapped to
YAML types by NimYAML.
========= ===========================================================
Nim type YAML tag
========= ===========================================================
char ``!n!system:char``
string ``!!string`` (or ``!n!nil:string`` if nil)
int ``!n!system:int32`` (independent on target architecture)
int8 ``!n!system:int8``
int16 ``!n!system:int16``
int32 ``!n!system:int32``
int64 ``!n!system:int64``
uint ``!n!system:uint32`` (independent from target architecture)
uint8 ``!n!system:uint8``
uint16 ``!n!system:uint16``
uint32 ``!n!system:uint32``
uint64 ``!n!system:uint64``
float ``!n!system:float64``
float32 ``!n!system:float32``
float64 ``!n!system:float64``
bool ``!!bool``
Time ``!!timestamp``
========= ===========================================================
Apart from these standard library types, NimYAML also supports all enum types
as scalar types. They will be serialized to their string representation.
Apart from the types listed here and enum tyes, no atomar types are supported.
Collection Types
================
Collection types in Nim are typically generic. As such, they take their
contained types as parameters inside parentheses as explained above. The
following types are supported:
============ ============================================================ ================================
Nim type YAML tag YAML structure
============ ============================================================ ================================
array ``!n!system:array(?;?)`` (first parameter like ``0..5``) sequence
seq ``!n!system:seq(?)`` (or ``!n!nil:seq`` if nil) sequence
set ``!n!system:set(?)`` sequence
Table ``!n!tables:Table(?;?)`` mapping
OrderedTable ``!n!tables:OrderedTable(?;?)`` sequence of single-pair mappings
============ ============================================================ ================================
Standard YAML Types
===================
NimYAML does not support all types defined in the YAML specification, **not even
those of the failsafe schema**. The reason is that the failsafe schema is
designed for dynamic type systems where a sequence can contain arbitrarily typed
values. This is not fully translatable into a static type system. NimYAML does
support some mechanisms to make working with heterogeneous collection structures
easier, see `Serialization Overview <serialization.html>`_.
Note that because the specification only defines that an implementation *should*
implement the failsafe schema, NimYAML is still compliant; it has valid reasons
not to implement the schema.
This is a full list of all types defined in the YAML specification or the
`YAML type registry <http://www.yaml.org/type/>`_. It gives an overview of which
types are supported by NimYAML, which may be supported in the future and which
will never be supported.
=============== ============================================
YAML type Status
=============== ============================================
``!!map`` Cannot be supported
``!!omap`` Cannot be supported
``!!pairs`` Cannot be supported
``!!set`` Cannot be supported
``!!seq`` Cannot be supported
``!!binary`` Currently not supported
``!!bool`` Maps to Nim's ``bool`` type
``!!float`` Not supported (user can choose)
``!!int`` Not supported (user can choose)
``!!merge`` Not supported and unlikely to be implemented
``!!null`` Used for reference types that are ``nil``
``!!str`` Maps to Nim's ``string`` type
``!!timestamp`` Maps to Nim's ``Time`` type
``!!value`` Not supported and unlikely to be implemented
``!!yaml`` Not supported and unlikely to be implemented
=============== ============================================
``!!int`` and ``!!float`` are not supported out of the box to let the user
choose where to map them (for example, ``!!int`` may map to ``int32`` or
``int64``, or the the generic ``int`` whose size is platform-dependent). If one
wants to use ``!!int``or ``!!float``, the process is to create a ``distinct``
type derived from the desired base type and then set its tag using
``setTagUri``.
``!!merge`` and ``!!value`` are not supported because the semantics of these
types would make a multi-pass loading process necessary and if one takes the
tag system seriously, ``!!merge`` can only be used with YAML's collection types,
which, as explained above, cannot be supported.

52
doc/serialization.txt
View File

@ -36,7 +36,8 @@ Supported Types
NimYAML supports a growing number of types of Nim's ``system`` module and
standard library, and it also supports user-defined object, tuple and enum types
out of the box.
out of the box. A complete list of explicitly supported types is available in
`Schema <schema.html>`_.
**Important**: NimYAML currently does not support polymorphism. This may be
added in the future.
@ -105,7 +106,7 @@ possible to dump and load cyclic data structures without further configuration.
It is possible for reference types to hold a ``nil`` value, which will be mapped
to the ``!!null`` YAML scalar type.
Pointer types are not supported because it seems dangerous to automatically
``ptr`` types are not supported because it seems dangerous to automatically
allocate memory which the user must then manually deallocate.
User Defined Types
@ -234,7 +235,7 @@ otherwise, it would be loaded as ``nil``.
As you might have noticed in the example above, the YAML tag of a ``seq``
depends on its generic type parameter. The same applies to ``Table``. So, a
table that maps ``int8`` to string sequences would be presented with the tag
``!nim:tables:Table(nim:system:int8,nim:system:seq(tag:yaml.org,2002:string))``.
``!n!tables:Table(tag:nimyaml.org,2016:int8,tag:nimyaml.org,2016:system:seq(tag:yaml.org,2002:string))``.
These tags are generated on the fly based on the types you instantiate
``Table`` or ``seq`` with.
@ -247,6 +248,51 @@ for example use it on a certain ``seq`` type:
setTagUri(seq[string], "!nim:my:seq")
Customizing Field Handling
==========================
NimYAML allows the user to specify special handling of certain object fields.
This configuration will be applied at compile time when NimYAML generates the
(de)serialization code for an object type. It is important that the
configuration happens before any YAML operations (e.g. ``load`` or ``dump``) are
executed on a variable of the object type.
Transient Fields
----------------
It may happen that certain fields of an object type are transient, i.e. they are
used in a way that makes (de)serializing them unnecessary. Such fields can be
marked as transient. This will cause them not to be serialized to YAML. They
will also not be accepted when loading the object.
Example:
.. code-block:: nim
type MyObject: object
storable: string
temporary: string
markAsTransient(MyObject, temporary)
Default Values
--------------
When you load YAML that has been written by a human, you might want to allow the
user to omit certain fields, which should then be filled with a default value.
You can do that like this:
.. code-block:: nim
type MyObject: object
required: string
optional: string
setDefaultValue(MyObject, optional, "default value")
Whenever ``MyObject`` now is loaded and the input stream does not contain the
field ``optional``, that field will be set to the value ``"default value"``.
Customize Serialization
=======================

12
doc/snippets/quickstart/00/00-code.nim Normal file
View File

@ -0,0 +1,12 @@
import yaml.serialization, streams
type Person = object
name : string
age : int32
var personList = newSeq[Person]()
personList.add(Person(name: "Karl Koch", age: 23))
personList.add(Person(name: "Peter Pan", age: 12))
var s = newFileStream("out.yaml", fmWrite)
dump(personList, s)
s.close()

9
doc/snippets/quickstart/00/01-out.yaml Normal file
View File

@ -0,0 +1,9 @@
%YAML 1.2
%TAG !n! tag:nimyaml.org,2016:
--- !n!system:seq(tag:nimyaml.org;2016:custom:Person)
-
name: Karl Koch
age: 23
-
name: Peter Pan
age: 12

1
doc/snippets/quickstart/00/title Normal file
View File

@ -0,0 +1 @@
Dumping Nim objects as YAML

9
doc/snippets/quickstart/01/00-code.nim Normal file
View File

@ -0,0 +1,9 @@
import yaml.serialization, streams
type Person = object
name : string
age : int32
var personList: seq[Person]
var s = newFileStream("in.yaml")
load(s, personList)
s.close()

4
doc/snippets/quickstart/01/01-in.yaml Normal file
View File

@ -0,0 +1,4 @@
%YAML 1.2
---
- { name: Karl Koch, age: 23 }
- { name: Peter Pan, age: 12 }

1
doc/snippets/quickstart/01/title Normal file
View File

@ -0,0 +1 @@
Loading Nim objects from YAML

16
doc/snippets/quickstart/02/00-code.nim Normal file
View File

@ -0,0 +1,16 @@
import yaml.serialization, yaml.presenter, streams
type Person = object
name: string
age: int32
var personList = newSeq[Person]()
personList.add(Person(name: "Karl Koch", age: 23))
personList.add(Person(name: "Peter Pan", age: 12))
var s = newFileStream("out.yaml", fmWrite)
dump(personList, s, options = defineOptions(
style = psCanonical,
indentationStep = 3,
newlines = nlLF,
outputVersion = ov1_1))
s.close()

17
doc/snippets/quickstart/02/01-out.yaml Normal file
View File

@ -0,0 +1,17 @@
%YAML 1.1
%TAG !n! tag:nimyaml.org,2016:
---
!n!system:seq(tag:nimyaml.org;2016:custom:Person) [
!n!custom:Person {
? !n!field "name"
: !!str "Karl Koch",
? !n!field "age"
: !n!system:int32 "23"
},
!n!custom:Person {
? !n!field "name"
: !!str "Peter Pan",
? !n!field "age"
: !n!system:int32 "12"
}
]

1
doc/snippets/quickstart/02/title Normal file
View File

@ -0,0 +1 @@
Customizing output style

20
doc/snippets/quickstart/03/00-code.nim Normal file
View File

@ -0,0 +1,20 @@
import yaml.serialization, streams
type
Node = ref NodeObj
NodeObj = object
name: string
left, right: Node
var node1, node2, node3: Node
new(node1); new(node2); new(node3)
node1.name = "Node 1"
node2.name = "Node 2"
node3.name = "Node 3"
node1.left = node2
node1.right = node3
node2.right = node3
node3.left = node1
var s = newFileStream("out.yaml", fmWrite)
dump(node1, s)
s.close()

12
doc/snippets/quickstart/03/01-out.yaml Normal file
View File

@ -0,0 +1,12 @@
%YAML 1.2
%TAG !n! tag:nimyaml.org,2016:
--- !n!custom:NodeObj &a
name: Node 1
left:
name: Node 2
left: !!null ~
right: &b
name: Node 3
left: *a
right: !!null ~
right: *b

1
doc/snippets/quickstart/03/title Normal file
View File

@ -0,0 +1 @@
Dumping reference types and cyclic structures

12
doc/snippets/quickstart/04/00-code.nim Normal file
View File

@ -0,0 +1,12 @@
import yaml.serialization, streams
type
Node = ref NodeObj
NodeObj = object
name: string
left, right: Node
var node1: Node
var s = newFileStream("in.yaml")
load(s, node1)
s.close()

12
doc/snippets/quickstart/04/01-in.yaml Normal file
View File

@ -0,0 +1,12 @@
%YAML 1.2
%TAG !n! tag:nimyaml.org,2016:
--- !n!custom:NodeObj &a
name: Node 1
left:
name: Node 2
left: ~
right: &b
name: Node 3
left: *a
right: ~
right: *b

1
doc/snippets/quickstart/04/title Normal file
View File

@ -0,0 +1 @@
Loading reference types and cyclic structures

13
doc/snippets/quickstart/05/00-code.nim Normal file
View File

@ -0,0 +1,13 @@
import yaml, streams
type Mob = object
level, experience: int32
drops: seq[string]
setTagUri(Mob, "!Mob")
setTagUri(seq[string], "!Drops")
var mob = Mob(level: 42, experience: 1800, drops:
@["Sword of Mob Slaying"])
var s = newFileStream("out.yaml", fmWrite)
dump(mob, s, tagStyle = tsAll)
s.close()

6
doc/snippets/quickstart/05/01-out.yaml Normal file
View File

@ -0,0 +1,6 @@
%YAML 1.2
%TAG !n! tag:nimyaml.org,2016:
--- !Mob
!n!field level: !n!system:int32 42
!n!field experience: !n!system:int32 1800
!n!field drops: !Drops [!!str Sword of Mob Slaying]

1
doc/snippets/quickstart/05/title Normal file
View File

@ -0,0 +1 @@
Defining a custom tag uri for a type

13
doc/snippets/quickstart/06/00-code.nim Normal file
View File

@ -0,0 +1,13 @@
import yaml.serialization, yaml.presenter, streams
type Person = object
name : string
age : int32
var personList = newSeq[Person]()
personList.add(Person(name: "Karl Koch", age: 23))
personList.add(Person(name: "Peter Pan", age: 12))
var s = newFileStream("out.yaml", fmWrite)
dump(personList, s,
options = defineOptions(style = psJson))
s.close()

11
doc/snippets/quickstart/06/01-out.yaml Normal file
View File

@ -0,0 +1,11 @@
[
{
"name": "Karl Koch",
"age": 23
},
{
"name": "Peter Pan",
"age": 12
}
]

1
doc/snippets/quickstart/06/title Normal file
View File

@ -0,0 +1 @@
Dumping Nim objects as JSON

10
doc/snippets/quickstart/07/00-code.nim Normal file
View File

@ -0,0 +1,10 @@
import yaml.serialization, streams
type Person = object
name : string
age : int32
var personList: seq[Person]
var s = newFileStream("in.yaml")
load(s, personList)
s.close()

10
doc/snippets/quickstart/07/01-in.yaml Normal file
View File

@ -0,0 +1,10 @@
[
{
"name": "Karl Koch",
"age": 23
},
{
"name": "Peter Pan",
"age": 12
}
]

1
doc/snippets/quickstart/07/title Normal file
View File

@ -0,0 +1 @@
Loading Nim objects from JSON

37
doc/snippets/quickstart/08/00/00-code.nim Normal file
View File

@ -0,0 +1,37 @@
import yaml, streams
type
Person = object
name: string
ContainerKind = enum
ckString, ckInt, ckBool, ckPerson, ckNone
Container = object
case kind: ContainerKind
of ckString:
strVal: string
of ckInt:
intVal: int
of ckBool:
boolVal: bool
of ckPerson:
personVal: Person
of ckNone:
discard
setTagUri(Person, nimTag("demo:Person"))
# tell NimYAML to use Container as implicit type.
# only possible with variant object types where
# each branch contains at most one object.
markAsImplicit(Container)
var list: seq[Container]
var s = newFileStream("in.yaml")
load(s, list)
s.close()
assert(list[0].kind == ckString)
assert(list[0].strVal == "this is a string")
# and so on

9
doc/snippets/quickstart/08/00/01-in.yaml Normal file
View File

@ -0,0 +1,9 @@
%YAML 1.2
%TAG !n! tag:nimyaml.org,2016:
---
- this is a string
- 42
- false
- !!str 23
- !n!demo:Person {name: Trillian}
- !!null

1
doc/snippets/quickstart/08/00/title Normal file
View File

@ -0,0 +1 @@
… With variant objects

51
doc/snippets/quickstart/08/01/00-code.nim Normal file
View File

@ -0,0 +1,51 @@
import yaml, streams
type Person = object
name: string
setTagUri(Person, nimTag("demo:Person"), yTagPerson)
var
s = newFileStream("in.yaml", fmRead)
context = newConstructionContext()
parser = newYamlParser(serializationTagLibrary)
events = parser.parse(s)
assert events.next().kind == yamlStartDoc
assert events.next().kind == yamlStartSeq
var nextEvent = events.peek()
while nextEvent.kind != yamlEndSeq:
var curTag = nextEvent.tag()
if curTag == yTagQuestionMark:
# we only support implicitly tagged scalars
assert nextEvent.kind == yamlScalar
case guessType(nextEvent.scalarContent)
of yTypeInteger: curTag = yTagInteger
of yTypeBoolTrue, yTypeBoolFalse:
curTag = yTagBoolean
of yTypeUnknown: curTag = yTagString
else: assert false, "Type not supported!"
elif curTag == yTagExclamationMark:
curTag = yTagString
case curTag
of yTagString:
var s: string
events.constructChild(context, s)
echo "got string: ", s
of yTagInteger:
var i: int32
events.constructChild(context, i)
echo "got integer: ", i
of yTagBoolean:
var b: bool
events.constructChild(context, b)
echo "got boolean: ", b
of yTagPerson:
var p: Person
events.constructChild(context, p)
echo "got Person with name: ", p.name
else: assert false, "unsupported tag: " & $curTag
nextEvent = events.peek()
assert events.next().kind == yamlEndSeq
assert events.next().kind == yamlEndDoc
assert events.finished()
s.close()

8
doc/snippets/quickstart/08/01/01-in.yaml Normal file
View File

@ -0,0 +1,8 @@
%YAML 1.2
%TAG !n! tag:nimyaml.org,2016:
--- !!seq
- this is a string
- 42
- false
- !!str 23
- !n!demo:Person {name: Trillian}

1
doc/snippets/quickstart/08/01/title Normal file
View File

@ -0,0 +1 @@
… With the Sequential API

1
doc/snippets/quickstart/08/title Normal file
View File

@ -0,0 +1 @@
Processing a Sequence of Heterogeneous Items

1
doc/snippets/quickstart/title Normal file
View File

@ -0,0 +1 @@
Quickstart

12
doc/style.css
View File

@ -54,12 +54,15 @@ header span:hover > ul {
}
header span ul a {
font-size: smaller;
font-family: "Source Code Pro", Menlo, "Courier New", Courier, monospace;
padding: 0 10px;
line-height: 40px;
}
header span ul.monospace a {
font-size: smaller;
font-family: "Source Code Pro", Menlo, "Courier New", Courier, monospace;
}
header a:link,
header a:visited {
background: inherit;
@ -108,6 +111,11 @@ dt a:before {
visibility: hidden;
}
#testingground {
margin-left: -50px;
margin-right: -50px;
}
#testingground textarea {
width: 100%;
height: 100%;

43
doc/testing.html → doc/testing.rst
View File

@ -1,32 +1,11 @@
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>NimYAML - Testing Ground</title>
==============
Testing Ground
==============
<link href="docutils.css" rel="stylesheet" type="text/css"/>
<link href="style.css" rel="stylesheet" type="text/css"/>
Input is being processed on the fly by a friendly web service and output is
updated as you type.
<link href='http://fonts.googleapis.com/css?family=Raleway:400,600,900' rel='stylesheet' type='text/css'/>
<link href='http://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
</head>
<body>
<header>
<a class="pagetitle" href="index.html">NimYAML</a>
<a href="index.html">Home</a>
<a href="testing.html">Testing Ground</a>
<span>Docs:</span>
<a href="api.html">Overview</a>
<a href="serialization.html">Serialization</a>
<a href="yaml.html">Module yaml</a>
</header>
<article id="documentId">
<div class="container">
<h1 class="title">Testing Ground</h1>
<p>Input is being processed on the fly by a friendly web service and
Output is updated as you type.</p>
</div>
</article>
.. raw:: html
<section id="testingground">
<table style="width: 100%; table-layout: fixed">
<thead>
@ -39,12 +18,12 @@
<tr>
<td style="width: 50%; height: 550px; vertical-align: top;">
<textarea id="yaml-input" style="width: 100%; height: 100%">
- test some
- {YAML: here}
- foo: bar
- test some
- {YAML: here}
- foo: bar
? [1, 2, 3]
: !!str "string"
-
-
? &amp;a anchor
: !!bool yes
? reference to anchor
@ -139,5 +118,3 @@
}
parse();
</script>
</body>
</html>

10
nimdoc.cfg
View File

@ -110,10 +110,16 @@ doc.file = """
<a href="testing.html">Testing Ground</a>
<span>Docs:</span>
<a href="api.html">Overview</a>
<a href="serialization.html">Serialization</a>
<span>
<a href="#">Serialization</a>
<ul>
<li><a href="serialization.html">Overview</a></li>
<li><a href="schema.html">Schema</a></li>
</ul>
</span>
<span>
<a href="#">Modules</a>
<ul>
<ul class="monospace">
<li><a href="yaml.html">yaml</a></li>
<li><a href="yaml.dom.html">yaml.dom</a></li>
<li><a href="yaml.hints.html">yaml.hints</a></li>

2
test/tests.nim
View File

@ -4,4 +4,4 @@
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
import tlex, tjson, tserialization, tdom
import tlex, tjson, tserialization, tdom, tparser, tquickstart

113
test/tparser.nim Normal file
View File

@ -0,0 +1,113 @@
# NimYAML - YAML implementation in Nim
# (c) Copyright 2016 Felix Krause
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
import os, osproc, terminal, strutils, streams, macros, unittest
import testEventParser, commonTestUtils
import "../yaml"
const devKitFolder = "yaml-dev-kit"
proc echoError(msg: string) =
styledWriteLine(stdout, fgRed, "[error] ", fgWhite, msg, resetStyle)
proc ensureDevKitCloneCorrect(pwd: string) {.compileTime.} =
let absolutePath = pwd / devKitFolder
if dirExists(absolutePath):
var isCorrectClone = true
if dirExists(absolutePath / ".git"):
let remoteUrl =
staticExec("cd \"" & absolutePath & "\" && git remote get-url origin")
if remoteUrl != "https://github.com/ingydotnet/yaml-dev-kit.git":
isCorrectClone = false
let branches = staticExec("cd \"" & absolutePath & "\" && git branch")
if "* data" notin branches.splitLines():
isCorrectClone = false
if isCorrectClone:
let updateOutput = staticExec("cd \"" & absolutePath & "\" && git pull")
#if uError != 0:
# echo "could not update yaml-dev-kit! please fix this problem and compile again."
# echo "output:\n"
# echo "$ git pull"
# echo updateOutput
# quit 1
else:
echo devKitFolder, " exists, but is not in expected state. Make sure it is a git repo,"
echo "cloned from https://github.com/ingydotnet/yaml-dev-kit.git, and the data branch"
echo "is active. Alternatively, delete the folder " & devKitFolder & '.'
quit 1
else:
let cloneOutput = staticExec("cd \"" & pwd &
"\" && git clone https://github.com/ingydotnet/yaml-dev-kit.git -b data")
#if cError != 0:
if not(dirExists(absolutePath)) or not(dirExists(absolutePath / ".git")) or
not(dirExists(absolutePath / "229Q")):
echo "could not clone https://github.com/ingydotnet/yaml-dev-kit.git. Make sure"
echo "you are connected to the internet and your proxy settings are correct. output:\n"
echo "$ git clone https://github.com/ingydotnet/yaml-dev-kit.git"
echo cloneOutput
quit 1
proc parserTest(path: string): bool =
var
tagLib = initExtendedTagLibrary()
parser = newYamlParser(tagLib)
actualIn = newFileStream(path / "in.yaml")
actual = parser.parse(actualIn)
expectedIn = newFileStream(path / "test.event")
expected = parseEventStream(expectedIn, tagLib)
defer:
actualIn.close()
expectedIn.close()
var i = 1
try:
while not actual.finished():
let actualEvent = actual.next()
if expected.finished():
echoError("At token #" & $i & ": Expected stream end, got " &
$actualEvent.kind)
return false
let expectedEvent = expected.next()
if expectedEvent != actualEvent:
printDifference(expectedEvent, actualEvent)
echoError("At token #" & $i &
": Actual tokens do not match expected tokens")
return false
i.inc()
if not expected.finished():
echoError("Got fewer tokens than expected, first missing " &
"token: " & $expected.next().kind)
return false
except:
let e = getCurrentException()
if e.parent of YamlParserError:
let pe = (ref YamlParserError)(e.parent)
echo "line ", pe.line, ", column ", pe.column, ": ", pe.msg
echo pe.lineContent
else: echo e.msg
echoError("Catched an exception at token #" & $i &
" test was not successful")
return false
result = true
macro genTests(): untyped =
let
pwd = staticExec("pwd")
absolutePath = '"' & (pwd / devKitFolder) & '"'
echo "[tparser] Generating tests from " & absolutePath
ensureDevKitCloneCorrect(pwd)
result = newStmtList()
# walkDir for some crude reason does not work with travis build
let dirItems = staticExec("ls -1d " & absolutePath / "*")
for dirPath in dirItems.splitLines():
if dirPath[^4..^1] in [".git", "name", "tags", "meta"]: continue
let title = slurp(dirPath / "===")
result.add(newCall("test",
newLit(strip(title) & " [" &
dirPath[^4..^1] & ']'), newCall("doAssert", newCall("parserTest",
newLit(dirPath)))))
result = newCall("suite", newLit("Parser Tests (from yaml-dev-kit)"), result)
genTests()

131
test/tquickstart.nim Normal file
View File

@ -0,0 +1,131 @@
import unittest, os, osproc, macros, strutils, streams
proc inputTest(basePath, path, nimPath: string): bool =
let
absolutePath = basePath / path
inFileOrig = absolutePath / "01-in.yaml"
inFileDest = absolutePath / "in.yaml"
codeFileOrig = absolutePath / "00-code.nim"
codeFileDest = absolutePath / "code.nim"
exeFileDest = when defined(windows): absolutePath / "code.exe" else:
absolutePath / "code"
copyFile(inFileOrig, inFileDest)
copyFile(codeFileOrig, codeFileDest)
defer:
removeFile(inFileDest)
removeFile(codeFileDest)
var process = startProcess(nimPath & " c --hints:off -p:" & escape(basePath) &
" code.nim", absolutePath, [], nil, {poStdErrToStdOut, poEvalCommand})
defer:
process.close()
if process.waitForExit() != 0:
echo "compiler output:"
echo "================\n"
echo process.outputStream().readAll()
result = false
else:
defer: removeFile(exeFileDest)
process.close()
process = startProcess(absolutePath / "code", absolutePath, [], nil,
{poStdErrToStdOut, poEvalCommand})
if process.waitForExit() != 0:
echo "executable output:"
echo "==================\n"
echo process.outputStream().readAll()
result = false
else: result = true
proc outputTest(basePath, path, nimPath: string): bool =
let
absolutePath = basePath / path
codeFileOrig = absolutePath / "00-code.nim"
codeFileDest = absolutePath / "code.nim"
exeFileDest = when defined(windows): absolutePath / "code.exe" else:
absolutePath / "code"
outFileExpected = absolutePath / "01-out.yaml"
outFileActual = absolutePath / "out.yaml"
copyFile(codeFileOrig, codeFileDest)
defer: removeFile(codeFileDest)
var process = startProcess(nimPath & " c --hints:off -p:" & escape(basePath) &
" code.nim", absolutePath, [], nil, {poStdErrToStdOut, poEvalCommand})
defer: process.close()
if process.waitForExit() != 0:
echo "compiler output:"
echo "================\n"
echo process.outputStream().readAll()
result = false
else:
defer: removeFile(exeFileDest)
process.close()
process = startProcess(absolutePath / "code", absolutePath, [], nil,
{poStdErrToStdOut, poEvalCommand})
if process.waitForExit() != 0:
echo "executable output:"
echo "==================\n"
echo process.outputStream().readAll()
result = false
else:
defer: removeFile(outFileActual)
var
expected = open(outFileExpected, fmRead)
actual = open(outFileActual, fmRead)
lineNumber = 1
defer:
expected.close()
actual.close()
var
expectedLine = ""
actualLine = ""
while true:
if expected.readLine(expectedLine):
if actual.readLine(actualLine):
if expectedLine != actualLine:
echo "difference at line #", lineNumber, ':'
echo "expected: ", escape(expectedLine)
echo " actual: ", escape(actualLine)
return false
else:
echo "actual output has fewer lines than expected; ",
"first missing line: #", lineNumber
echo "expected: ", escape(expectedLine)
return false
else:
if actual.readLine(actualLine):
echo "actual output has more lines than expected; ",
"first unexpected line: #", lineNumber
echo "content: ", escape(actualLine)
return false
else: break
lineNumber.inc()
result = true
proc testsFor(path: string, root: bool = true, titlePrefix: string = ""):
NimNode {.compileTime.} =
result = newStmtList()
let
baseDir = staticExec("pwd")
nimPathRaw = staticExec("which nim")
nimPath = if nimPathRaw[0] == '/': nimPathRaw else: baseDir / nimPathRaw
title = titlePrefix & slurp(baseDir / path / "title").splitLines()[0]
if fileExists(path / "00-code.nim"):
var test = newCall("test", newLit(title))
if fileExists(path / "01-in.yaml"):
test.add(newCall("doAssert", newCall("inputTest", newLit(baseDir),
newLit(path), newLit(nimPath))))
elif fileExists(path / "01-out.yaml"):
test.add(newCall("doAssert", newCall("outputTest", newLit(baseDir),
newLit(path), newLit(nimPath))))
else:
echo "Error: neither 01-in.yaml nor 01-out.yaml exists in " & path & '!'
quit 1
result.add(test)
for kind, childPath in walkDir(path):
if kind == pcDir:
if childPath != path / "nimcache":
result.add(testsFor(childPath, false, if root: "" else: title & ' '))
if root:
result = newCall("suite", newLit(title), result)
macro genTests(): untyped = testsFor("doc/snippets/quickstart")
genTests()

210
test/tserialization.nim
View File

@ -5,7 +5,7 @@
# distribution, for details about the copyright.
import "../yaml"
import unittest, strutils, streams, tables
import unittest, strutils, streams, tables, times
type
MyTuple = tuple
@ -38,6 +38,40 @@ type
of akDog:
barkometer: int
DumbEnum = enum
deA, deB, deC, deD
NonVariantWithTransient = object
a, b, c, d: string
VariantWithTransient = object
gStorable, gTemporary: string
case kind: DumbEnum
of deA, deB:
cStorable, cTemporary: string
of deC:
alwaysThere: int
of deD:
neverThere: int
WithDefault = object
a, b, c, d: string
WithIgnoredField = object
x, y: int
markAsTransient(NonVariantWithTransient, a)
markAsTransient(NonVariantWithTransient, c)
markAsTransient(VariantWithTransient, gTemporary)
markAsTransient(VariantWithTransient, cTemporary)
markAsTransient(VariantWithTransient, neverThere)
setDefaultValue(WithDefault, b, "b")
setDefaultValue(WithDefault, d, "d")
ignoreInputKey(WithIgnoredField, "z")
proc `$`(v: BetterInt): string {.borrow.}
proc `==`(left, right: BetterInt): bool {.borrow.}
@ -45,6 +79,8 @@ setTagUri(TrafficLight, "!tl")
setTagUri(Node, "!example.net:Node")
setTagUri(BetterInt, "!test:BetterInt")
const yamlDirs = "%YAML 1.2\n%TAG !n! tag:nimyaml.org,2016:\n--- "
proc representObject*(value: BetterInt, ts: TagStyle = tsNone,
c: SerializationContext, tag: TagId) {.raises: [].} =
var
@ -78,8 +114,8 @@ template expectConstructionError(li, co: int, message: string, body: typed) =
let e = (ref YamlConstructionError)(getCurrentException())
doAssert li == e.line, "Expected error line " & $li & ", was " & $e.line
doAssert co == e.column, "Expected error column " & $co & ", was " & $e.column
doAssert message == e.msg, "Expected error message " & escape(message) &
", got " & escape(e.msg)
doAssert message == e.msg, "Expected error message \n" & escape(message) &
", got \n" & escape(e.msg)
proc newNode(v: string): ref Node =
new(result)
@ -104,7 +140,7 @@ suite "Serialization":
test "Dump integer without fixed length":
var input = -4247
var output = dump(input, tsNone, asTidy, blockOnly)
assertStringEqual "%YAML 1.2\n--- \n\"-4247\"", output
assertStringEqual yamlDirs & "\n\"-4247\"", output
when sizeof(int) == sizeof(int64):
input = int(int32.high) + 1
@ -162,7 +198,7 @@ suite "Serialization":
assert(result == 14)
test "Load nil string":
let input = newStringStream("!nim:nil:string \"\"")
let input = newStringStream("!<tag:nimyaml.org,2016:nil:string> \"\"")
var result: string
load(input, result)
assert isNil(result)
@ -170,7 +206,15 @@ suite "Serialization":
test "Dump nil string":
let input: string = nil
var output = dump(input, tsNone, asTidy, blockOnly)
assertStringEqual "%YAML 1.2\n--- \n!nim:nil:string \"\"", output
assertStringEqual yamlDirs & "\n!n!nil:string \"\"", output
test "Load timestamps":
let input = "[2001-12-15T02:59:43.1Z, 2001-12-14t21:59:43.10-05:00, 2001-12-14 21:59:43.10-5]"
var result: seq[Time]
load(input, result)
assert result.len() == 3
# currently, there is no good way of checking the result content, because
# the parsed Time may have any timezone offset.
test "Load string sequence":
let input = newStringStream(" - a\n - b")
@ -183,10 +227,10 @@ suite "Serialization":
test "Dump string sequence":
var input = @["a", "b"]
var output = dump(input, tsNone, asTidy, blockOnly)
assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output
assertStringEqual yamlDirs & "\n- a\n- b", output
test "Load nil seq":
let input = newStringStream("!nim:nil:seq \"\"")
let input = newStringStream("!<tag:nimyaml.org,2016:nil:seq> \"\"")
var result: seq[int]
load(input, result)
assert isNil(result)
@ -194,7 +238,7 @@ suite "Serialization":
test "Dump nil seq":
let input: seq[int] = nil
var output = dump(input, tsNone, asTidy, blockOnly)
assertStringEqual "%YAML 1.2\n--- \n!nim:nil:seq \"\"", output
assertStringEqual yamlDirs & "\n!n!nil:seq \"\"", output
test "Load char set":
let input = newStringStream("- a\n- b")
@ -207,7 +251,7 @@ suite "Serialization":
test "Dump char set":
var input = {'a', 'b'}
var output = dump(input, tsNone, asTidy, blockOnly)
assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output
assertStringEqual yamlDirs & "\n- a\n- b", output
test "Load array":
let input = newStringStream("- 23\n- 42\n- 47")
@ -220,7 +264,7 @@ suite "Serialization":
test "Dump array":
let input = [23'i32, 42'i32, 47'i32]
var output = dump(input, tsNone, asTidy, blockOnly)
assertStringEqual "%YAML 1.2\n--- \n- 23\n- 42\n- 47", output
assertStringEqual yamlDirs & "\n- 23\n- 42\n- 47", output
test "Load Table[int, string]":
let input = newStringStream("23: dreiundzwanzig\n42: zweiundvierzig")
@ -235,7 +279,7 @@ suite "Serialization":
input[23] = "dreiundzwanzig"
input[42] = "zweiundvierzig"
var output = dump(input, tsNone, asTidy, blockOnly)
assertStringEqual("%YAML 1.2\n--- \n23: dreiundzwanzig\n42: zweiundvierzig",
assertStringEqual(yamlDirs & "\n23: dreiundzwanzig\n42: zweiundvierzig",
output)
test "Load OrderedTable[tuple[int32, int32], string]":
@ -259,8 +303,8 @@ suite "Serialization":
input.add((a: 23'i32, b: 42'i32), "dreiundzwanzigzweiundvierzig")
input.add((a: 13'i32, b: 47'i32), "dreizehnsiebenundvierzig")
var output = dump(input, tsRootOnly, asTidy, blockOnly)
assertStringEqual("""%YAML 1.2
--- !nim:tables:OrderedTable(nim:tuple(nim:system:int32,nim:system:int32),tag:yaml.org,2002:str)
assertStringEqual(yamlDirs &
"""!n!tables:OrderedTable(tag:nimyaml.org;2016:tuple(tag:nimyaml.org;2016:system:int32;tag:nimyaml.org;2016:system:int32);tag:yaml.org;2002:str)
-
?
a: 23
@ -284,11 +328,11 @@ suite "Serialization":
test "Dump Sequences in Sequence":
let input = @[@[1.int32, 2.int32, 3.int32], @[4.int32, 5.int32], @[6.int32]]
var output = dump(input, tsNone)
assertStringEqual "%YAML 1.2\n--- \n- [1, 2, 3]\n- [4, 5]\n- [6]", output
assertStringEqual yamlDirs & "\n- [1, 2, 3]\n- [4, 5]\n- [6]", output
test "Load Enum":
let input =
newStringStream("!nim:system:seq(tl)\n- !tl tlRed\n- tlGreen\n- tlYellow")
newStringStream("!<tag:nimyaml.org,2016:system:seq(tl)>\n- !tl tlRed\n- tlGreen\n- tlYellow")
var result: seq[TrafficLight]
load(input, result)
assert result.len == 3
@ -299,7 +343,7 @@ suite "Serialization":
test "Dump Enum":
let input = @[tlRed, tlGreen, tlYellow]
var output = dump(input, tsNone, asTidy, blockOnly)
assertStringEqual "%YAML 1.2\n--- \n- tlRed\n- tlGreen\n- tlYellow", output
assertStringEqual yamlDirs & "\n- tlRed\n- tlGreen\n- tlYellow", output
test "Load Tuple":
let input = newStringStream("str: value\ni: 42\nb: true")
@ -312,7 +356,7 @@ suite "Serialization":
test "Dump Tuple":
let input = (str: "value", i: 42.int32, b: true)
var output = dump(input, tsNone)
assertStringEqual "%YAML 1.2\n--- \nstr: value\ni: 42\nb: y", output
assertStringEqual yamlDirs & "\nstr: value\ni: 42\nb: y", output
test "Load Tuple - unknown field":
let input = "str: value\nfoo: bar\ni: 42\nb: true"
@ -358,8 +402,8 @@ suite "Serialization":
test "Dump custom object":
let input = Person(firstnamechar: 'P', surname: "Pan", age: 12)
var output = dump(input, tsNone, asTidy, blockOnly)
assertStringEqual(
"%YAML 1.2\n--- \nfirstnamechar: P\nsurname: Pan\nage: 12", output)
assertStringEqual(yamlDirs &
"\nfirstnamechar: P\nsurname: Pan\nage: 12", output)
test "Load custom object - unknown field":
let input = " firstnamechar: P\n surname: Pan\n age: 12\n occupation: free"
@ -380,8 +424,8 @@ suite "Serialization":
load(input, result)
test "Load sequence with explicit tags":
let input = newStringStream("--- !nim:system:seq(" &
"tag:yaml.org,2002:str)\n- !!str one\n- !!str two")
let input = newStringStream(yamlDirs & "!n!system:seq(" &
"tag:yaml.org;2002:str)\n- !!str one\n- !!str two")
var result: seq[string]
load(input, result)
assert result[0] == "one"
@ -390,12 +434,12 @@ suite "Serialization":
test "Dump sequence with explicit tags":
let input = @["one", "two"]
var output = dump(input, tsAll, asTidy, blockOnly)
assertStringEqual("%YAML 1.2\n--- !nim:system:seq(" &
"tag:yaml.org,2002:str) \n- !!str one\n- !!str two", output)
assertStringEqual(yamlDirs & "!n!system:seq(" &
"tag:yaml.org;2002:str) \n- !!str one\n- !!str two", output)
test "Load custom object with explicit root tag":
let input = newStringStream(
"--- !nim:custom:Person\nfirstnamechar: P\nsurname: Pan\nage: 12")
"--- !<tag:nimyaml.org,2016:custom:Person>\nfirstnamechar: P\nsurname: Pan\nage: 12")
var result: Person
load(input, result)
assert result.firstnamechar == 'P'
@ -405,9 +449,8 @@ suite "Serialization":
test "Dump custom object with explicit root tag":
let input = Person(firstnamechar: 'P', surname: "Pan", age: 12)
var output = dump(input, tsRootOnly, asTidy, blockOnly)
assertStringEqual("%YAML 1.2\n" &
"--- !nim:custom:Person \nfirstnamechar: P\nsurname: Pan\nage: 12",
output)
assertStringEqual(yamlDirs &
"!n!custom:Person \nfirstnamechar: P\nsurname: Pan\nage: 12", output)
test "Load custom variant object":
let input = newStringStream(
@ -427,8 +470,8 @@ suite "Serialization":
let input = @[Animal(name: "Bastet", kind: akCat, purringIntensity: 7),
Animal(name: "Anubis", kind: akDog, barkometer: 13)]
var output = dump(input, tsNone, asTidy, blockOnly)
assertStringEqual """%YAML 1.2
---
assertStringEqual yamlDirs & """
-
-
name: Bastet
@ -450,6 +493,82 @@ suite "Serialization":
expectConstructionError(1, 32, "While constructing Animal: Missing field: \"purringIntensity\""):
load(input, result)
test "Load non-variant object with transient fields":
let input = "{b: b, d: d}"
var result: NonVariantWithTransient
load(input, result)
assert isNil(result.a)
assert result.b == "b"
assert isNil(result.c)
assert result.d == "d"
test "Load non-variant object with transient fields - unknown field":
let input = "{b: b, c: c, d: d}"
var result: NonVariantWithTransient
expectConstructionError(1, 9, "While constructing NonVariantWithTransient: Field \"c\" is transient and may not occur in input"):
load(input, result)
test "Dump non-variant object with transient fields":
let input = NonVariantWithTransient(a: "a", b: "b", c: "c", d: "d")
let output = dump(input, tsNone, asTidy, blockOnly)
assertStringEqual yamlDirs & "\nb: b\nd: d", output
test "Load variant object with transient fields":
let input = "[[gStorable: gs, kind: deB, cStorable: cs], [gStorable: a, kind: deD]]"
var result: seq[VariantWithTransient]
load(input, result)
assert result.len == 2
assert result[0].kind == deB
assert result[0].gStorable == "gs"
assert result[0].cStorable == "cs"
assert result[1].kind == deD
assert result[1].gStorable == "a"
test "Load variant object with transient fields":
let input = "[gStorable: gc, kind: deD, neverThere: foo]"
var result: VariantWithTransient
expectConstructionError(1, 38, "While constructing VariantWithTransient: Field \"neverThere\" is transient and may not occur in input"):
load(input, result)
test "Dump variant object with transient fields":
let input = @[VariantWithTransient(kind: deB, gStorable: "gs",
gTemporary: "gt", cStorable: "cs", cTemporary: "ct"),
VariantWithTransient(kind: deD, gStorable: "a", gTemporary: "b",
neverThere: 42)]
let output = dump(input, tsNone, asTidy, blockOnly)
assertStringEqual yamlDirs & """
-
-
gStorable: gs
-
kind: deB
-
cStorable: cs
-
-
gStorable: a
-
kind: deD""", output
test "Load object with ignored key":
let input = "[{x: 1, y: 2}, {x: 3, z: 4, y: 5}, {z: [1, 2, 3], x: 4, y: 5}]"
var result: seq[WithIgnoredField]
load(input, result)
assert result.len == 3
assert result[0].x == 1
assert result[0].y == 2
assert result[1].x == 3
assert result[1].y == 5
assert result[2].x == 4
assert result[2].y == 5
test "Load object with ignored key - unknown field":
let input = "{x: 1, y: 2, zz: 3}"
var result: WithIgnoredField
expectConstructionError(1, 16, "While constructing WithIgnoredField: Unknown field: \"zz\""):
load(input, result)
test "Dump cyclic data structure":
var
a = newNode("a")
@ -459,8 +578,7 @@ suite "Serialization":
b.next = c
c.next = a
var output = dump(a, tsRootOnly, asTidy, blockOnly)
assertStringEqual """%YAML 1.2
--- !example.net:Node &a
assertStringEqual yamlDirs & """!example.net:Node &a
value: a
next:
value: b
@ -469,8 +587,7 @@ next:
next: *a""", output
test "Load cyclic data structure":
let input = newStringStream("""%YAML 1.2
--- !nim:system:seq(example.net:Node)
let input = newStringStream(yamlDirs & """!n!system:seq(example.net:Node)
- &a
value: a
next: &b
@ -497,6 +614,24 @@ next:
assert(result[1].next == result[2])
assert(result[2].next == result[0])
test "Load object with default values":
let input = "a: abc\nc: dce"
var result: WithDefault
load(input, result)
assert result.a == "abc"
assert result.b == "b"
assert result.c == "dce"
assert result.d == "d"
test "Load object with partly default values":
let input = "a: abc\nb: bcd\nc: cde"
var result: WithDefault
load(input, result)
assert result.a == "abc"
assert result.b == "bcd"
assert result.c == "cde"
assert result.d == "d"
test "Load nil values":
let input = newStringStream("- ~\n- !!str ~")
var result: seq[ref string]
@ -517,8 +652,8 @@ next:
input.add(new string)
input[1][] = "~"
var output = dump(input, tsRootOnly, asTidy, blockOnly)
assertStringEqual(
"%YAML 1.2\n--- !nim:system:seq(tag:yaml.org,2002:str) \n- !!null ~\n- !!str ~",
assertStringEqual(yamlDirs &
"!n!system:seq(tag:yaml.org;2002:str) \n- !!null ~\n- !!str ~",
output)
test "Custom constructObject":
@ -532,8 +667,7 @@ next:
test "Custom representObject":
let input = @[1.BetterInt, 9998887.BetterInt, 98312.BetterInt]
var output = dump(input, tsAll, asTidy, blockOnly)
assertStringEqual """%YAML 1.2
--- !nim:system:seq(test:BetterInt)
assertStringEqual yamlDirs & """!n!system:seq(test:BetterInt)
- !test:BetterInt 1
- !test:BetterInt 9_998_887
- !test:BetterInt 98_312""", output

83
test/yamlTestSuite.nim
View File

@ -1,83 +0,0 @@
# NimYAML - YAML implementation in Nim
# (c) Copyright 2016 Felix Krause
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
import os, terminal, strutils, streams
import testEventParser, commonTestUtils
import "../yaml"
const gitCmd =
"git clone https://github.com/ingydotnet/yaml-dev-kit.git -b data"
proc echoError(msg: string) =
styledWriteLine(stdout, fgRed, "[error] ", fgWhite, msg, resetStyle)
proc echoInfo(msg: string) =
styledWriteLine(stdout, fgGreen, "[info] ", fgWhite, msg, resetStyle)
removeDir("yaml-dev-kit")
if execShellCmd(gitCmd) != 0:
echoError("Could not check out yaml-dev-kit (no internet connection?)")
quit(1)
var gotErrors = false
for kind, dirPath in walkDir("yaml-dev-kit"):
block curTest:
if kind == pcDir:
if dirPath[^4..^1] in [".git", "name", "tags", "meta"]: continue
var
tagLib = initExtendedTagLibrary()
parser = newYamlParser(tagLib)
actualIn = newFileStream(dirPath / "in.yaml")
actual = parser.parse(actualIn)
expectedIn = newFileStream(dirPath / "test.event")
expected = parseEventStream(expectedIn, tagLib)
styledWriteLine(stdout, fgBlue, "[test] ", fgWhite, dirPath[^4..^1],
": ", strip(readFile(dirPath / "===")), resetStyle)
var i = 1
try:
while not actual.finished():
let actualEvent = actual.next()
if expected.finished():
echoError("At token #" & $i & ": Expected stream end, got " &
$actualEvent.kind)
gotErrors = true
actualIn.close()
expectedIn.close()
break curTest
let expectedEvent = expected.next()
if expectedEvent != actualEvent:
printDifference(expectedEvent, actualEvent)
echoError("At token #" & $i &
": Actual tokens do not match expected tokens")
gotErrors = true
actualIn.close()
expectedIn.close()
break curTest
i.inc()
if not expected.finished():
echoError("Got fewer tokens than expected, first missing " &
"token: " & $expected.next().kind)
gotErrors = true
except:
gotErrors = true
let e = getCurrentException()
if e.parent of YamlParserError:
let pe = (ref YamlParserError)(e.parent)
echo "line ", pe.line, ", column ", pe.column, ": ", pe.msg
echo pe.lineContent
else: echo e.msg
echoError("Catched an exception at token #" & $i &
" test was not successful")
actualIn.close()
expectedIn.close()
if gotErrors:
echoError("There were errors while running the tests")
quit(1)
else:
echoInfo("All tests were successful")
quit(0)

14
yaml.nim
View File

@ -10,12 +10,12 @@
##
## There are three high-level APIs which are probably most useful:
##
## * The serialization API in `serialization <yaml/serialization.html>`_ enables
## * The serialization API in `serialization <yaml.serialization.html>`_ enables
## you to load YAML data directly into native Nim types, and reversely dump
## native Nim types as YAML.
## * The DOM API in `dom <yaml/dom.html>`_ parses YAML files in a tree structure
## * The DOM API in `dom <yaml.dom.html>`_ parses YAML files in a tree structure
## which you can navigate.
## * The JSON API in `tojson <yaml/tojson.html>`_ parses YAML files into the
## * The JSON API in `tojson <yaml.tojson.html>`_ parses YAML files into the
## Nim stdlib's JSON structure, which may be useful if you have other modules
## which expect JSON input. Note that the serialization API is able to write
## and load JSON; you do not need the JSON API for that.
@ -24,18 +24,18 @@
## enables you to process YAML input as data stream which does not need to be
## loaded into RAM completely at once. It consists of the following modules:
##
## * The stream API in `stream <yaml/stream.html>`_ defines the central type for
## * The stream API in `stream <yaml.stream.html>`_ defines the central type for
## stream processing, ``YamlStream``. It also contains definitions and
## constructor procs for stream events.
## * The parser API in `parser <yaml/parser.html>`_ gives you direct access to
## * The parser API in `parser <yaml.parser.html>`_ gives you direct access to
## the YAML parser's output.
## * The presenter API in `presenter <yaml/presenter.html>`_ gives you direct
## access to the presenter, i.e. the module that renders a YAML character
## stream.
## * The taglib API in `taglib <yaml/taglib.html>`_ provides a data structure
## * The taglib API in `taglib <yaml.taglib.html>`_ provides a data structure
## for keeping track of YAML tags that are generated by the parser or used in
## the presenter.
## * The hints API in `hints <yaml/hints.html>`_ provides a simple proc for
## * The hints API in `hints <yaml.hints.html>`_ provides a simple proc for
## guessing the type of a scalar value.
import yaml.dom, yaml.hints, yaml.parser, yaml.presenter,

2
yaml.nimble
View File

@ -1,6 +1,6 @@
# Package
version = "0.7.0"
version = "0.8.0"
author = "Felix Krause"
description = "YAML 1.2 implementation for Nim"
license = "MIT"

97
yaml/hints.nim
View File

@ -33,10 +33,11 @@ type
## ``yTypeBoolTrue`` ``y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON``
## ``yTypeBoolFalse`` ``n|N|no|No|NO|false|False|FALSE|off|Off|OFF``
## ``yTypeNull`` ``~ | null | Null | NULL``
## ``yTypeTimestamp`` see `here <http://yaml.org/type/timestamp.html>`_.
## ``yTypeUnknown`` ``*``
## ================== =========================
yTypeInteger, yTypeFloat, yTypeFloatInf, yTypeFloatNaN, yTypeBoolTrue,
yTypeBoolFalse, yTypeNull, yTypeUnknown
yTypeBoolFalse, yTypeNull, yTypeUnknown, yTypeTimestamp
YamlTypeHintState = enum
ythInitial,
@ -59,7 +60,16 @@ type
ythPointLowerIN, ythPointLowerN, ythPointLowerNA,
ythMinus, yth0, ythInt, ythDecimal, ythNumE, ythNumEPlusMinus, ythExponent
ythMinus, yth0, ythInt1, ythInt1Zero, ythInt2, ythInt2Zero, ythInt3,
ythInt3Zero, ythInt4, ythInt4Zero, ythInt,
ythDecimal, ythNumE, ythNumEPlusMinus, ythExponent,
ythYearMinus, ythMonth1, ythMonth2, ythMonthMinus, ythMonthMinusNoYmd,
ythDay1, ythDay1NoYmd, ythDay2, ythDay2NoYmd,
ythAfterDayT, ythAfterDaySpace, ythHour1, ythHour2, ythHourColon,
ythMinute1, ythMinute2, ythMinuteColon, ythSecond1, ythSecond2, ythFraction,
ythAfterTimeSpace, ythAfterTimeZ, ythAfterTimePlusMinus, ythTzHour1,
ythTzHour2, ythTzHourColon, ythTzMinute1, ythTzMinute2
macro typeHintStateMachine(c: untyped, content: untyped): typed =
yAssert content.kind == nnkStmtList
@ -101,20 +111,78 @@ template advanceTypeHint(ch: char) {.dirty.} =
typeHintStateMachine ch:
of '~': ythInitial => ythNULL
of '.':
[yth0, ythInt] => ythDecimal
[yth0, ythInt1, ythInt2, ythInt3, ythInt4, ythInt] => ythDecimal
[ythInitial, ythMinus] => ythPoint
of '+': ythNumE => ythNumEPlusMinus
ythSecond2 => ythFraction
of '+':
ythNumE => ythNumEPlusMinus
[ythFraction, ythSecond2] => ythAfterTimePlusMinus
of '-':
ythInitial => ythMinus
ythNumE => ythNumEPlusMinus
[ythInt4, ythInt4Zero] => ythYearMinus
ythMonth1 => ythMonthMinusNoYmd
ythMonth2 => ythMonthMinus
[ythFraction, ythSecond2] => ythAfterTimePlusMinus
of ':':
[ythHour1, ythHour2] => ythHourColon
ythMinute2 => ythMinuteColon
[ythTzHour1, ythTzHour2] => ythTzHourColon
of '0':
[ythInitial, ythMinus] => yth0
ythInitial => ythInt1Zero
ythMinus => yth0
[ythNumE, ythNumEPlusMinus] => ythExponent
[ythInt, ythDecimal, ythExponent] => nil
ythInt1 => ythInt2
ythInt1Zero => ythInt2Zero
ythInt2 => ythInt3
ythInt2Zero => ythInt3Zero
ythInt3 => ythInt4
ythInt3Zero => ythInt4Zero
ythInt4 => ythInt
ythYearMinus => ythMonth1
ythMonth1 => ythMonth2
ythMonthMinus => ythDay1
ythMonthMinusNoYmd => ythDay1NoYmd
ythDay1 => ythDay2
ythDay1NoYmd => ythDay2NoYmd
[ythAfterDaySpace, ythAfterDayT] => ythHour1
ythHour1 => ythHour2
ythHourColon => ythMinute1
ythMinute1 => ythMinute2
ythMinuteColon => ythSecond1
ythSecond1 => ythSecond2
ythAfterTimePlusMinus => ythTzHour1
ythTzHour1 => ythTzHour2
ythTzHourColon => ythTzMinute1
ythTzMinute1 => ythTzMinute2
[ythInt, ythDecimal, ythExponent, ythFraction] => nil
of '1'..'9':
[ythInitial, ythMinus] => ythInt
ythInitial => ythInt1
ythInt1 => ythInt2
ythInt1Zero => ythInt2Zero
ythInt2 => ythInt3
ythInt2Zero => ythInt3Zero
ythInt3 => ythInt4
ythInt3Zero => ythInt4Zero
[ythInt4, ythMinus] => ythInt
[ythNumE, ythNumEPlusMinus] => ythExponent
[ythInt, ythDecimal, ythExponent] => nil
ythYearMinus => ythMonth1
ythMonth1 => ythMonth2
ythMonthMinus => ythDay1
ythMonthMinusNoYmd => ythDay1NoYmd
ythDay1 => ythDay2
ythDay1NoYmd => ythDay2NoYmd
[ythAfterDaySpace, ythAfterDayT] => ythHour1
ythHour1 => ythHour2
ythHourColon => ythMinute1
ythMinute1 => ythMinute2
ythMinuteColon => ythSecond1
ythSecond1 => ythSecond2
ythAfterTimePlusMinus => ythTzHour1
ythTzHour1 => ythTzHour2
ythTzHourColon => ythTzMinute1
ythTzMinute1 => ythTzMinute2
[ythInt, ythDecimal, ythExponent, ythFraction] => nil
of 'a':
ythF => ythLowerFA
ythPointN => ythPointNA
@ -174,7 +242,9 @@ template advanceTypeHint(ch: char) {.dirty.} =
of 'S':
ythFAL => ythFALS
ythYE => ythYES
of 't', 'T': ythInitial => ythT
of 't', 'T':
ythInitial => ythT
[ythDay1, ythDay2, ythDay1NoYmd, ythDay2NoYmd] => ythAfterDayT
of 'u':
ythN => ythLowerNU
ythLowerTR => ythLowerTRU
@ -182,6 +252,11 @@ template advanceTypeHint(ch: char) {.dirty.} =
ythN => ythNU
ythTR => ythTRU
of 'y', 'Y': ythInitial => ythY
of 'Z': [ythSecond2, ythFraction, ythAfterTimeSpace] => ythAfterTimeZ
of ' ', '\t':
[ythSecond2, ythFraction] => ythAfterTimeSpace
[ythDay1, ythDay2, ythDay1NoYmd, ythDay2NoYmd] => ythAfterDaySpace
[ythAfterTimeSpace, ythAfterDaySpace] => nil
proc guessType*(scalar: string): TypeHint {.raises: [].} =
## Parse scalar string according to the RegEx table documented at
@ -192,8 +267,10 @@ proc guessType*(scalar: string): TypeHint {.raises: [].} =
of ythNULL: result = yTypeNull
of ythTRUE, ythON, ythYES, ythY: result = yTypeBoolTrue
of ythFALSE, ythOFF, ythNO, ythN: result = yTypeBoolFalse
of ythInt, yth0: result = yTypeInteger
of ythInt1, ythInt2, ythInt3, ythInt4, ythInt, yth0: result = yTypeInteger
of ythDecimal, ythExponent: result = yTypeFloat
of ythPointINF: result = yTypeFloatInf
of ythPointNAN: result = yTypeFloatNaN
of ythDay2, ythSecond2, ythFraction, ythAfterTimeZ, ythTzHour1, ythTzHour2,
ythTzMinute1, ythTzMinute2: result = yTypeTimestamp
else: result = yTypeUnknown

21
yaml/presenter.nim
View File

@ -413,12 +413,10 @@ proc writeTagAndAnchor(target: PresenterTarget, tag: TagId,
try:
if tag notin [yTagQuestionMark, yTagExclamationMark]:
let tagUri = tagLib.uri(tag)
if tagUri.startsWith(tagLib.secondaryPrefix):
target.append("!!")
target.append(tagUri[18..tagUri.high])
target.append(' ')
elif tagUri.startsWith("!"):
target.append(tagUri)
let (handle, length) = tagLib.searchHandle(tagUri)
if length > 0:
target.append(handle)
target.append(tagUri[length..tagUri.high])
target.append(' ')
else:
target.append("!<")
@ -461,8 +459,15 @@ proc doPresent(s: var YamlStream, target: PresenterTarget,
of ov1_2: target.append("%YAML 1.2" & newline)
of ov1_1: target.append("%YAML 1.1" & newLine)
of ovNone: discard
if tagLib.secondaryPrefix != yamlTagRepositoryPrefix:
target.append("%TAG !! " & tagLib.secondaryPrefix & newline)
for prefix, handle in tagLib.handles():
if handle == "!":
if prefix != "!":
target.append("%TAG ! " & prefix & newline)
elif handle == "!!":
if prefix != yamlTagRepositoryPrefix:
target.append("%TAG !! " & prefix & newline)
else:
target.append("%TAG " & handle & ' ' & prefix & newline)
target.append("--- ")
except:
var e = newException(YamlPresenterOutputError, "")

518
yaml/serialization.nim
View File

@ -16,8 +16,11 @@
## type. Please consult the serialization guide on the NimYAML website for more
## information.
import tables, typetraits, strutils, macros, streams
import tables, typetraits, strutils, macros, streams, times
import parser, taglib, presenter, stream, ../private/internal, hints
export stream
# *something* in here needs externally visible `==`(x,y: AnchorId),
# but I cannot figure out what. binding it would be the better option.
type
SerializationContext* = ref object
@ -101,9 +104,17 @@ proc lazyLoadTag(uri: string): TagId {.inline, raises: [].} =
proc safeTagUri(id: TagId): string {.raises: [].} =
try:
let uri = serializationTagLibrary.uri(id)
if uri.len > 0 and uri[0] == '!': return uri[1..uri.len - 1]
else: return uri
var
uri = serializationTagLibrary.uri(id)
i = 0
# '!' is not allowed inside a tag handle
if uri.len > 0 and uri[0] == '!': uri = uri[1..^1]
# ',' is not allowed after a tag handle in the suffix because it's a flow
# indicator
for c in uri.mitems():
if c == ',': c = ';'
inc(i)
return uri
except KeyError: internalError("Unexpected KeyError for TagId " & $id)
proc constructionError(s: YamlStream, msg: string): ref YamlConstructionError =
@ -310,12 +321,78 @@ proc representObject*(value: char, ts: TagStyle, c: SerializationContext,
## represents a char value as YAML scalar
c.put(scalarEvent("" & value, tag, yAnchorNone))
proc yamlTag*(T: typedesc[Time]): TagId {.inline, raises: [].} = yTagTimestamp
proc constructObject*(s: var YamlStream, c: ConstructionContext,
result: var Time)
{.raises: [YamlConstructionError, YamlStreamError].} =
constructScalarItem(s, item, Time):
if guessType(item.scalarContent) == yTypeTimestamp:
var
tmp = newStringOfCap(60)
pos = 8
c: char
while pos < item.scalarContent.len():
c = item.scalarContent[pos]
if c in {' ', '\t', 'T', 't'}: break
inc(pos)
if pos == item.scalarContent.len():
tmp.add(item.scalarContent)
tmp.add("T00:00:00+00:00")
else:
tmp.add(item.scalarContent[0 .. pos - 1])
if c in {' ', '\t'}:
while true:
inc(pos)
c = item.scalarContent[pos]
if c notin {' ', '\t'}: break
else: inc(pos)
tmp.add("T")
let timeStart = pos
inc(pos, 7)
var fractionStart = -1
while pos < item.scalarContent.len():
c = item.scalarContent[pos]
if c in {'+', '-', 'Z', ' ', '\t'}: break
elif c == '.': fractionStart = pos
inc(pos)
if fractionStart == -1:
tmp.add(item.scalarContent[timeStart .. pos - 1])
else:
tmp.add(item.scalarContent[timeStart .. fractionStart - 1])
if c in {'Z', ' ', '\t'}: tmp.add("+00:00")
else:
tmp.add(c)
inc(pos)
let tzStart = pos
inc(pos)
if pos < item.scalarContent.len() and item.scalarContent[pos] != ':':
inc(pos)
if pos - tzStart == 1: tmp.add('0')
tmp.add(item.scalarContent[tzStart .. pos - 1])
if pos == item.scalarContent.len(): tmp.add(":00")
elif pos + 2 == item.scalarContent.len():
tmp.add(":0")
tmp.add(item.scalarContent[pos + 1])
else:
tmp.add(item.scalarContent[pos .. pos + 2])
let info = tmp.parse("yyyy-M-d'T'H-mm-sszzz")
result = info.toTime()
else:
raise s.constructionError("Not a parsable timestamp: " &
escape(item.scalarContent))
proc representObject*(value: Time, ts: TagStyle, c: SerializationContext,
tag: TagId) {.raises: [ValueError].} =
let tmp = value.getGMTime()
c.put(scalarEvent(tmp.format("yyyy-MM-dd'T'HH:mm:ss'Z'")))
proc yamlTag*[I](T: typedesc[seq[I]]): TagId {.inline, raises: [].} =
let uri = "!nim:system:seq(" & safeTagUri(yamlTag(I)) & ')'
let uri = nimTag("system:seq(" & safeTagUri(yamlTag(I)) & ')')
result = lazyLoadTag(uri)
proc yamlTag*[I](T: typedesc[set[I]]): TagId {.inline, raises: [].} =
let uri = "!nim:system:set(" & safeTagUri(yamlTag(I)) & ')'
let uri = nimTag("system:set(" & safeTagUri(yamlTag(I)) & ')')
result = lazyLoadTag(uri)
proc constructObject*[T](s: var YamlStream, c: ConstructionContext,
@ -357,8 +434,8 @@ proc representObject*[T](value: seq[T]|set[T], ts: TagStyle,
proc yamlTag*[I, V](T: typedesc[array[I, V]]): TagId {.inline, raises: [].} =
const rangeName = name(I)
let uri = "!nim:system:array(" & rangeName[6..rangeName.high()] & "," &
safeTagUri(yamlTag(V)) & ')'
let uri = nimTag("system:array(" & rangeName[6..rangeName.high()] & ';' &
safeTagUri(yamlTag(V)) & ')')
result = lazyLoadTag(uri)
proc constructObject*[I, T](s: var YamlStream, c: ConstructionContext,
@ -388,8 +465,8 @@ proc representObject*[I, T](value: array[I, T], ts: TagStyle,
proc yamlTag*[K, V](T: typedesc[Table[K, V]]): TagId {.inline, raises: [].} =
try:
let uri = "!nim:tables:Table(" & safeTagUri(yamlTag(K)) & "," &
safeTagUri(yamlTag(V)) & ")"
let uri = nimTag("tables:Table(" & safeTagUri(yamlTag(K)) & ';' &
safeTagUri(yamlTag(V)) & ")")
result = lazyLoadTag(uri)
except KeyError:
# cannot happen (theoretically, you know)
@ -427,8 +504,8 @@ proc representObject*[K, V](value: Table[K, V], ts: TagStyle,
proc yamlTag*[K, V](T: typedesc[OrderedTable[K, V]]): TagId
{.inline, raises: [].} =
try:
let uri = "!nim:tables:OrderedTable(" & safeTagUri(yamlTag(K)) & "," &
safeTagUri(yamlTag(V)) & ")"
let uri = nimTag("tables:OrderedTable(" & safeTagUri(yamlTag(K)) & ';' &
safeTagUri(yamlTag(V)) & ")")
result = lazyLoadTag(uri)
except KeyError:
# cannot happen (theoretically, you know)
@ -472,7 +549,7 @@ proc representObject*[K, V](value: OrderedTable[K, V], ts: TagStyle,
proc yamlTag*(T: typedesc[object|enum]):
TagId {.inline, raises: [].} =
var uri = "!nim:custom:" & (typetraits.name(type(T)))
var uri = nimTag("custom:" & (typetraits.name(type(T))))
try: serializationTagLibrary.tags[uri]
except KeyError: serializationTagLibrary.registerUri(uri)
@ -480,7 +557,7 @@ proc yamlTag*(T: typedesc[tuple]):
TagId {.inline, raises: [].} =
var
i: T
uri = "!nim:tuple("
uri = nimTag("tuple(")
first = true
for name, value in fieldPairs(i):
if first: first = false
@ -502,7 +579,15 @@ proc fieldCount(t: typedesc): int {.compileTime.} =
inc(result)
if child.kind == nnkRecCase:
for bIndex in 1..<len(child):
inc(result, child[bIndex][1].len)
var recListIndex = 0
case child[bIndex].kind
of nnkOfBranch:
while child[bIndex][recListIndex].kind == nnkIntLit:
inc(recListIndex)
of nnkElse: discard
else: internalError("Unexpected child kind: " & $child[bIndex].kind)
if child[bIndex].len > recListIndex:
inc(result, child[bIndex][recListIndex].len)
macro matchMatrix(t: typedesc): untyped =
result = newNimNode(nnkBracket)
@ -517,20 +602,69 @@ proc checkDuplicate(s: NimNode, tName: string, name: string, i: int,
newLit("While constructing " & tName & ": Duplicate field: " &
escape(name))))))
proc checkMissing(s: NimNode, tName: string, name: string, i: int,
matched: NimNode): NimNode {.compileTime.} =
let
implicitVariantObjectMarker {.compileTime.} =
newIdentNode(":implicitVariantObject")
transientBitvectorProc {.compileTime.} =
newIdentNode(":transientObject")
defaultBitvectorProc {.compileTime.} =
newIdentNode(":defaultBitvector")
defaultValueGetter {.compileTime.} =
newIdentNode(":defaultValueGetter")
ignoredKeyListProc {.compileTime.} =
newIdentNode(":ignoredKeyList")
var
transientVectors {.compileTime.} = newSeq[set[int16]]()
defaultVectors {.compileTime.} = newSeq[set[int16]]()
ignoredKeyLists {.compileTime.} = newSeq[seq[string]]()
proc addDefaultOr(tName: string, i: int, o: NimNode,
field, elseBranch, defaultValues: NimNode): NimNode {.compileTime.} =
let
dbp = defaultBitvectorProc
t = newIdentNode(tName)
result = quote do:
when compiles(`dbp`(`t`)):
when `i` in defaultVectors[`dbp`(`t`)]:
`o`.`field` = `defaultValues`.`field`
else: `elseBranch`
else: `elseBranch`
proc checkMissing(s: NimNode, t: typedesc, tName: string, field: NimNode,
i: int, matched, o, defaultValues: NimNode):
NimNode {.compileTime.} =
result = newIfStmt((newCall("not", newNimNode(nnkBracketExpr).add(matched,
newLit(i))), newNimNode(nnkRaiseStmt).add(newCall(
newLit(i))), addDefaultOr(tName, i, o, field,
newNimNode(nnkRaiseStmt).add(newCall(
bindSym("constructionError"), s, newLit("While constructing " &
tName & ": Missing field: " & escape(name))))))
tName & ": Missing field: " & escape($field)))), defaultValues)))
proc markAsFound(i: int, matched: NimNode): NimNode {.compileTime.} =
newAssignment(newNimNode(nnkBracketExpr).add(matched, newLit(i)),
newLit(true))
macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, o: typed,
proc ifNotTransient(tSym: NimNode, fieldIndex: int, content: openarray[NimNode],
elseError: bool = false, s: NimNode = nil, tName, fName: string = nil):
NimNode {.compileTime.} =
var stmts = newStmtList(content)
result = quote do:
when `tSym` == -1 or `fieldIndex` notin transientVectors[`tSym`]:
`stmts`
if elseError:
result[0].add(newNimNode(nnkElse).add(quote do:
raise constructionError(`s`, "While constructing " & `tName` &
": Field \"" & `fName` & "\" is transient and may not occur in input")
))
macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, tIndex: int, o: typed,
matched: typed): typed =
result = newStmtList()
let
dbp = defaultBitvectorProc
defaultValues = genSym(nskConst, "defaultValues")
result = quote do:
when compiles(`dbp`(`t`)):
const `defaultValues` = `defaultValueGetter`(`t`)
let
tDecl = getType(t)
tName = $tDecl[1]
@ -538,25 +672,49 @@ macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, o: typed,
var field = 0
for child in tDesc[2].children:
if child.kind == nnkRecCase:
result.add(checkMissing(s, tName, $child[0], field, matched))
result.add(checkMissing(s, t, tName, child[0], field, matched, o,
defaultValues))
for bIndex in 1 .. len(child) - 1:
let discChecks = newStmtList()
for item in child[bIndex][1].children:
var
curValues = newNimNode(nnkCurly)
recListIndex = 0
case child[bIndex].kind
of nnkOfBranch:
while recListIndex < child[bIndex].len and
child[bIndex][recListIndex].kind == nnkIntLit:
curValues.add(child[bIndex][recListIndex])
inc(recListIndex)
of nnkElse: discard
else: internalError("Unexpected child kind: " & $child[bIndex].kind)
for item in child[bIndex][recListIndex].children:
inc(field)
discChecks.add(checkMissing(s, tName, $item, field, matched))
result.add(newIfStmt((infix(newDotExpr(o, newIdentNode($child[0])),
"==", child[bIndex][0]), discChecks)))
discChecks.add(checkMissing(s, t, tName, item, field, matched, o,
defaultValues))
result.add(ifNotTransient(tIndex, field,
[newIfStmt((infix(newDotExpr(o, newIdentNode($child[0])),
"in", curValues), discChecks))]))
else:
result.add(checkMissing(s, tName, $child, field, matched))
result.add(ifNotTransient(tIndex, field,
[checkMissing(s, t, tName, child, field, matched, o, defaultValues)]))
inc(field)
macro constructFieldValue(t: typedesc, stream: untyped, context: untyped,
name: untyped, o: untyped, matched: untyped): typed =
macro fetchTransientIndex(t: typedesc, tIndex: untyped): typed =
quote do:
when compiles(`transientBitvectorProc`(`t`)):
const `tIndex` = `transientBitvectorProc`(`t`)
else:
const `tIndex` = -1
macro constructFieldValue(t: typedesc, tIndex: int, stream: untyped,
context: untyped, name: untyped, o: untyped,
matched: untyped): typed =
let
tDecl = getType(t)
tName = $tDecl[1]
tDesc = getType(tDecl[1])
result = newNimNode(nnkCaseStmt).add(name)
result = newStmtList()
var caseStmt = newNimNode(nnkCaseStmt).add(name)
var fieldIndex = 0
for child in tDesc[2].children:
if child.kind == nnkRecCase:
@ -572,10 +730,26 @@ macro constructFieldValue(t: typedesc, stream: untyped, context: untyped,
newCall("constructChild", stream, context, newIdentNode("value")),
newAssignment(discriminant, newIdentNode("value")),
markAsFound(fieldIndex, matched)))
result.add(disOb)
caseStmt.add(disOb)
var alreadyUsedSet = newNimNode(nnkCurly)
for bIndex in 1 .. len(child) - 1:
let discTest = infix(discriminant, "==", child[bIndex][0])
for item in child[bIndex][1].children:
var recListIndex = 0
var discTest: NimNode
case child[bIndex].kind
of nnkOfBranch:
discTest = newNimNode(nnkCurly)
while child[bIndex][recListIndex].kind == nnkIntLit:
discTest.add(child[bIndex][recListIndex])
alreadyUsedSet.add(child[bIndex][recListIndex])
inc(recListIndex)
discTest = infix(discriminant, "in", discTest)
of nnkElse:
discTest = infix(discriminant, "notin", alreadyUsedSet)
else:
internalError("Unexpected child kind: " & $child[bIndex].kind)
doAssert child[bIndex][recListIndex].kind == nnkRecList
for item in child[bIndex][recListIndex].children:
inc(fieldIndex)
yAssert item.kind == nnkSym
var ob = newNimNode(nnkOfBranch).add(newStrLitNode($item))
@ -586,31 +760,43 @@ macro constructFieldValue(t: typedesc, stream: untyped, context: untyped,
newCall(bindSym("constructionError"), stream,
infix(newStrLitNode("Field " & $item & " not allowed for " &
$child[0] & " == "), "&", prefix(discriminant, "$"))))))
ob.add(newStmtList(checkDuplicate(stream, tName, $item, fieldIndex,
matched), ifStmt, markAsFound(fieldIndex, matched)))
result.add(ob)
ob.add(ifNotTransient(tIndex, fieldIndex,
[checkDuplicate(stream, tName, $item, fieldIndex, matched),
ifStmt, markAsFound(fieldIndex, matched)], true, stream, tName,
$item))
caseStmt.add(ob)
else:
yAssert child.kind == nnkSym
var ob = newNimNode(nnkOfBranch).add(newStrLitNode($child))
let field = newDotExpr(o, newIdentNode($child))
ob.add(newStmtList(
checkDuplicate(stream, tName, $child, fieldIndex, matched),
ob.add(ifNotTransient(tIndex, fieldIndex,
[checkDuplicate(stream, tName, $child, fieldIndex, matched),
newCall("constructChild", stream, context, field),
markAsFound(fieldIndex, matched)))
result.add(ob)
markAsFound(fieldIndex, matched)], true, stream, tName, $child))
caseStmt.add(ob)
inc(fieldIndex)
result.add(newNimNode(nnkElse).add(newNimNode(nnkRaiseStmt).add(
caseStmt.add(newNimNode(nnkElse).add(newNimNode(nnkRaiseStmt).add(
newCall(bindSym("constructionError"), stream,
infix(newLit("While constructing " & tName & ": Unknown field: "), "&",
newCall(bindSym("escape"), name))))))
result.add(caseStmt)
proc isVariantObject(t: typedesc): bool {.compileTime.} =
let tDesc = getType(t)
if tDesc.kind != nnkObjectTy: return false
var tDesc = getType(t)
if tDesc.kind == nnkBracketExpr: tDesc = getType(tDesc[1])
if tDesc.kind != nnkObjectTy:
return false
for child in tDesc[2].children:
if child.kind == nnkRecCase: return true
return false
macro injectIgnoredKeyList(t: typedesc, ident: untyped): typed =
result = quote do:
when compiles(`ignoredKeyListProc`(`t`)):
const `ident` = ignoredKeyLists[`ignoredKeyListproc`(`t`)]
else:
const `ident` = newSeq[string]()
proc constructObject*[O: object|tuple](
s: var YamlStream, c: ConstructionContext, result: var O)
{.raises: [YamlConstructionError, YamlStreamError].} =
@ -620,13 +806,13 @@ proc constructObject*[O: object|tuple](
const
startKind = when isVariantObject(O): yamlStartSeq else: yamlStartMap
endKind = when isVariantObject(O): yamlEndSeq else: yamlEndMap
fetchTransientIndex(O, tIndex)
if e.kind != startKind:
raise s.constructionError("While constructing " &
typetraits.name(O) & ": Expected " & $startKind & ", got " & $e.kind)
when isVariantObject(O): reset(result) # make discriminants writeable
injectIgnoredKeyList(O, ignoredKeyList)
while s.peek.kind != endKind:
# todo: check for duplicates in input and raise appropriate exception
# also todo: check for missing items and raise appropriate exception
e = s.next()
when isVariantObject(O):
if e.kind != yamlStartMap:
@ -652,7 +838,17 @@ proc constructObject*[O: object|tuple](
raise s.constructionError("While constructing " &
typetraits.name(O) & ": Unknown field: " & escape(name))
else:
constructFieldValue(O, s, c, name, result, matched)
if name notin ignoredKeyList:
constructFieldValue(O, tIndex, s, c, name, result, matched)
else:
e = s.next()
var depth = int(e.kind in {yamlStartMap, yamlStartSeq})
while depth > 0:
case s.next().kind
of yamlStartMap, yamlStartSeq: inc(depth)
of yamlEndMap, yamlEndSeq: dec(depth)
of yamlScalar: discard
else: internalError("Unexpected event kind.")
when isVariantObject(O):
e = s.next()
if e.kind != yamlEndMap:
@ -666,23 +862,106 @@ proc constructObject*[O: object|tuple](
raise s.constructionError("While constructing " &
typetraits.name(O) & ": Missing field: " & escape(fname))
inc(i)
else: ensureAllFieldsPresent(s, O, result, matched)
else: ensureAllFieldsPresent(s, O, tIndex, result, matched)
proc representObject*[O: object|tuple](value: O, ts: TagStyle,
macro genRepresentObject(t: typedesc, value, childTagStyle: typed): typed =
result = newStmtList()
let tSym = genSym(nskConst, ":tSym")
result.add(quote do:
when compiles(`transientBitvectorProc`(`t`)):
const `tSym` = `transientBitvectorProc`(`t`)
else:
const `tSym` = -1
)
let
tDecl = getType(t)
tDesc = getType(tDecl[1])
isVO = isVariantObject(t)
var fieldIndex = 0'i16
for child in tDesc[2].children:
if child.kind == nnkRecCase:
let
fieldName = $child[0]
fieldAccessor = newDotExpr(value, newIdentNode(fieldName))
result.add(quote do:
c.put(startMapEvent(yTagQuestionMark, yAnchorNone))
c.put(scalarEvent(`fieldName`, if `childTagStyle` == tsNone:
yTagQuestionMark else: yTagNimField, yAnchorNone))
representChild(`fieldAccessor`, `childTagStyle`, c)
c.put(endMapEvent())
)
let enumName = $getTypeInst(child[0])
var caseStmt = newNimNode(nnkCaseStmt).add(fieldAccessor)
for bIndex in 1 .. len(child) - 1:
var curBranch: NimNode
var recListIndex = 0
case child[bIndex].kind
of nnkOfBranch:
curBranch = newNimNode(nnkOfBranch)
while child[bIndex][recListIndex].kind == nnkIntLit:
curBranch.add(newCall(enumName, newLit(child[bIndex][recListIndex].intVal)))
inc(recListIndex)
of nnkElse:
curBranch = newNimNode(nnkElse)
else:
internalError("Unexpected child kind: " & $child[bIndex].kind)
doAssert child[bIndex][recListIndex].kind == nnkRecList
var curStmtList = newStmtList()
if child[bIndex][recListIndex].len > 0:
for item in child[bIndex][recListIndex].children:
inc(fieldIndex)
let
name = $item
itemAccessor = newDotExpr(value, newIdentNode(name))
curStmtList.add(quote do:
when `tSym` == -1 or `fieldIndex` notin transientVectors[`tSym`]:
c.put(startMapEvent(yTagQuestionMark, yAnchorNone))
c.put(scalarEvent(`name`, if `childTagStyle` == tsNone:
yTagQuestionMark else: yTagNimField, yAnchorNone))
representChild(`itemAccessor`, `childTagStyle`, c)
c.put(endMapEvent())
)
else:
curStmtList.add(newNimNode(nnkDiscardStmt).add(newEmptyNode()))
curBranch.add(curStmtList)
caseStmt.add(curBranch)
result.add(caseStmt)
else:
let
name = $child
childAccessor = newDotExpr(value, newIdentNode(name))
result.add(quote do:
when `tSym` == -1 or `fieldIndex` notin transientVectors[`tSym`]:
when `isVO`: c.put(startMapEvent(yTagQuestionMark, yAnchorNone))
c.put(scalarEvent(`name`, if `childTagStyle` == tsNone:
yTagQuestionMark else: yTagNimField, yAnchorNone))
representChild(`childAccessor`, `childTagStyle`, c)
when `isVO`: c.put(endMapEvent())
)
inc(fieldIndex)
proc representObject*[O: object](value: O, ts: TagStyle,
c: SerializationContext, tag: TagId) =
## represents a Nim object or tuple as YAML mapping
let childTagStyle = if ts == tsRootOnly: tsNone else: ts
when isVariantObject(O): c.put(startSeqEvent(tag, yAnchorNone))
else: c.put(startMapEvent(tag, yAnchorNone))
for name, value in fieldPairs(value):
when isVariantObject(O): c.put(startMapEvent(yTagQuestionMark, yAnchorNone))
c.put(scalarEvent(name, if childTagStyle == tsNone: yTagQuestionMark else:
yTagNimField, yAnchorNone))
representChild(value, childTagStyle, c)
when isVariantObject(O): c.put(endMapEvent())
genRepresentObject(O, value, childTagStyle)
when isVariantObject(O): c.put(endSeqEvent())
else: c.put(endMapEvent())
proc representObject*[O: tuple](value: O, ts: TagStyle,
c: SerializationContext, tag: TagId) =
let childTagStyle = if ts == tsRootOnly: tsNone else: ts
var fieldIndex = 0'i16
c.put(startMapEvent(tag, yAnchorNone))
for name, fvalue in fieldPairs(value):
c.put(scalarEvent(name, if childTagStyle == tsNone:
yTagQuestionMark else: yTagNimField, yAnchorNone))
representChild(fvalue, childTagStyle, c)
inc(fieldIndex)
c.put(endMapEvent())
proc constructObject*[O: enum](s: var YamlStream, c: ConstructionContext,
result: var O)
{.raises: [YamlConstructionError, YamlStreamError].} =
@ -738,15 +1017,18 @@ macro constructImplicitVariantObject(s, c, r, possibleTagIds: untyped,
))
ifStmt.add(newNimNode(nnkElse).add(newNimNode(nnkTryStmt).add(
newStmtList(raiseStmt), newNimNode(nnkExceptBranch).add(
newIdentNode("KeyError"), newStmtList(newCall("internalError",
newStrLitNode("Unexcpected KeyError")))
newIdentNode("KeyError"),
newNimNode(nnkDiscardStmt).add(newEmptyNode())
))))
result = newStmtList(newCall("reset", r), ifStmt)
macro isImplicitVariantObject(o: typed): untyped =
result = newCall("compiles", newCall(implicitVariantObjectMarker, o))
proc constructChild*[T](s: var YamlStream, c: ConstructionContext,
result: var T) =
let item = s.peek()
when compiles(implicitVariantObject(result)):
when isImplicitVariantObject(result):
var possibleTagIds = newSeq[TagId]()
case item.kind
of yamlScalar:
@ -768,6 +1050,8 @@ proc constructChild*[T](s: var YamlStream, c: ConstructionContext,
raise s.constructionError("not implemented!")
of yTypeUnknown:
possibleTagIds.add(yamlTag(string))
of yTypeTimestamp:
possibleTagIds.add(yamlTag(Time))
of yTagExclamationMark:
possibleTagIds.add(yamlTag(string))
else:
@ -915,7 +1199,7 @@ proc representChild*[O](value: ref O, ts: TagStyle, c: SerializationContext) =
proc representChild*[O](value: O, ts: TagStyle,
c: SerializationContext) =
when compiles(implicitVariantObject(value)):
when isImplicitVariantObject(value):
# todo: this would probably be nicer if constructed with a macro
var count = 0
for name, field in fieldPairs(value):
@ -1032,3 +1316,121 @@ proc dump*[K](value: K, tagStyle: TagStyle = tsRootOnly,
try: result = present(events, serializationTagLibrary, options)
except YamlStreamError:
internalError("Unexpected exception: " & getCurrentException().repr)
proc canBeImplicit(t: typedesc): bool {.compileTime.} =
let tDesc = getType(t)
if tDesc.kind != nnkObjectTy: return false
if tDesc[2].len != 1: return false
if tDesc[2][0].kind != nnkRecCase: return false
var foundEmptyBranch = false
for i in 1.. tDesc[2][0].len - 1:
case tDesc[2][0][i][1].len # branch contents
of 0:
if foundEmptyBranch: return false
else: foundEmptyBranch = true
of 1: discard
else: return false
return true
macro setImplicitVariantObjectMarker(t: typedesc): untyped =
result = quote do:
when compiles(`transientBitvectorProc`(`t`)):
{.fatal: "Cannot mark object with transient fields as implicit".}
proc `implicitVariantObjectMarker`*(unused: `t`) = discard
template markAsImplicit*(t: typedesc): typed =
## Mark a variant object type as implicit. This requires the type to consist
## of nothing but a case expression and each branch of the case expression
## containing exactly one field - with the exception that one branch may
## contain zero fields.
when canBeImplicit(t):
# this will be checked by means of compiles(implicitVariantObject(...))
setImplicitVariantObjectMarker(t)
else:
{. fatal: "This type cannot be marked as implicit" .}
macro getFieldIndex(t: typedesc, field: untyped): untyped =
let
tDecl = getType(t)
tName = $tDecl[1]
tDesc = getType(tDecl[1])
fieldName = $field
var
fieldIndex = 0
found = false
block outer:
for child in tDesc[2].children:
if child.kind == nnkRecCase:
for bIndex in 1 .. len(child) - 1:
var bChildIndex = 0
case child[bIndex].kind
of nnkOfBranch:
while child[bIndex][bChildIndex].kind == nnkIntLit: inc(bChildIndex)
of nnkElse: discard
else:
internalError("Unexpected child kind: " &
$child[bIndex][bChildIndex].kind)
yAssert child[bIndex][bChildIndex].kind == nnkRecList
for item in child[bIndex][bChildIndex].children:
inc(fieldIndex)
yAssert item.kind == nnkSym
if $item == fieldName:
found = true
break outer
else:
yAssert child.kind == nnkSym
if $child == fieldName:
found = true
break
inc(fieldIndex)
if not found:
let msg = "Type " & tName & " has no field " & fieldName & '!'
result = quote do: {.fatal: `msg`.}
else:
result = newNimNode(nnkInt16Lit)
result.intVal = fieldIndex
macro markAsTransient*(t: typedesc, field: untyped): typed =
let nextBitvectorIndex = transientVectors.len
result = quote do:
when compiles(`implicitVariantObjectMarker`(`t`)):
{.fatal: "Cannot mark fields of implicit variant objects as transient." .}
when not compiles(`transientBitvectorProc`(`t`)):
proc `transientBitvectorProc`*(myType: typedesc[`t`]): int
{.compileTime.} =
`nextBitvectorIndex`
static: transientVectors.add({})
static:
transientVectors[`transientBitvectorProc`(`t`)].incl(getFieldIndex(`t`, `field`))
macro setDefaultValue*(t: typedesc, field: untyped, value: typed): typed =
let
dSym = genSym(nskVar, ":default")
nextBitvectorIndex = defaultVectors.len
result = quote do:
when compiles(`transientBitvectorProc`(`t`)):
{.fatal: "Cannot set default value of transient field".}
elif compiles(`implicitVariantObjectMarker`(`t`)):
{.fatal: "Cannot set default value of implicit variant objects.".}
when not compiles(`defaultValueGetter`(`t`)):
var `dSym` {.compileTime.}: `t`
template `defaultValueGetter`(t: typedesc[`t`]): auto =
`dSym`
proc `defaultBitvectorProc`*(myType: typedesc[`t`]): int
{.compileTime.} = `nextBitvectorIndex`
static: defaultVectors.add({})
static:
`defaultValueGetter`(`t`).`field` = `value`
defaultVectors[`defaultBitvectorProc`(`t`)].incl(getFieldIndex(`t`, `field`))
macro ignoreInputKey*(t: typedesc, name: string{lit}): typed =
let nextIgnoredKeyList = ignoredKeyLists.len
result = quote do:
when not compiles(`ignoredKeyListProc`(`t`)):
proc `ignoredKeyListProc`*(t: typedesc[`t`]): int {.compileTime.} =
`nextIgnoredKeyList`
static: ignoredKeyLists.add(@[])
when `name` in ignoredKeyLists[`ignoredKeyListProc`(`t`)]:
{.fatal: "Input key " & `name` & " is already ignored!".}
static:
ignoredKeyLists[`ignoredKeyListProc`(`t`)].add(`name`)

166
yaml/taglib.nim
View File

@ -12,7 +12,7 @@
## and create own tags. It also enables you to define tags for types used with
## the serialization API.
import tables, macros, hashes
import tables, macros, hashes, strutils
type
TagId* = distinct int ## \
@ -41,7 +41,7 @@ type
## `initExtendedTagLibrary <#initExtendedTagLibrary>`_.
tags*: Table[string, TagId]
nextCustomTagId*: TagId
secondaryPrefix*: string
tagHandles: Table[string, string]
const
# failsafe schema
@ -94,11 +94,15 @@ const
## for seqs that are nil. This tag is used regardless of the seq's generic
## type parameter.
yFirstCustomTagId* : TagId = 1000.TagId ## \
yFirstStaticTagId* : TagId = 1000.TagId ## \
## The first ``TagId`` assigned by the ``setTagId`` templates.
yFirstCustomTagId* : TagId = 10000.TagId ## \
## The first ``TagId`` which should be assigned to an URI that does not
## exist in the ``YamlTagLibrary`` which is used for parsing.
yamlTagRepositoryPrefix* = "tag:yaml.org,2002:"
nimyamlTagRepositoryPrefix* = "tag:nimyaml.org,2016:"
proc `==`*(left, right: TagId): bool {.borrow.}
proc hash*(id: TagId): Hash {.borrow.}
@ -130,7 +134,7 @@ proc initTagLibrary*(): TagLibrary {.raises: [].} =
## ``yFirstCustomTagId``.
new(result)
result.tags = initTable[string, TagId]()
result.secondaryPrefix = yamlTagRepositoryPrefix
result.tagHandles = {"!": "!", yamlTagRepositoryPrefix : "!!"}.toTable()
result.nextCustomTagId = yFirstCustomTagId
proc registerUri*(tagLib: TagLibrary, uri: string): TagId {.raises: [].} =
@ -146,6 +150,9 @@ proc uri*(tagLib: TagLibrary, id: TagId): string {.raises: [KeyError].} =
if iId == id: return iUri
raise newException(KeyError, "Unknown tag id: " & $id)
template y(suffix: string): string = yamlTagRepositoryPrefix & suffix
template n(suffix: string): string = nimyamlTagRepositoryPrefix & suffix
proc initFailsafeTagLibrary*(): TagLibrary {.raises: [].} =
## Contains only:
## - ``!``
@ -156,9 +163,9 @@ proc initFailsafeTagLibrary*(): TagLibrary {.raises: [].} =
result = initTagLibrary()
result.tags["!"] = yTagExclamationMark
result.tags["?"] = yTagQuestionMark
result.tags["tag:yaml.org,2002:str"] = yTagString
result.tags["tag:yaml.org,2002:seq"] = yTagSequence
result.tags["tag:yaml.org,2002:map"] = yTagMapping
result.tags[y"str"] = yTagString
result.tags[y"seq"] = yTagSequence
result.tags[y"map"] = yTagMapping
proc initCoreTagLibrary*(): TagLibrary {.raises: [].} =
## Contains everything in ``initFailsafeTagLibrary`` plus:
@ -167,10 +174,10 @@ proc initCoreTagLibrary*(): TagLibrary {.raises: [].} =
## - ``!!int``
## - ``!!float``
result = initFailsafeTagLibrary()
result.tags["tag:yaml.org,2002:null"] = yTagNull
result.tags["tag:yaml.org,2002:bool"] = yTagBoolean
result.tags["tag:yaml.org,2002:int"] = yTagInteger
result.tags["tag:yaml.org,2002:float"] = yTagFloat
result.tags[y"null"] = yTagNull
result.tags[y"bool"] = yTagBoolean
result.tags[y"int"] = yTagInteger
result.tags[y"float"] = yTagFloat
proc initExtendedTagLibrary*(): TagLibrary {.raises: [].} =
## Contains everything from ``initCoreTagLibrary`` plus:
@ -183,29 +190,29 @@ proc initExtendedTagLibrary*(): TagLibrary {.raises: [].} =
## - ``!!value``
## - ``!!yaml``
result = initCoreTagLibrary()
result.tags["tag:yaml.org,2002:omap"] = yTagOrderedMap
result.tags["tag:yaml.org,2002:pairs"] = yTagPairs
result.tags["tag:yaml.org,2002:binary"] = yTagBinary
result.tags["tag:yaml.org,2002:merge"] = yTagMerge
result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp
result.tags["tag:yaml.org,2002:value"] = yTagValue
result.tags["tag:yaml.org,2002:yaml"] = yTagYaml
result.tags[y"omap"] = yTagOrderedMap
result.tags[y"pairs"] = yTagPairs
result.tags[y"binary"] = yTagBinary
result.tags[y"merge"] = yTagMerge
result.tags[y"timestamp"] = yTagTimestamp
result.tags[y"value"] = yTagValue
result.tags[y"yaml"] = yTagYaml
proc initSerializationTagLibrary*(): TagLibrary =
result = initTagLibrary()
result.tagHandles[nimyamlTagRepositoryPrefix] = "!n!"
result.tags["!"] = yTagExclamationMark
result.tags["?"] = yTagQuestionMark
result.tags["tag:yaml.org,2002:str"] = yTagString
result.tags["tag:yaml.org,2002:null"] = yTagNull
result.tags["tag:yaml.org,2002:bool"] = yTagBoolean
result.tags["tag:yaml.org,2002:float"] = yTagFloat
result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp
result.tags["tag:yaml.org,2002:value"] = yTagValue
result.tags["tag:yaml.org,2002:binary"] = yTagBinary
result.tags["!nim:field"] = yTagNimField
result.tags["!nim:nil:string"] = yTagNimNilString
result.tags["!nim:nil:seq"] = yTagNimNilSeq
result.tags[y"str"] = yTagString
result.tags[y"null"] = yTagNull
result.tags[y"bool"] = yTagBoolean
result.tags[y"float"] = yTagFloat
result.tags[y"timestamp"] = yTagTimestamp
result.tags[y"value"] = yTagValue
result.tags[y"binary"] = yTagBinary
result.tags[n"field"] = yTagNimField
result.tags[n"nil:string"] = yTagNimNilString
result.tags[n"nil:seq"] = yTagNimNilSeq
var
serializationTagLibrary* = initSerializationTagLibrary() ## \
@ -217,7 +224,7 @@ var
## `serializable <#serializable,stmt,stmt>`_.
var
nextStaticTagId {.compileTime.} = 100.TagId ## \
nextStaticTagId {.compileTime.} = yFirstStaticTagId ## \
## used for generating unique TagIds with ``setTagUri``.
registeredUris {.compileTime.} = newSeq[string]() ## \
## Since Table doesn't really work at compile time, we also store
@ -249,61 +256,66 @@ template setTagUri*(t: typedesc, uri: string, idName: untyped): typed =
static:
registeredUris.add(uri)
nextStaticTagId = TagId(int(nextStaticTagId) + 1)
when nextStaticTagId == yFirstCustomTagId:
{.fatal: "Too many tags!".}
serializationTagLibrary.tags[uri] = idName
proc yamlTag*(T: typedesc[t]): TagId {.inline, raises: [].} = idName
## autogenerated
proc canBeImplicit(t: typedesc): bool {.compileTime.} =
let tDesc = getType(t)
if tDesc.kind != nnkObjectTy: return false
if tDesc[2].len != 1: return false
if tDesc[2][0].kind != nnkRecCase: return false
var foundEmptyBranch = false
for i in 1.. tDesc[2][0].len - 1:
case tDesc[2][0][i][1].len # branch contents
of 0:
if foundEmptyBranch: return false
else: foundEmptyBranch = true
of 1: discard
else: return false
return true
template markAsImplicit*(t: typedesc): typed =
## Mark a variant object type as implicit. This requires the type to consist
## of nothing but a case expression and each branch of the case expression
## containing exactly one field - with the exception that one branch may
## contain zero fields.
when canBeImplicit(t):
# this will be checked by means of compiles(implicitVariantObject(...))
proc implicitVariantObject*(unused: t) = discard
else:
{. fatal: "This type cannot be marked as implicit" .}
static:
# standard YAML tags used by serialization
registeredUris.add("!")
registeredUris.add("?")
registeredUris.add("tag:yaml.org,2002:str")
registeredUris.add("tag:yaml.org,2002:null")
registeredUris.add("tag:yaml.org,2002:bool")
registeredUris.add("tag:yaml.org,2002:float")
registeredUris.add("tag:yaml.org,2002:timestamp")
registeredUris.add("tag:yaml.org,2002:value")
registeredUris.add("tag:yaml.org,2002:binary")
registeredUris.add(y"str")
registeredUris.add(y"null")
registeredUris.add(y"bool")
registeredUris.add(y"float")
registeredUris.add(y"timestamp")
registeredUris.add(y"value")
registeredUris.add(y"binary")
# special tags used by serialization
registeredUris.add("!nim:field")
registeredUris.add("!nim:nil:string")
registeredUris.add("!nim:nil:seq")
registeredUris.add(n"field")
registeredUris.add(n"nil:string")
registeredUris.add(n"nil:seq")
# tags for Nim's standard types
setTagUri(char, "!nim:system:char", yTagNimChar)
setTagUri(int8, "!nim:system:int8", yTagNimInt8)
setTagUri(int16, "!nim:system:int16", yTagNimInt16)
setTagUri(int32, "!nim:system:int32", yTagNimInt32)
setTagUri(int64, "!nim:system:int64", yTagNimInt64)
setTagUri(uint8, "!nim:system:uint8", yTagNimUInt8)
setTagUri(uint16, "!nim:system:uint16", yTagNimUInt16)
setTagUri(uint32, "!nim:system:uint32", yTagNimUInt32)
setTagUri(uint64, "!nim:system:uint64", yTagNimUInt64)
setTagUri(float32, "!nim:system:float32", yTagNimFloat32)
setTagUri(float64, "!nim:system:float64", yTagNimFloat64)
setTagUri(char, n"system:char", yTagNimChar)
setTagUri(int8, n"system:int8", yTagNimInt8)
setTagUri(int16, n"system:int16", yTagNimInt16)
setTagUri(int32, n"system:int32", yTagNimInt32)
setTagUri(int64, n"system:int64", yTagNimInt64)
setTagUri(uint8, n"system:uint8", yTagNimUInt8)
setTagUri(uint16, n"system:uint16", yTagNimUInt16)
setTagUri(uint32, n"system:uint32", yTagNimUInt32)
setTagUri(uint64, n"system:uint64", yTagNimUInt64)
setTagUri(float32, n"system:float32", yTagNimFloat32)
setTagUri(float64, n"system:float64", yTagNimFloat64)
proc registerHandle*(tagLib: TagLibrary, handle, prefix: string) =
## Registers a handle for a prefix. When presenting any tag that starts with
## this prefix, the handle is used instead. Also causes the presenter to
## output a TAG directive for the handle.
taglib.tagHandles[prefix] = handle
proc searchHandle*(tagLib: TagLibrary, tag: string):
tuple[handle: string, len: int] {.raises: [].} =
## search in the registered tag handles for one whose prefix matches the start
## of the given tag. If multiple registered handles match, the one with the
## longest prefix is returned. If no registered handle matches, (nil, 0) is
## returned.
result.len = 0
for key, value in tagLib.tagHandles:
if key.len > result.len:
if tag.startsWith(key):
result.len = key.len
result.handle = value
iterator handles*(tagLib: TagLibrary): tuple[prefix, handle: string] =
## iterate over registered tag handles that may be used as shortcuts
## (e.g. ``!n!`` for ``tag:nimyaml.org,2016:``)
for key, value in tagLib.tagHandles: yield (key, value)
proc nimTag*(suffix: string): string =
## prepends NimYAML's tag repository prefix to the given suffix. For example,
## ``nimTag("system:char")`` yields ``"tag:nimyaml.org,2016:system:char"``.
nimyamlTagRepositoryPrefix & suffix