Merge branch 'devel' - release 0.7.0

.gitignore

@@ -1,9 +1,9 @@
 nimcache
 test/tests
-test/parsing
-test/lexing
-test/serializing
-test/constructingJson
+test/tlex
+test/tdom
+test/tserialization
+test/tjson
 test/yamlTestSuite
 test/*.exe
 test/*.pdb

.travis.yml

@@ -0,0 +1,48 @@
+# Copied from https://github.com/nim-lang/Nim/wiki/TravisCI
+language: c
+env:
+  # Build and test against the master and devel branches of Nim
+  #- BRANCH=master TODO! does not work currently, waiting for 0.15.0
+  - BRANCH=devel
+compiler:
+  # Build and test using both gcc and clang
+  - gcc
+  - clang
+matrix:
+  allow_failures:
+    # Ignore failures when building against the devel Nim branch
+    # - env: BRANCH=devel TODO! ignore those once stable Nim can build NimYAML
+  fast_finish: true
+install:
+  - |
+    if [ ! -x nim-$BRANCH/bin/nim ]; then
+      git clone -b $BRANCH --depth 1 git://github.com/nim-lang/nim nim-$BRANCH/
+      cd nim-$BRANCH
+      git clone -b $BRANCH --depth 1 git://github.com/nim-lang/csources csources/
+      cd csources
+      sh build.sh
+      cd ..
+      rm -rf csources
+      bin/nim c koch
+      ./koch boot -d:release
+    else
+      cd nim-$BRANCH
+      git fetch origin
+      if ! git merge FETCH_HEAD | grep "Already up-to-date"; then
+        bin/nim c koch
+        ./koch boot -d:release
+      fi
+    fi
+    cd ..
+before_script:
+    - export PATH="nim-$BRANCH/bin${PATH:+:$PATH}"
+script:
+    - nim tests --verbosity:0
+    - nim yamlTestSuite --verbosity:0
+cache:
+  directories:
+    - nim-master
+    - nim-devel
+branches:
+  except:
+    - gh-pages

README.md

@@ -1,5 +1,7 @@
 # NimYAML - YAML implementation for Nim
 
+[![Build Status](https://travis-ci.org/flyx/NimYAML.svg?branch=devel)](https://travis-ci.org/flyx/NimYAML)
+
 NimYAML is a pure Nim YAML implementation without any dependencies other than
 Nim's standard library. It enables you to serialize Nim objects to a YAML stream
 and back. It also provides a low-level event-based API, and a document object

bench/jsonBench.nim

@@ -1,15 +1,15 @@
-import "../yaml", common
+import "../yaml", commonBench
 
 from nimlets_yaml import objKind
 
-import math, strutils, stopwatch, terminal, algorithm
+import math, strutils, stopwatch, terminal, algorithm, random, json
 
-proc cmp(left, right: clock): int = cmp(left.nanoseconds(), right.nanoseconds()) 
+proc cmp(left, right: clock): int = cmp(left.nanoseconds(), right.nanoseconds())
 
 type
     ObjectKind = enum
         otMap, otSequence
-    
+
     Level = tuple
         kind: ObjectKind
         len: int
@@ -33,10 +33,10 @@ proc genString(maxLen: int): string =
 proc genJsonString(size: int, maxStringLen: int): string =
     ## Generates a random JSON string.
     ## size is in KiB, mayStringLen in characters.
-    
+
     randomize(size * maxStringLen)
     result = "{"
-    
+
     let targetSize = size * 1024
     var
         indentation = 2
@@ -44,13 +44,13 @@ proc genJsonString(size: int, maxStringLen: int): string =
         curSize = 1
         justOpened = true
     levels.add((kind: otMap, len: 0))
-    
+
     while levels.len > 0:
         let
             objectCloseProbability =
                 float(levels[levels.high].len + levels.high) * 0.025
             closeObject = random(1.0) <= objectCloseProbability
-        
+
         if (closeObject and levels.len > 1) or curSize > targetSize:
             indentation -= 2
             if justOpened:
@@ -67,9 +67,9 @@ proc genJsonString(size: int, maxStringLen: int): string =
             curSize += 1
             discard levels.pop()
             continue
-        
+
         levels[levels.high].len += 1
-        
+
         if justOpened:
             justOpened = false
             result.add("\x0A")
@@ -79,7 +79,7 @@ proc genJsonString(size: int, maxStringLen: int): string =
             result.add(",\x0A")
             result.add(repeat(' ', indentation))
             curSize += indentation + 2
-        
+
         case levels[levels.high].kind
         of otMap:
             let key = genString(maxStringLen)
@@ -88,12 +88,12 @@ proc genJsonString(size: int, maxStringLen: int): string =
             curSize += key.len + 2
         of otSequence:
             discard
-        
+
         let
             objectValueProbability =
                 0.8 / float(levels.len * levels.len)
             generateObjectValue = random(1.0) <= objectValueProbability
-        
+
         if generateObjectValue:
             let objectKind = if random(2) == 0: otMap else: otSequence
             case objectKind
@@ -126,7 +126,7 @@ proc genJsonString(size: int, maxStringLen: int): string =
                     discard
             else:
                 discard
-            
+
             result.add(s)
             curSize += s.len
 
@@ -138,7 +138,7 @@ var
     json100k = genJsonString(100, 32)
     tagLib   = initCoreTagLibrary()
     parser = newYamlParser(initCoreTagLibrary())
-    
+
 block:
     multibench(cJson1k, 100):
         let res = parseJson(json1k)
@@ -152,30 +152,21 @@ block:
 block:
     multibench(cJson100k, 100):
         let res = parseJson(json100k)
-        assert res.kind == JObject    
+        assert res.kind == JObject
 
 block:
     multibench(cYaml1k, 100):
-        var
-            s = newStringStream(json1k)
-            events = parser.parse(s)
-        let res = constructJson(events)
+        let res = loadToJson(json1k)
         assert res[0].kind == JObject
 
 block:
     multibench(cYaml10k, 100):
-        var
-            s = newStringStream(json10k)
-            events = parser.parse(s)
-        let res = constructJson(events)
+        let res = loadToJson(json10k)
         assert res[0].kind == JObject
 
 block:
     multibench(cYaml100k, 100):
-        var
-            s = newStringStream(json100k)
-            events = parser.parse(s)
-        let res = constructJson(events)
+        let res = loadToJson(json100k)
         assert res[0].kind == JObject
 
 block:

bench/yamlBench.nim

@@ -1,5 +1,5 @@
-import "../yaml", common
-import math, strutils, stopwatch, terminal, algorithm
+import "../yaml", commonBench
+import math, strutils, stopwatch, terminal, algorithm, random, streams
 
 from nimlets_yaml import objKind
 
@@ -32,17 +32,16 @@ proc genKey(): string =
             let c = random(26 + 26 + 10)
             if c < 26: result.add(char(c + 65))
             elif c < 52: result.add(char(c + 97 - 26))
-            else: result.add(char(c + 48 - 52)) 
+            else: result.add(char(c + 48 - 52))
     else: result = genString(31) & char(random(26) + 65)
 
 proc genYamlString(size: int, maxStringLen: int,
                    style: PresentationStyle): string =
     ## Generates a random YAML string.
     ## size is in KiB, mayStringLen in characters.
-    
+
     randomize(size * maxStringLen * ord(style))
-    result = "{"
-    
+
     let targetSize = size * 1024
     var
         target = newStringStream()
@@ -53,13 +52,13 @@ proc genYamlString(size: int, maxStringLen: int,
             levels.add((kind: yMapping, len: 0))
             yield startDocEvent()
             yield startMapEvent()
-            
+
             while levels.len > 0:
                 let
                     objectCloseProbability =
                         float(levels[levels.high].len + levels.high) * 0.025
                     closeObject = random(1.0) <= objectCloseProbability
-        
+
                 if (closeObject and levels.len > 1) or curSize > targetSize:
                     case levels[levels.high].kind
                     of yMapping: yield endMapEvent()
@@ -68,19 +67,19 @@ proc genYamlString(size: int, maxStringLen: int,
                     curSize += 1
                     discard levels.pop()
                     continue
-        
+
                 levels[levels.high].len += 1
                 if levels[levels.high].kind == yMapping:
                     let key = genKey()
                     yield scalarEvent(key)
-    
+
                 let
                     objectValueProbability =
                         0.8 / float(levels.len * levels.len)
                     generateObjectValue = random(1.0) <= objectValueProbability
                     hasTag = random(2) == 0
                 var tag = yTagQuestionMark
-    
+
                 if generateObjectValue:
                     let objectKind = if random(3) == 0: ySequence else: yMapping
                     case objectKind
@@ -120,7 +119,7 @@ proc genYamlString(size: int, maxStringLen: int,
                             if hasTag: tag = yTagNull
                         else: discard
                     else: discard
-        
+
                     yield scalarEvent(s, tag)
                     curSize += s.len
             yield endDocEvent()
@@ -128,32 +127,35 @@ proc genYamlString(size: int, maxStringLen: int,
     present(yStream, target, initExtendedTagLibrary(),
             defineOptions(style=style, outputVersion=ov1_1))
     result = target.data
-    
+
 var
-    cYaml1k, cYaml10k, cYaml100k, cLibYaml1k, cLibYaml10k, cLibYaml100k: int64
+    cYaml1k, cYaml10k, cYaml100k, cLibYaml1k, cLibYaml10k, cLibYaml100k,
+        cYaml1m, cLibYaml1m: int64
     yaml1k   = genYamlString(1, 32, psDefault)
     yaml10k  = genYamlString(10, 32, psDefault)
     yaml100k = genYamlString(100, 32, psDefault)
+    yaml1m  = genYamlString(1000, 32, psDefault)
     tagLib   = initExtendedTagLibrary()
     parser = newYamlParser(tagLib)
 
 block:
     multibench(cYaml1k, 100):
-        var s = newStringStream(yaml1k)
-        let res = loadDOM(s)
+        let res = loadDOM(yaml1k)
         assert res.root.kind == yMapping
 
 block:
     multibench(cYaml10k, 100):
-        var
-            s = newStringStream(yaml10k)
-        let res = loadDOM(s)
+        let res = loadDOM(yaml10k)
         assert res.root.kind == yMapping
 
 block:
     multibench(cYaml100k, 100):
-        var s = newStringStream(yaml100k)
-        let res = loadDOM(s)
+        let res = loadDOM(yaml100k)
+        assert res.root.kind == yMapping
+
+block:
+    multibench(cYaml1m, 2):
+        let res = loadDOM(yaml1m)
         assert res.root.kind == yMapping
 
 block:
@@ -171,6 +173,11 @@ block:
         let res = nimlets_yaml.load(yaml100k)
         assert res[0].objKind == nimlets_yaml.YamlObjKind.Map
 
+block:
+    multibench(cLibYaml1m, 2):
+        let res = nimlets_yaml.load(yaml1m)
+        assert res[0].objKind == nimlets_yaml.YamlObjKind.Map
+
 proc writeResult(caption: string, num: int64) =
     styledWriteLine(stdout, resetStyle, caption, fgGreen, $num, resetStyle, "μs")
 
@@ -188,4 +195,8 @@ writeResult "LibYAML: ", cLibYaml10k div 1000
 setForegroundColor(fgWhite)
 writeStyled "100k input\n----------\n"
 writeResult "NimYAML: ", cYaml100k div 1000
-writeResult "LibYAML: ", cLibYaml100k div 1000
+writeResult "LibYAML: ", cLibYaml100k div 1000
+setForegroundColor(fgWhite)
+writeStyled "1m input\n---------\n"
+writeResult "NimYAML: ", cYaml1m div 1000
+writeResult "LibYAML: ", cLibYaml1m div 1000

config.nims

@@ -21,6 +21,12 @@ task serializationTests, "Run serialization tests":
 task documentation, "Generate documentation":
   exec "mkdir -p docout"
   exec r"nim doc2 -o:docout/yaml.html --docSeeSrcUrl:https://github.com/flyx/NimYAML/blob/`git log -n 1 --format=%H` yaml"
+  # bash! ah-ah \\ savior of the universe
+  for file in listFiles("yaml"):
+    let packageName = file[5..^5]
+    exec r"nim doc2 -o:docout/yaml." & packageName &
+        ".html --docSeeSrcUrl:https://github.com/flyx/NimYAML/blob/yaml/`git log -n 1 --format=%H` " &
+        file
   exec r"nim rst2html -o:docout/index.html doc/index.txt"
   exec r"nim rst2html -o:docout/api.html doc/api.txt"
   exec r"nim rst2html -o:docout/serialization.html doc/serialization.txt"
@@ -28,10 +34,10 @@ task documentation, "Generate documentation":
   setCommand "nop"
 
 task bench, "Benchmarking":
-  --d:release
   --r
   --w:off
   --hints:off
+  --d:release
   setCommand "c", "bench/bench"
 
 task clean, "Remove all generated files":

doc/api.txt

@@ -25,8 +25,8 @@ Intermediate Representation
 ===========================
 
 The base of all YAML processing with NimYAML is the
-`YamlStream <yaml.html#YamlStream>`_. This is basically an iterator over
-`YamlStreamEvent <yaml.html#YamlStreamEvent>`_ objects. Every proc that
+`YamlStream <yaml.stream.html#YamlStream>`_. This is basically an iterator over
+`YamlStreamEvent <yaml.stream.html#YamlStreamEvent>`_ objects. Every proc that
 represents a single stage of the loading or dumping process will either take a
 ``YamlStream`` as input or return a ``YamlStream``. Procs that implement the
 whole process in one step hide the ``YamlStream`` from the user. Every proc that
@@ -45,16 +45,17 @@ Loading YAML
 ============
 
 If you want to load YAML character data directly into a native Nim variable, you
-can use `load <yaml.html#load,Stream,K>`_. This is the easiest and recommended
-way to load YAML data. The following paragraphs will explain the steps involved.
+can use `load <yaml.serialization.html#load,,K>`_. This is the easiest and
+recommended way to load YAML data. This section gives an overview about how
+``load`` is implemented. It is absolutely possible to reimplement the loading
+step using the low-level API.
 
-For parsing, a `YamlParser <yaml.html#YamlParser>`_ object is needed. This
-object stores some state while parsing that may be useful for error reporting to
-the user. The
-`parse <yaml.html#parse,YamlParser,Stream>`_ proc implements the YAML processing
-step of the same name. All syntax errors in the input character stream are
-processed by ``parse``, which will raise a ``YamlParserError`` if it encounters
-a syntax error.
+For parsing, a `YamlParser <yaml.parser.html#YamlParser>`_ object is needed.
+This object stores some state while parsing that may be useful for error
+reporting to the user. The `parse <yaml.parser.html#parse,YamlParser,Stream>`_
+proc implements the YAML processing step of the same name. All syntax errors in
+the input character stream are processed by ``parse``, which will raise a
+``YamlParserError`` if it encounters a syntax error.
 
 Transforming a ``YamlStream`` to a native YAML object is done via
 ``construct``. It skips the ``compose`` step for efficiency reasons. As Nim is
@@ -67,19 +68,21 @@ Dumping YAML
 ============
 
 Dumping is preferredly done with
-`dump <yaml.html#dump,K,Stream,PresentationStyle,TagStyle,AnchorStyle,int>`_,
-which serializes a native Nim variable to a character stream. Like ``load``, you
-can use the steps involved separately.
+`dump <yaml.serialization.html#dump,K,Stream,TagStyle,AnchorStyle,PresentationOptions>`_,
+which serializes a native Nim variable to a character stream. As with ``load``,
+the following paragraph describes how ``dump`` is implemented using the
+low-level API.
 
-You transform a variable into a ``YamlStream`` with
-`represent <yaml.html#represent,T,TagStyle,AnchorStyle>`_. Depending on the
-``AnchorStyle`` you specify, this will transform ``ref`` variables with multiple
-instances into anchored elements and aliases (for ``asTidy`` and ``asAlways``)
-or write the same element into all places it occurs (for ``asNone``). Be aware
-that if you use ``asNone``, the value you serialize might not round-trip.
+A Nim value is transformed into a ``YamlStream`` with
+`represent <yaml.serialization.html#represent,T,TagStyle,AnchorStyle>`_.
+Depending on the ``AnchorStyle`` you specify, this will transform ``ref``
+variables with multiple instances into anchored elements and aliases (for
+``asTidy`` and ``asAlways``) or write the same element into all places it
+occurs (for ``asNone``). Be aware that if you use ``asNone``, the value you
+serialize might not round-trip.
 
 Transforming a ``YamlStream`` into YAML character data is done with
-`present <yaml.html#present,YamlStream,Stream,TagLibrary,PresentationStyle,int>`_.
+`present <yaml.presenter.html#present,YamlStream,Stream,TagLibrary,PresentationOptions>`_.
 You can choose from multiple presentation styles. ``psJson`` is not able to
 process some features of ``YamlStream`` s, the other styles support all features
 and are guaranteed to round-trip to the same ``YamlStream`` if you parse the
@@ -90,11 +93,12 @@ The Document Object Model
 
 Much like XML, YAML also defines a *document object model*. If you cannot or do
 not want to load a YAML character stream to native Nim types, you can instead
-load it into a ``YamlDocument``. This ``YamlDocument`` can also be serialized
-into a YAML character stream. All tags will be preserved exactly as they are
-when transforming from and to a ``YamlDocument``. The only important thing to
-remember is that when a value has no tag, it will get the non-specific tag
-``"!"`` for quoted scalars and ``"?"`` for all other nodes.
+load it into a `YamlDocument <yaml.dom.html#YamlDocument>`_. This
+``YamlDocument`` can also be serialized into a YAML character stream. All tags
+will be preserved exactly as they are when transforming from and to a
+``YamlDocument``. The only important thing to remember is that when a value has
+no tag, it will get the non-specific tag ``"!"`` for quoted scalars and ``"?"``
+for all other nodes.
 
 While tags are preserved, anchors will be resolved during loading and re-added
 during serialization. It is allowed for a ``YamlNode`` to occur multiple times

doc/processing.svg

@@ -68,18 +68,18 @@
       <line x1="375" y1="20" x2="595" y2="20" stroke="black" stroke-width="2"/>
     </g>
   </defs>
-  
+
   <line x1="145" y1="0" x2="145" y2="420" stroke="gray"/>
   <text x="55" y="25" fill="gray">Application</text>
   <text x="160" y="25" fill="gray">YAML</text>
-  
+
   <g id="boxes" text-anchor="middle" transform="translate(0 190)">
     <g id="native" transform="translate(0 0)">
       <use xlink:href="#box"/>
       <text x="60" y="25" font-style="italic">Native</text>
       <text x="60" y="45" font-weight="bold">Nim Types</text>
     </g>
-        
+
     <g id="representation" transform="translate(170 0)">
       <use xlink:href="#box"/>
       <text x="60" y="25" font-style="italic">Representation</text>
@@ -87,7 +87,7 @@
         <text x="60" y="45" font-weight="bold">YamlDocument</text>
       </a>
     </g>
-  
+
     <g id="serialization" transform="translate(320 0)">
       <use xlink:href="#box"/>
       <text x="60" y="25" font-style="italic">Serialization</text>
@@ -95,7 +95,7 @@
         <text x="60" y="45" font-weight="bold">YamlStream</text>
       </a>
     </g>
-    
+
     <g id="presentation" transform="translate(470 0)">
       <use xlink:href="#box"/>
       <text x="60" y="25" font-style="italic">Presentation</text>
@@ -104,87 +104,87 @@
       </a>
     </g>
   </g>
-  
+
   <g id="atomics" text-anchor="middle" transform="translate(0 130)">
     <g id="represent" transform="translate(65 10)">
       <use xlink:href="#atomic-right-long"/>
       <text x="80" y="25" font-style="italic">represent</text>
     </g>
-    
+
     <g id="serialize" transform="translate(235 10)">
       <use xlink:href="#atomic-right"/>
-      <a xlink:href="yaml.html#serialize,YamlDocument,TagLibrary,AnchorStyle" target="_top">
+      <a xlink:href="yaml.dom.html#serialize,YamlDocument,TagLibrary,AnchorStyle" target="_top">
         <text x="70" y="25" font-style="italic" font-weight="bold">serialize</text>
       </a>
     </g>
-    
+
     <g id="present" transform="translate(385 10)">
       <use xlink:href="#atomic-right"/>
-      <a xlink:href="yaml.html#present,YamlStream,Stream,TagLibrary,PresentationStyle,int" target="_top">
+      <a xlink:href="yaml.presenter.html#present,YamlStream,Stream,TagLibrary,PresentationOptions" target="_top">
         <text x="70" y="25" font-style="italic" font-weight="bold">present</text>
       </a>
     </g>
-    
+
     <g id="construct" transform="translate(65 120)">
       <use xlink:href="#atomic-left-long"/>
       <text x="80" y="35" font-style="italic">construct</text>
     </g>
-    
+
     <g id="compose" transform="translate(235 120)">
       <use xlink:href="#atomic-left"/>
-      <a xlink:href="yaml.html#compose,YamlStream,TagLibrary" target="_top">
+      <a xlink:href="yaml.dom.html#compose,YamlStream,TagLibrary" target="_top">
         <text x="70" y="35" font-style="italic" font-weight="bold">compose</text>
       </a>
     </g>
-    
+
     <g id="parse" transform="translate(385 120)">
       <use xlink:href="#atomic-left"/>
-      <a xlink:href="yaml.html#parse,YamlParser,Stream" target="_top">
+      <a xlink:href="yaml.parser.html#parse,YamlParser,Stream" target="_top">
         <text x="70" y="35" font-style="italic" font-weight="bold">parse</text>
       </a>
     </g>
   </g>
-  
+
   <g id="skipping-things" text-anchor="middle" transform="translate(0 90)">
     <g id="skipping-represent" transform="translate(55 10)">
       <use xlink:href="#skipping-right"/>
-      <a xlink:href="yaml.html#represent,T,TagStyle,AnchorStyle" target="_top">
+      <a xlink:href="yaml.serialization.html#represent,T,TagStyle,AnchorStyle" target="_top">
           <text x="175" y="25" font-weight="bold">represent</text>
       </a>
     </g>
     <g id="skipping-construct" transform="translate(55 160)">
       <use xlink:href="#skipping-left"/>
-      <a xlink:href="yaml.html#construct,YamlStream,T" target="_top">
+      <a xlink:href="yaml.serialization.html#construct,YamlStream,T" target="_top">
           <text x="175" y="75" font-weight="bold">construct</text>
       </a>
     </g>
   </g>
-  
+
   <g id="whole-lines" text-anchor="middle" transform="translate(0 20)">
     <g id="dump">
       <use xlink:href="#whole-right"/>
-      <a xlink:href="yaml.html#dump,K,Stream,PresentationStyle,TagStyle,AnchorStyle,int" target="_top">
+      <a xlink:href="yaml.serialization.html#dump,K,Stream,TagStyle,AnchorStyle,PresentationOptions" target="_top">
           <text x="325" y="25" font-style="italic" font-weight="bold">dump</text>
       </a>
     </g>
     <g id="dumpDOM">
       <use xlink:href="#dom-right"/>
-      <a xlink:href="yaml.html#dumpDOM,YamlDocument,Stream,PresentationStyle,AnchorStyle,int" target="_top">
-        <text x="380" y="65" font-weight="bold">dumpDOM</text> 
+      <a xlink:href="yaml.dom.html#dumpDOM,YamlDocument,Stream,AnchorStyle,PresentationOptions" target="_top">
+        <text x="380" y="65" font-weight="bold">dumpDOM</text>
       </a>
     </g>
     <g id="loadDOM" transform="translate(0 320)">
       <use xlink:href="#dom-left"/>
-      <a xlink:href="yaml.html#loadDOM,Stream" target="_top">
+      <a xlink:href="yaml.dom.html#loadDOM," target="_top">
         <text x="380" y="25" font-weight="bold">loadDOM</text>
       </a>
     </g>
     <g id="load" transform="translate(0 360)">
       <use xlink:href="#whole-left"/>
-      <a xlink:href="yaml.html#load,Stream,K" target="_top">
+      <a xlink:href="yaml.serialization.html#load,,K" target="_top">
           <text x="325" y="25" font-style="italic" font-weight="bold">load</text>
       </a>
     </g>
   </g>
-  
+
 </svg>

doc/serialization.txt

@@ -8,10 +8,11 @@ Introduction
 NimYAML tries hard to make transforming YAML characters streams to native Nim
 types and vice versa as easy as possible. In simple scenarios, you might not
 need anything else than the two procs
-`dump <yaml.html#dump,K,Stream,PresentationStyle,TagStyle,AnchorStyle,int>`_ and
-`load <yaml.html#load,Stream,K>`_. On the other side, the process should be as
-customizable as possible to allow the user to tightly control how the generated
-YAML character stream will look and how a YAML character stream is interpreted.
+`dump <yaml.serialization.html#dump,K,Stream,TagStyle,AnchorStyle,PresentationOptions>`_
+and `load <yaml.serialization.html#load,,K>`_. On the other side, the process
+should be as customizable as possible to allow the user to tightly control how
+the generated YAML character stream will look and how a YAML character stream is
+interpreted.
 
 An important thing to remember in NimYAML is that unlike in interpreted
 languages like Ruby, Nim cannot load a YAML character stream without knowing the
@@ -42,7 +43,8 @@ added in the future.
 
 This also means that NimYAML is generally able to work with object, tuple and
 enum types defined in the standard library or a third-party library without
-further configuration.
+further configuration, given that all fields of the object are accessible at the
+code point where NimYAML's facilities are invoked.
 
 Scalar Types
 ------------
@@ -113,7 +115,7 @@ For an object or tuple type to be directly usable with NimYAML, the following
 conditions must be met:
 
 - Every type contained in the object/tuple must be supported
-- All fields of an object type must be accessible from the code position where 
+- All fields of an object type must be accessible from the code position where
   you call NimYAML. If an object has non-public member fields, it can only be
   processed in the module where it is defined.
 - The object may not have a generic parameter
@@ -141,7 +143,7 @@ For example, this type:
   type
     AnimalKind = enum
       akCat, akDog
-  
+
     Animal = object
       name: string
       case kind: AnimalKind
@@ -175,7 +177,7 @@ list in order to load it:
 
 .. code-block:: nim
   import yaml
-  
+
   type
     ContainerKind = enum
       ckInt, ckString, ckNone
@@ -187,9 +189,9 @@ list in order to load it:
         strVal: string
       of ckNone:
         discard
-  
+
   markAsImplicit(Container)
-  
+
   var
     list: seq[Container]
     s = newFileStream("in.yaml")
@@ -232,7 +234,7 @@ otherwise, it would be loaded as ``nil``.
 As you might have noticed in the example above, the YAML tag of a ``seq``
 depends on its generic type parameter. The same applies to ``Table``. So, a
 table that maps ``int8`` to string sequences would be presented with the tag
-``!nim:tables:Table(nim:system:int8,nim:system:seq(tag:yaml.org,2002:string))``. 
+``!nim:tables:Table(nim:system:int8,nim:system:seq(tag:yaml.org,2002:string))``.
 These tags are generated on the fly based on the types you instantiate
 ``Table`` or ``seq`` with.
 
@@ -317,21 +319,20 @@ representObject
 .. code-block:: nim
 
     proc representObject*(value: MyObject, ts: TagStyle = tsNone,
-                          c: SerializationContext): RawYamlStream {.raises: [].}
+                          c: SerializationContext, tag: TagId): {.raises: [].}
 
-This proc should return an iterator over ``YamlStreamEvent`` which represents
-the type. Follow the following guidelines when implementing a custom
-``representObject`` proc:
+This proc should push a list of tokens that represent the type into the
+serialization context via ``c.put``. Follow the following guidelines when
+implementing a custom ``representObject`` proc:
 
 - You can use the helper template
   `presentTag <yaml.html#presentTag,typedesc,TagStyle>`_ for outputting the
   tag.
-- Always output the first tag with a ``yAnchorNone``. Anchors will be set
+- Always output the first token with a ``yAnchorNone``. Anchors will be set
   automatically by ``ref`` type handling.
 - When outputting non-scalar types, you should use the ``representObject``
   implementation of the child types, if possible.
-- Check if the given ``TagStyle`` equals ``tsRootOnly`` and if yes, change it
-  to ``tsNone`` for the child values.
+- Always use the ``tag`` parameter as tag for the first token you generate.
 - Never write a ``representObject`` proc for ``ref`` types.
 
 The following example for representing to a YAML scalar is the actual
@@ -339,28 +340,21 @@ implementation of representing ``int`` types:
 
 .. code-block:: nim
 
-    proc representObject*[T: uint8|uint16|uint32|uint64](
-            value: T, ts: TagStyle, c: SerializationContext):
-            RawYamlStream {.raises: [].} =
-        result = iterator(): YamlStreamEvent =
-            yield scalarEvent($value, presentTag(T, ts), yAnchorNone)
+    proc representObject*[T: int8|int16|int32|int64](value: T, ts: TagStyle,
+        c: SerializationContext, tag: TagId) {.raises: [].} =
+      ## represents an integer value as YAML scalar
+      c.put(scalarEvent($value, tag, yAnchorNone))
 
 The following example for representing to a YAML non-scalar is the actual
-implementation of representing ``seq`` types:
+implementation of representing ``seq`` and ``set`` types:
 
 .. code-block:: nim
 
-    proc representObject*[T](value: seq[T], ts: TagStyle,
-            c: SerializationContext): RawYamlStream {.raises: [].} =
-        result = iterator(): YamlStreamEvent =
-            let childTagStyle = if ts == tsRootOnly: tsNone else: ts
-            yield YamlStreamEvent(kind: yamlStartSequence,
-                                  seqTag: presentTag(seq[T], ts),
-                                  seqAnchor: yAnchorNone)
-            for item in value:
-                var events = representObject(item, childTagStyle, c)
-                while true:
-                    let event = events()
-                    if finished(events): break
-                    yield event
-            yield YamlStreamEvent(kind: yamlEndSequence)
+    proc representObject*[T](value: seq[T]|set[T], ts: TagStyle,
+        c: SerializationContext, tag: TagId) {.raises: [YamlStreamError].} =
+      ## represents a Nim seq as YAML sequence
+      let childTagStyle = if ts == tsRootOnly: tsNone else: ts
+      c.put(startSeqEvent(tag))
+      for item in value:
+        representChild(item, childTagStyle, c)
+      c.put(endSeqEvent())

doc/style.css

@@ -18,6 +18,11 @@ header a {
     padding-right: 5px;
 }
 
+header a.active {
+    background: #877 !important;
+    color: black !important;
+}
+
 header span {
     display: inline-block;
     line-height: 50px;
@@ -27,6 +32,34 @@ header span {
     padding-right: 5px;
 }
 
+header span a {
+    display: block;
+}
+
+header span ul {
+    display: none;
+    position: absolute;
+    top: 100%;
+    list-style: none;
+    background: #111;
+    margin: 0;
+}
+
+header span ul:after {
+    content: ""; clear: both; display: block;
+}
+
+header span:hover > ul {
+    display: block;
+}
+
+header span ul a {
+    font-size: smaller;
+    font-family: "Source Code Pro", Menlo, "Courier New", Courier, monospace;
+    padding: 0 10px;
+    line-height: 40px;
+}
+
 header a:link,
 header a:visited {
     background: inherit;

nimdoc.cfg

@@ -98,7 +98,7 @@ doc.file = """
 
   <link href="docutils.css" rel="stylesheet" type="text/css"/>
   <link href="style.css" rel="stylesheet" type="text/css"/>
-  
+
   <link href='http://fonts.googleapis.com/css?family=Raleway:400,600,900' rel='stylesheet' type='text/css'/>
   <link href='http://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>
 </head>
@@ -111,7 +111,20 @@ doc.file = """
   <span>Docs:</span>
   <a href="api.html">Overview</a>
   <a href="serialization.html">Serialization</a>
-  <a href="yaml.html">Module yaml</a>
+  <span>
+    <a href="#">Modules</a>
+    <ul>
+      <li><a href="yaml.html">yaml</a></li>
+      <li><a href="yaml.dom.html">yaml.dom</a></li>
+      <li><a href="yaml.hints.html">yaml.hints</a></li>
+      <li><a href="yaml.parser.html">yaml.parser</a></li>
+      <li><a href="yaml.presenter.html">yaml.presenter</a></li>
+      <li><a href="yaml.serialization.html">yaml.serialization</a></li>
+      <li><a href="yaml.stream.html">yaml.stream</a></li>
+      <li><a href="yaml.taglib.html">yaml.taglib</a></li>
+      <li><a href="yaml.tojson.html">yaml.tojson</a></li>
+    </ul>
+  </span>
 </header>
 <article id="documentId">
   <div class="container">

private/dom.nim

@@ -1,195 +0,0 @@
-#            NimYAML - YAML implementation in Nim
-#        (c) Copyright 2015 Felix Krause
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-
-proc newYamlNode*(content: string, tag: string = "?"): YamlNode =
-  YamlNode(kind: yScalar, content: content, tag: tag)
-
-proc newYamlNode*(children: openarray[YamlNode], tag: string = "?"):
-    YamlNode =
-  YamlNode(kind: ySequence, children: @children, tag: tag)
-
-proc newYamlNode*(pairs: openarray[tuple[key, value: YamlNode]],
-                  tag: string = "?"): YamlNode =
-  YamlNode(kind: yMapping, pairs: @pairs, tag: tag)
-
-proc initYamlDoc*(root: YamlNode): YamlDocument = result.root = root
-
-proc composeNode(s: var YamlStream, tagLib: TagLibrary,
-                 c: ConstructionContext):
-    YamlNode {.raises: [YamlStreamError, YamlConstructionError].} =
-  let start = s.next()
-  new(result)
-  try:
-    case start.kind
-    of yamlStartMap:
-      result.tag = tagLib.uri(start.mapTag)
-      result.kind = yMapping
-      result.pairs = newSeq[tuple[key, value: YamlNode]]()
-      while s.peek().kind != yamlEndMap:
-        let
-          key = composeNode(s, tagLib, c)
-          value = composeNode(s, tagLib, c)
-        result.pairs.add((key: key, value: value))
-      discard s.next()
-      if start.mapAnchor != yAnchorNone:
-        assert(not c.refs.hasKey(start.mapAnchor))
-        c.refs[start.mapAnchor] = cast[pointer](result)
-    of yamlStartSeq:
-      result.tag = tagLib.uri(start.seqTag)
-      result.kind = ySequence
-      result.children = newSeq[YamlNode]()
-      while s.peek().kind != yamlEndSeq:
-        result.children.add(composeNode(s, tagLib, c))
-      if start.seqAnchor != yAnchorNone:
-        assert(not c.refs.hasKey(start.seqAnchor))
-        c.refs[start.seqAnchor] = cast[pointer](result)
-      discard s.next()
-    of yamlScalar:
-      result.tag = tagLib.uri(start.scalarTag)
-      result.kind = yScalar
-      shallowCopy(result.content, start.scalarContent)
-      if start.scalarAnchor != yAnchorNone:
-        assert(not c.refs.hasKey(start.scalarAnchor))
-        c.refs[start.scalarAnchor] = cast[pointer](result)
-    of yamlAlias:
-      result = cast[YamlNode](c.refs[start.aliasTarget])
-    else: assert false, "Malformed YamlStream"
-  except KeyError:
-    raise newException(YamlConstructionError,
-                       "Wrong tag library: TagId missing")
-
-proc compose*(s: var YamlStream, tagLib: TagLibrary): YamlDocument
-    {.raises: [YamlStreamError, YamlConstructionError].} =
-  var context = newConstructionContext()
-  assert s.next().kind == yamlStartDoc, "Malformed YamlStream"
-  result.root = composeNode(s, tagLib, context)
-  assert s.next().kind == yamlEndDoc, "Malformed YamlStream"
-
-proc loadDOM*(s: Stream): YamlDocument
-    {.raises: [IOError, YamlParserError, YamlConstructionError].} =
-  var
-    tagLib = initExtendedTagLibrary()
-    parser = newYamlParser(tagLib)
-    events = parser.parse(s)
-  try: result = compose(events, tagLib)
-  except YamlStreamError:
-    let e = getCurrentException()
-    if e.parent of YamlParserError:
-      raise (ref YamlParserError)(e.parent)
-    elif e.parent of IOError:
-      raise (ref IOError)(e.parent)
-    else: assert false, "Never happens: " & e.parent.repr
-
-proc serializeNode(n: YamlNode, c: SerializationContext, a: AnchorStyle,
-                   tagLib: TagLibrary): RawYamlStream {.raises: [].}=
-  let p = cast[pointer](n)
-  if a != asNone and c.refs.hasKey(p):
-    try:
-      if c.refs[p] == yAnchorNone:
-        c.refs[p] = c.nextAnchorId
-        c.nextAnchorId = AnchorId(int(c.nextAnchorId) + 1)
-    except KeyError: assert false, "Can never happen"
-    result = iterator(): YamlStreamEvent {.raises: [].} =
-      var event: YamlStreamEvent
-      try: event = aliasEvent(c.refs[p])
-      except KeyError: assert false, "Can never happen"
-      yield event
-    return
-  var
-    tagId: TagId
-    anchor: AnchorId
-  try:
-    if a == asAlways:
-      c.refs[p] = c.nextAnchorId
-      c.nextAnchorId = AnchorId(int(c.nextAnchorId) + 1)
-    else: c.refs[p] = yAnchorNone
-    tagId = if tagLib.tags.hasKey(n.tag): tagLib.tags[n.tag] else:
-            tagLib.registerUri(n.tag)
-    case a
-    of asNone: anchor = yAnchorNone
-    of asTidy: anchor = cast[AnchorId](n)
-    of asAlways: anchor = c.refs[p]
-  except KeyError: assert false, "Can never happen"
-  result = iterator(): YamlStreamEvent =
-    case n.kind
-    of yScalar: yield scalarEvent(n.content, tagId, anchor)
-    of ySequence:
-      yield startSeqEvent(tagId, anchor)
-      for item in n.children:
-        var events = serializeNode(item, c, a, tagLib)
-        while true:
-          let event = events()
-          if finished(events): break
-          yield event
-      yield endSeqEvent()
-    of yMapping:
-      yield startMapEvent(tagId, anchor)
-      for i in n.pairs:
-        var events = serializeNode(i.key, c, a, tagLib)
-        while true:
-          let event = events()
-          if finished(events): break
-          yield event
-        events = serializeNode(i.value, c, a, tagLib)
-        while true:
-          let event = events()
-          if finished(events): break
-          yield event
-      yield endMapEvent()
-
-template processAnchoredEvent(target: expr, c: SerializationContext): stmt =
-  try:
-    let anchorId = c.refs[cast[pointer](target)]
-    if anchorId != yAnchorNone: target = anchorId
-    else: target = yAnchorNone
-  except KeyError: assert false, "Can never happen"
-  yield event
-
-proc serialize*(doc: YamlDocument, tagLib: TagLibrary, a: AnchorStyle = asTidy):
-    YamlStream {.raises: [].} =
-  var
-    context = newSerializationContext(a)
-    events = serializeNode(doc.root, context, a, tagLib)
-  if a == asTidy:
-    var backend = iterator(): YamlStreamEvent {.raises: [].} =
-      var output = newSeq[YamlStreamEvent]()
-      while true:
-        let event = events()
-        if finished(events): break
-        output.add(event)
-      yield startDocEvent()
-      for event in output.mitems():
-        case event.kind
-        of yamlScalar: processAnchoredEvent(event.scalarAnchor, context)
-        of yamlStartMap: processAnchoredEvent(event.mapAnchor, context)
-        of yamlStartSeq: processAnchoredEvent(event.seqAnchor, context)
-        else: yield event
-      yield endDocEvent()
-    result = initYamlStream(backend)
-  else:
-    var backend = iterator(): YamlStreamEvent {.raises: [].} =
-      yield startDocEvent()
-      while true:
-        let event = events()
-        if finished(events): break
-        yield event
-      yield endDocEvent()
-    result = initYamlStream(backend)
-
-proc dumpDOM*(doc: YamlDocument, target: Stream,
-              anchorStyle: AnchorStyle = asTidy,
-              options: PresentationOptions = defaultPresentationOptions)
-    {.raises: [YamlPresenterJsonError, YamlPresenterOutputError].} =
-  ## Dump a YamlDocument as YAML character stream.
-  var
-    tagLib = initExtendedTagLibrary()
-    events = serialize(doc, tagLib,
-                       if options.style == psJson: asNone else: anchorStyle)
-  try:
-    present(events, target, tagLib, options)
-  except YamlStreamError:
-    # serializing object does not raise any errors, so we can ignore this
-    assert false, "Can never happen"

private/events.nim

@@ -1,73 +0,0 @@
-#            NimYAML - YAML implementation in Nim
-#        (c) Copyright 2015 Felix Krause
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-
-proc `==`*(left: YamlStreamEvent, right: YamlStreamEvent): bool =
-  if left.kind != right.kind: return false
-  case left.kind
-  of yamlStartDoc, yamlEndDoc, yamlEndMap, yamlEndSeq: result = true
-  of yamlStartMap:
-    result = left.mapAnchor == right.mapAnchor and left.mapTag == right.mapTag
-  of yamlStartSeq:
-    result = left.seqAnchor == right.seqAnchor and left.seqTag == right.seqTag
-  of yamlScalar:
-    result = left.scalarAnchor == right.scalarAnchor and
-             left.scalarTag == right.scalarTag and
-             left.scalarContent == right.scalarContent
-  of yamlAlias: result = left.aliasTarget == right.aliasTarget
-
-proc `$`*(event: YamlStreamEvent): string =
-  result = $event.kind & '('
-  case event.kind
-  of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard
-  of yamlStartMap:
-    result &= "tag=" & $event.mapTag
-    if event.mapAnchor != yAnchorNone: result &= ", anchor=" & $event.mapAnchor
-  of yamlStartSeq:
-    result &= "tag=" & $event.seqTag
-    if event.seqAnchor != yAnchorNone: result &= ", anchor=" & $event.seqAnchor
-  of yamlScalar:
-    result &= "tag=" & $event.scalarTag
-    if event.scalarAnchor != yAnchorNone:
-      result &= ", anchor=" & $event.scalarAnchor
-    result &= ", content=\"" & event.scalarContent & '\"'
-  of yamlAlias:
-    result &= "aliasTarget=" & $event.aliasTarget
-  result &= ")"
-
-proc tag*(event: YamlStreamEvent): TagId =
-  case event.kind
-  of yamlStartMap: result = event.mapTag
-  of yamlStartSeq: result = event.seqTag
-  of yamlScalar: result = event.scalarTag
-  else: raise newException(FieldError, "Event " & $event.kind & " has no tag")
-
-proc startDocEvent*(): YamlStreamEvent =
-  result = YamlStreamEvent(kind: yamlStartDoc)
-    
-proc endDocEvent*(): YamlStreamEvent =
-  result = YamlStreamEvent(kind: yamlEndDoc)
-    
-proc startMapEvent*(tag: TagId = yTagQuestionMark,
-                    anchor: AnchorId = yAnchorNone): YamlStreamEvent =
-  result = YamlStreamEvent(kind: yamlStartMap, mapTag: tag, mapAnchor: anchor)
-                             
-proc endMapEvent*(): YamlStreamEvent =
-  result = YamlStreamEvent(kind: yamlEndMap)
-    
-proc startSeqEvent*(tag: TagId = yTagQuestionMark,
-                    anchor: AnchorId = yAnchorNone): YamlStreamEvent =
-  result = YamlStreamEvent(kind: yamlStartSeq, seqTag: tag, seqAnchor: anchor)
-                             
-proc endSeqEvent*(): YamlStreamEvent =
-  result = YamlStreamEvent(kind: yamlEndSeq)
-    
-proc scalarEvent*(content: string = "", tag: TagId = yTagQuestionMark,
-                  anchor: AnchorId = yAnchorNone): YamlStreamEvent =
-  result = YamlStreamEvent(kind: yamlScalar, scalarTag: tag,
-                           scalarAnchor: anchor, scalarContent: content)
-
-proc aliasEvent*(anchor: AnchorId): YamlStreamEvent =
-  result = YamlStreamEvent(kind: yamlAlias, aliasTarget: anchor)

private/internal.nim

@@ -0,0 +1,29 @@
+#            NimYAML - YAML implementation in Nim
+#        (c) Copyright 2016 Felix Krause
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+
+template internalError*(s: string) =
+  when not defined(release):
+    let ii = instantiationInfo()
+    echo "[NimYAML] Error in file ", ii.filename, " at line ", ii.line, ":"
+    echo s
+    when not defined(JS):
+      echo "[NimYAML] Stacktrace:"
+      try: writeStackTrace()
+      except: discard
+    echo "[NimYAML] Please report this bug."
+    quit 1
+template yAssert*(e: typed) =
+  when not defined(release):
+    if not e:
+      let ii = instantiationInfo()
+      echo "[NimYAML] Error in file ", ii.filename, " at line ", ii.line, ":"
+      echo "assertion failed!"
+      when not defined(JS):
+        echo "[NimYAML] Stacktrace:"
+        try: writeStackTrace()
+        except: discard
+      echo "[NimYAML] Please report this bug."
+      quit 1

private/serialization.nim

@@ -1,883 +0,0 @@
-#            NimYAML - YAML implementation in Nim
-#        (c) Copyright 2015 Felix Krause
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-
-proc newConstructionContext*(): ConstructionContext =
-  new(result)
-  result.refs = initTable[AnchorId, pointer]()
-
-proc newSerializationContext*(s: AnchorStyle): SerializationContext =
-  new(result)
-  result.refs = initTable[pointer, AnchorId]()
-  result.style = s
-  result.nextAnchorId = 0.AnchorId
-
-template presentTag*(t: typedesc, ts: TagStyle): TagId =
-  ## Get the TagId that represents the given type in the given style
-  if ts == tsNone: yTagQuestionMark else: yamlTag(t)
-
-proc lazyLoadTag(uri: string): TagId {.inline, raises: [].} =
-  try: result = serializationTagLibrary.tags[uri]
-  except KeyError: result = serializationTagLibrary.registerUri(uri)
-
-proc safeTagUri(id: TagId): string {.raises: [].} =
-  try:
-    let uri = serializationTagLibrary.uri(id)
-    if uri.len > 0 and uri[0] == '!': return uri[1..uri.len - 1]
-    else: return uri
-  except KeyError:
-    # cannot happen (theoretically, you know)
-    assert(false)
-
-template constructScalarItem*(s: var YamlStream, i: expr,
-                              t: typedesc, content: untyped) =
-  ## Helper template for implementing ``constructObject`` for types that
-  ## are constructed from a scalar. ``i`` is the identifier that holds
-  ## the scalar as ``YamlStreamEvent`` in the content. Exceptions raised in
-  ## the content will be automatically catched and wrapped in
-  ## ``YamlConstructionError``, which will then be raised.
-  let i = s.next()
-  if i.kind != yamlScalar:
-    raise newException(YamlConstructionError, "Expected scalar")
-  try: content
-  except YamlConstructionError: raise
-  except Exception:
-    var e = newException(YamlConstructionError,
-        "Cannot construct to " & name(t) & ": " & item.scalarContent)
-    e.parent = getCurrentException()
-    raise e
-
-proc yamlTag*(T: typedesc[string]): TagId {.inline, noSideEffect, raises: [].} =
-  yTagString
-
-proc constructObject*(s: var YamlStream, c: ConstructionContext,
-                      result: var string)
-    {.raises: [YamlConstructionError, YamlStreamError].} =
-  ## costructs a string from a YAML scalar
-  constructScalarItem(s, item, string):
-    result = item.scalarContent
-
-proc representObject*(value: string, ts: TagStyle,
-        c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} =
-  ## represents a string as YAML scalar
-  result = iterator(): YamlStreamEvent =
-    yield scalarEvent(value, tag, yAnchorNone)
-
-proc constructObject*[T: int8|int16|int32|int64](
-    s: var YamlStream, c: ConstructionContext, result: var T)
-    {.raises: [YamlConstructionError, YamlStreamError].} =
-  ## constructs an integer value from a YAML scalar
-  constructScalarItem(s, item, T):
-    result = T(parseBiggestInt(item.scalarContent))
-
-proc constructObject*(s: var YamlStream, c: ConstructionContext,
-                      result: var int)
-    {.raises: [YamlConstructionError, YamlStreamError], inline.} =
-  ## constructs an integer of architecture-defined length by loading it into
-  ## int32 and then converting it.
-  var i32Result: int32
-  constructObject(s, c, i32Result)
-  result = int(i32Result)
-
-proc representObject*[T: int8|int16|int32|int64](value: T, ts: TagStyle,
-    c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} =
-  ## represents an integer value as YAML scalar
-  result = iterator(): YamlStreamEvent =
-    yield scalarEvent($value, tag, yAnchorNone)
-
-proc representObject*(value: int, tagStyle: TagStyle,
-                      c: SerializationContext, tag: TagId): RawYamlStream
-    {.raises: [], inline.}=
-  ## represent an integer of architecture-defined length by casting it to int32.
-  ## on 64-bit systems, this may cause a type conversion error.
-
-  # currently, sizeof(int) is at least sizeof(int32).
-  representObject(int32(value), tagStyle, c, tag)
-
-{.push overflowChecks: on.}
-proc parseBiggestUInt(s: string): uint64 =
-  result = 0
-  for c in s:
-    if c in {'0'..'9'}: result *= 10.uint64 + (uint64(c) - uint64('0'))
-    elif c == '_': discard
-    else: raise newException(ValueError, "Invalid char in uint: " & c)
-{.pop.}
-
-proc constructObject*[T: uint8|uint16|uint32|uint64](
-    s: var YamlStream, c: ConstructionContext, result: var T)
-    {.raises: [YamlConstructionError, YamlStreamError].} =
-  ## construct an unsigned integer value from a YAML scalar
-  constructScalarItem(s, item, T):
-    result = T(parseBiggestUInt(item.scalarContent))
-
-proc constructObject*(s: var YamlStream, c: ConstructionContext,
-                      result: var uint)
-    {.raises: [YamlConstructionError, YamlStreamError], inline.} =
-  ## represent an unsigned integer of architecture-defined length by loading it
-  ## into uint32 and then converting it.
-  var u32Result: uint32
-  constructObject(s, c, u32Result)
-  result= uint(u32Result)
-
-proc representObject*[T: uint8|uint16|uint32|uint64](value: T, ts: TagStyle,
-    c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} =
-  ## represents an unsigned integer value as YAML scalar
-  result = iterator(): YamlStreamEvent =
-    yield scalarEvent($value, tag, yAnchorNone)
-
-proc representObject*(value: uint, ts: TagStyle, c: SerializationContext,
-    tag: TagId): RawYamlStream {.raises: [], inline.} =
-  ## represent an unsigned integer of architecture-defined length by casting it
-  ## to int32. on 64-bit systems, this may cause a type conversion error.
-  representObject(uint32(value), ts, c, tag)
-
-proc constructObject*[T: float|float32|float64](
-    s: var YamlStream, c: ConstructionContext, result: var T)
-    {.raises: [YamlConstructionError, YamlStreamError].} =
-  ## construct a float value from a YAML scalar
-  constructScalarItem(s, item, T):
-    let hint = guessType(item.scalarContent)
-    case hint
-    of yTypeFloat:
-      discard parseBiggestFloat(item.scalarContent, result)
-    of yTypeInteger:
-      discard parseBiggestFloat(item.scalarContent, result)
-    of yTypeFloatInf:
-        if item.scalarContent[0] == '-': result = NegInf
-        else: result = Inf
-    of yTypeFloatNaN: result = NaN
-    else:
-      raise newException(YamlConstructionError,
-          "Cannot construct to float: " & item.scalarContent)
-
-proc representObject*[T: float|float32|float64](value: T, ts: TagStyle,
-    c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} =
-  ## represents a float value as YAML scalar
-  result = iterator(): YamlStreamEvent =
-    var asString: string
-    case value
-    of Inf: asString = ".inf"
-    of NegInf: asString = "-.inf"
-    of NaN: asString = ".nan"
-    else: asString = $value
-    yield scalarEvent(asString, tag, yAnchorNone)
-
-proc yamlTag*(T: typedesc[bool]): TagId {.inline, raises: [].} = yTagBoolean
-
-proc constructObject*(s: var YamlStream, c: ConstructionContext,
-                      result: var bool)
-    {.raises: [YamlConstructionError, YamlStreamError].} =
-  ## constructs a bool value from a YAML scalar
-  constructScalarItem(s, item, bool):
-    case guessType(item.scalarContent)
-    of yTypeBoolTrue: result = true
-    of yTypeBoolFalse: result = false
-    else:
-      raise newException(YamlConstructionError,
-          "Cannot construct to bool: " & item.scalarContent)
-
-proc representObject*(value: bool, ts: TagStyle, c: SerializationContext,
-    tag: TagId): RawYamlStream  {.raises: [].} =
-  ## represents a bool value as a YAML scalar
-  result = iterator(): YamlStreamEvent =
-    yield scalarEvent(if value: "y" else: "n", tag, yAnchorNone)
-
-proc constructObject*(s: var YamlStream, c: ConstructionContext,
-                      result: var char)
-    {.raises: [YamlConstructionError, YamlStreamError].} =
-  ## constructs a char value from a YAML scalar
-  constructScalarItem(s, item, char):
-    if item.scalarContent.len != 1:
-      raise newException(YamlConstructionError,
-          "Cannot construct to char (length != 1): " & item.scalarContent)
-    else: result = item.scalarContent[0]
-
-proc representObject*(value: char, ts: TagStyle, c: SerializationContext,
-    tag: TagId): RawYamlStream {.raises: [].} =
-  ## represents a char value as YAML scalar
-  result = iterator(): YamlStreamEvent =
-    yield scalarEvent("" & value, tag, yAnchorNone)
-
-proc yamlTag*[I](T: typedesc[seq[I]]): TagId {.inline, raises: [].} =
-  let uri = "!nim:system:seq(" & safeTagUri(yamlTag(I)) & ')'
-  result = lazyLoadTag(uri)
-
-proc yamlTag*[I](T: typedesc[set[I]]): TagId {.inline, raises: [].} =
-  let uri = "!nim:system:set(" & safeTagUri(yamlTag(I)) & ')'
-  result = lazyLoadTag(uri)
-
-proc constructObject*[T](s: var YamlStream, c: ConstructionContext,
-                         result: var seq[T])
-    {.raises: [YamlConstructionError, YamlStreamError].} =
-  ## constructs a Nim seq from a YAML sequence
-  let event = s.next()
-  if event.kind != yamlStartSeq:
-    raise newException(YamlConstructionError, "Expected sequence start")
-  result = newSeq[T]()
-  while s.peek().kind != yamlEndSeq:
-    var item: T
-    constructChild(s, c, item)
-    result.add(item)
-  discard s.next()
-
-proc constructObject*[T](s: var YamlStream, c: ConstructionContext,
-                         result: var set[T])
-    {.raises: [YamlConstructionError, YamlStreamError].} =
-  ## constructs a Nim seq from a YAML sequence
-  let event = s.next()
-  if event.kind != yamlStartSeq:
-    raise newException(YamlConstructionError, "Expected sequence start")
-  result = {}
-  while s.peek().kind != yamlEndSeq:
-    var item: T
-    constructChild(s, c, item)
-    result.incl(item)
-  discard s.next()
-
-proc representObject*[T](value: seq[T]|set[T], ts: TagStyle,
-    c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} =
-  ## represents a Nim seq as YAML sequence
-  result = iterator(): YamlStreamEvent =
-    let childTagStyle = if ts == tsRootOnly: tsNone else: ts
-    yield startSeqEvent(tag)
-    for item in value:
-      var events = representChild(item, childTagStyle, c)
-      while true:
-        let event = events()
-        if finished(events): break
-        yield event
-    yield endSeqEvent()
-
-proc yamlTag*[I, V](T: typedesc[array[I, V]]): TagId {.inline, raises: [].} =
-  const rangeName = name(I)
-  let uri = "!nim:system:array(" & rangeName[6..rangeName.high()] & "," &
-      safeTagUri(yamlTag(V)) & ')'
-  result = lazyLoadTag(uri)
-
-proc constructObject*[I, T](s: var YamlStream, c: ConstructionContext,
-                         result: var array[I, T])
-    {.raises: [YamlConstructionError, YamlStreamError].} =
-  ## constructs a Nim array from a YAML sequence
-  var event = s.next()
-  if event.kind != yamlStartSeq:
-    raise newException(YamlConstructionError, "Expected sequence start")
-  for index in low(I)..high(I):
-    event = s.peek()
-    if event.kind == yamlEndSeq:
-      raise newException(YamlConstructionError, "Too few array values")
-    constructChild(s, c, result[index])
-  event = s.next()
-  if event.kind != yamlEndSeq:
-    raise newException(YamlConstructionError, "Too much array values")
-
-proc representObject*[I, T](value: array[I, T], ts: TagStyle,
-    c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} =
-  ## represents a Nim array as YAML sequence
-  result = iterator(): YamlStreamEvent =
-    let childTagStyle = if ts == tsRootOnly: tsNone else: ts
-    yield startSeqEvent(tag)
-    for item in value:
-      var events = representChild(item, childTagStyle, c)
-      while true:
-        let event = events()
-        if finished(events): break
-        yield event
-    yield endSeqEvent()
-
-proc yamlTag*[K, V](T: typedesc[Table[K, V]]): TagId {.inline, raises: [].} =
-  try:
-    let uri = "!nim:tables:Table(" & safeTagUri(yamlTag(K)) & "," &
-        safeTagUri(yamlTag(V)) & ")"
-    result = lazyLoadTag(uri)
-  except KeyError:
-    # cannot happen (theoretically, you know)
-    assert(false)
-
-proc constructObject*[K, V](s: var YamlStream, c: ConstructionContext,
-                            result: var Table[K, V])
-    {.raises: [YamlConstructionError, YamlStreamError].} =
-  ## constructs a Nim Table from a YAML mapping
-  let event = s.next()
-  if event.kind != yamlStartMap:
-    raise newException(YamlConstructionError, "Expected map start, got " &
-                       $event.kind)
-  result = initTable[K, V]()
-  while s.peek.kind != yamlEndMap:
-    var
-      key: K
-      value: V
-    constructChild(s, c, key)
-    constructChild(s, c, value)
-    if result.contains(key):
-      raise newException(YamlConstructionError, "Duplicate table key!")
-    result[key] = value
-  discard s.next()
-
-proc representObject*[K, V](value: Table[K, V], ts: TagStyle,
-    c: SerializationContext, tag: TagId): RawYamlStream {.raises:[].} =
-  ## represents a Nim Table as YAML mapping
-  result = iterator(): YamlStreamEvent =
-    let childTagStyle = if ts == tsRootOnly: tsNone else: ts
-    yield startMapEvent(tag)
-    for key, value in value.pairs:
-      var events = representChild(key, childTagStyle, c)
-      while true:
-        let event = events()
-        if finished(events): break
-        yield event
-      events = representChild(value, childTagStyle, c)
-      while true:
-        let event = events()
-        if finished(events): break
-        yield event
-    yield endMapEvent()
-
-proc yamlTag*[K, V](T: typedesc[OrderedTable[K, V]]): TagId
-    {.inline, raises: [].} =
-  try:
-    let uri = "!nim:tables:OrderedTable(" & safeTagUri(yamlTag(K)) & "," &
-        safeTagUri(yamlTag(V)) & ")"
-    result = lazyLoadTag(uri)
-  except KeyError:
-    # cannot happen (theoretically, you know)
-    assert(false)
-
-proc constructObject*[K, V](s: var YamlStream, c: ConstructionContext,
-                            result: var OrderedTable[K, V])
-    {.raises: [YamlConstructionError, YamlStreamError].} =
-  ## constructs a Nim OrderedTable from a YAML mapping
-  let event = s.next()
-  if event.kind != yamlStartSeq:
-    raise newException(YamlConstructionError, "Expected seq start, got " &
-                       $event.kind)
-  result = initOrderedTable[K, V]()
-  while s.peek.kind != yamlEndSeq:
-    var
-      key: K
-      value: V
-    if s.next().kind != yamlStartMap:
-      raise newException(YamlConstructionError,
-          "Expected map start, got " & $event.kind)
-    constructChild(s, c, key)
-    constructChild(s, c, value)
-    if s.next().kind != yamlEndMap:
-      raise newException(YamlConstructionError,
-          "Expected map end, got " & $event.kind)
-    if result.contains(key):
-      raise newException(YamlConstructionError, "Duplicate table key!")
-    result.add(key, value)
-  discard s.next()
-
-proc representObject*[K, V](value: OrderedTable[K, V], ts: TagStyle,
-    c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} =
-  result = iterator(): YamlStreamEvent =
-    let childTagStyle = if ts == tsRootOnly: tsNone else: ts
-    yield startSeqEvent(tag)
-    for key, value in value.pairs:
-      yield startMapEvent()
-      var events = representChild(key, childTagStyle, c)
-      while true:
-        let event = events()
-        if finished(events): break
-        yield event
-      events = representChild(value, childTagStyle, c)
-      while true:
-        let event = events()
-        if finished(events): break
-        yield event
-      yield endMapEvent()
-    yield endSeqEvent()
-
-proc yamlTag*(T: typedesc[object|enum]):
-    TagId {.inline, raises: [].} =
-  var uri = "!nim:custom:" & (typetraits.name(type(T)))
-  try: serializationTagLibrary.tags[uri]
-  except KeyError: serializationTagLibrary.registerUri(uri)
-
-proc yamlTag*(T: typedesc[tuple]):
-    TagId {.inline, raises: [].} =
-  var
-    i: T
-    uri = "!nim:tuple("
-    first = true
-  for name, value in fieldPairs(i):
-    if first: first = false
-    else: uri.add(",")
-    uri.add(safeTagUri(yamlTag(type(value))))
-  uri.add(")")
-  try: serializationTagLibrary.tags[uri]
-  except KeyError: serializationTagLibrary.registerUri(uri)
-
-macro constructFieldValue(t: typedesc, stream: expr, context: expr,
-                           name: expr, o: expr): stmt =
-  let tDesc = getType(getType(t)[1])
-  result = newNimNode(nnkCaseStmt).add(name)
-  for child in tDesc[2].children:
-    if child.kind == nnkRecCase:
-      let
-        discriminant = newDotExpr(o, newIdentNode($child[0]))
-        discType = newCall("type", discriminant)
-      var disOb = newNimNode(nnkOfBranch).add(newStrLitNode($child[0]))
-      disOb.add(newStmtList(
-          newNimNode(nnkVarSection).add(
-              newNimNode(nnkIdentDefs).add(
-                  newIdentNode("value"), discType, newEmptyNode())),
-          newCall("constructChild", stream, context, newIdentNode("value")),
-          newAssignment(discriminant, newIdentNode("value"))))
-      result.add(disOb)
-      for bIndex in 1 .. len(child) - 1:
-        let discTest = infix(discriminant, "==", child[bIndex][0])
-        for item in child[bIndex][1].children:
-          assert item.kind == nnkSym
-          var ob = newNimNode(nnkOfBranch).add(newStrLitNode($item))
-          let field = newDotExpr(o, newIdentNode($item))
-          var ifStmt = newIfStmt((cond: discTest, body: newStmtList(
-              newCall("constructChild", stream, context, field))))
-          ifStmt.add(newNimNode(nnkElse).add(newNimNode(nnkRaiseStmt).add(
-              newCall("newException", newIdentNode("YamlConstructionError"),
-              infix(newStrLitNode("Field " & $item & " not allowed for " &
-              $child[0] & " == "), "&", prefix(discriminant, "$"))))))
-          ob.add(newStmtList(ifStmt))
-          result.add(ob)
-    else:
-      assert child.kind == nnkSym
-      var ob = newNimNode(nnkOfBranch).add(newStrLitNode($child))
-      let field = newDotExpr(o, newIdentNode($child))
-      ob.add(newStmtList(newCall("constructChild", stream, context, field)))
-      result.add(ob)
-
-proc isVariantObject(t: typedesc): bool {.compileTime.} =
-  let tDesc = getType(t)
-  if tDesc.kind != nnkObjectTy: return false
-  for child in tDesc[2].children:
-    if child.kind == nnkRecCase: return true
-  return false
-
-proc constructObject*[O: object|tuple](
-    s: var YamlStream, c: ConstructionContext, result: var O)
-    {.raises: [YamlConstructionError, YamlStreamError].} =
-  ## constructs a Nim object or tuple from a YAML mapping
-  let e = s.next()
-  const
-    startKind = when isVariantObject(O): yamlStartSeq else: yamlStartMap
-    endKind = when isVariantObject(O): yamlEndSeq else: yamlEndMap
-  if e.kind != startKind:
-    raise newException(YamlConstructionError, "While constructing " &
-        typetraits.name(O) & ": Expected map start, got " & $e.kind)
-  when isVariantObject(O): reset(result) # make discriminants writeable
-  while s.peek.kind != endKind:
-    # todo: check for duplicates in input and raise appropriate exception
-    # also todo: check for missing items and raise appropriate exception
-    var e = s.next()
-    when isVariantObject(O):
-      if e.kind != yamlStartMap:
-        raise newException(YamlConstructionError,
-            "Expected single-pair map, got " & $e.kind)
-      e = s.next()
-    if e.kind != yamlScalar:
-      raise newException(YamlConstructionError,
-          "Expected field name, got " & $e.kind)
-    let name = e.scalarContent
-    when result is tuple:
-      for fname, value in fieldPairs(result):
-        if fname == name:
-          constructChild(s, c, value)
-          break
-    else:
-      constructFieldValue(O, s, c, name, result)
-    when isVariantObject(O):
-      e = s.next()
-      if e.kind != yamlEndMap:
-        raise newException(YamlConstructionError,
-            "Expected end of single-pair map, got " & $e.kind)
-  discard s.next()
-
-proc representObject*[O: object|tuple](value: O, ts: TagStyle,
-    c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} =
-  ## represents a Nim object or tuple as YAML mapping
-  result = iterator(): YamlStreamEvent =
-    let childTagStyle = if ts == tsRootOnly: tsNone else: ts
-    when isVariantObject(O):
-      yield startSeqEvent(tag, yAnchorNone)
-    else:
-      yield startMapEvent(tag, yAnchorNone)
-    for name, value in fieldPairs(value):
-      when isVariantObject(O):
-        yield startMapEvent(yTagQuestionMark, yAnchorNone)
-      yield scalarEvent(name,
-          if childTagStyle == tsNone: yTagQuestionMark else:
-          yTagNimField, yAnchorNone)
-      var events = representChild(value, childTagStyle, c)
-      while true:
-        let event = events()
-        if finished(events): break
-        yield event
-      when isVariantObject(O):
-        yield endMapEvent()
-    when isVariantObject(O):
-      yield endSeqEvent()
-    else:
-      yield endMapEvent()
-
-proc constructObject*[O: enum](s: var YamlStream, c: ConstructionContext,
-                               result: var O)
-    {.raises: [YamlConstructionError, YamlStreamError].} =
-  ## constructs a Nim enum from a YAML scalar
-  let e = s.next()
-  if e.kind != yamlScalar:
-    raise newException(YamlConstructionError, "Expected scalar, got " &
-                       $e.kind)
-  try: result = parseEnum[O](e.scalarContent)
-  except ValueError:
-    var ex = newException(YamlConstructionError, "Cannot parse '" &
-        e.scalarContent & "' as " & type(O).name)
-    ex.parent = getCurrentException()
-    raise ex
-
-proc representObject*[O: enum](value: O, ts: TagStyle,
-    c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} =
-  ## represents a Nim enum as YAML scalar
-  result = iterator(): YamlStreamEvent =
-    yield scalarEvent($value, tag, yAnchorNone)
-
-proc yamlTag*[O](T: typedesc[ref O]): TagId {.inline, raises: [].} = yamlTag(O)
-
-macro constructImplicitVariantObject(s, c, r, possibleTagIds: expr,
-                                     t: typedesc): stmt =
-  let tDesc = getType(getType(t)[1])
-  assert tDesc.kind == nnkObjectTy
-  let recCase = tDesc[2][0]
-  assert recCase.kind == nnkRecCase
-  let
-    discriminant = newDotExpr(r, newIdentNode($recCase[0]))
-    discType = newCall("type", discriminant)
-  var ifStmt = newNimNode(nnkIfStmt)
-  for i in 1 .. recCase.len - 1:
-    assert recCase[i].kind == nnkOfBranch
-    var branch = newNimNode(nnkElifBranch)
-    var branchContent = newStmtList(newAssignment(discriminant, recCase[i][0]))
-    case recCase[i][1].len
-    of 0:
-      branch.add(infix(newIdentNode("yTagNull"), "in", possibleTagIds))
-      branchContent.add(newNimNode(nnkDiscardStmt).add(newCall("next", s)))
-    of 1:
-      let field = newDotExpr(r, newIdentNode($recCase[i][1][0]))
-      branch.add(infix(
-          newCall("yamlTag", newCall("type", field)), "in", possibleTagIds))
-      branchContent.add(newCall("constructChild", s, c, field))
-    else: assert false
-    branch.add(branchContent)
-    ifStmt.add(branch)
-  let raiseStmt = newNimNode(nnkRaiseStmt).add(
-      newCall("newException", newIdentNode("YamlConstructionError"),
-      infix(newStrLitNode("This value type does not map to any field in " &
-                          typetraits.name(t) & ": "), "&",
-            newCall("uri", newIdentNode("serializationTagLibrary"),
-              newNimNode(nnkBracketExpr).add(possibleTagIds, newIntLitNode(0)))
-      )
-  ))
-  ifStmt.add(newNimNode(nnkElse).add(newNimNode(nnkTryStmt).add(
-      newStmtList(raiseStmt), newNimNode(nnkExceptBranch).add(
-        newIdentNode("KeyError"), newStmtList(newCall("assert", newLit(false)))
-  ))))
-  result = newStmtList(newCall("reset", r), ifStmt)
-
-proc constructChild*[T](s: var YamlStream, c: ConstructionContext,
-                        result: var T) =
-  let item = s.peek()
-  when compiles(implicitVariantObject(result)):
-    var possibleTagIds = newSeq[TagId]()
-    case item.kind
-    of yamlScalar:
-      case item.scalarTag
-      of yTagQuestionMark:
-        case guessType(item.scalarContent)
-        of yTypeInteger:
-          possibleTagIds.add([yamlTag(int), yamlTag(int8), yamlTag(int16),
-                              yamlTag(int32), yamlTag(int64)])
-          if item.scalarContent[0] != '-':
-            possibleTagIds.add([yamlTag(uint), yamlTag(uint8), yamlTag(uint16),
-                                yamlTag(uint32), yamlTag(uint64)])
-        of yTypeFloat, yTypeFloatInf, yTypeFloatNaN:
-          possibleTagIds.add([yamlTag(float), yamlTag(float32),
-                              yamlTag(float64)])
-        of yTypeBoolTrue, yTypeBoolFalse:
-          possibleTagIds.add(yamlTag(bool))
-        of yTypeNull:
-          raise newException(YamlConstructionError, "not implemented!")
-        of yTypeUnknown:
-          possibleTagIds.add(yamlTag(string))
-      of yTagExclamationMark:
-        possibleTagIds.add(yamlTag(string))
-      else:
-        possibleTagIds.add(item.scalarTag)
-    of yamlStartMap:
-      if item.mapTag in [yTagQuestionMark, yTagExclamationMark]:
-        raise newException(YamlConstructionError,
-            "Complex value of implicit variant object type must have a tag.")
-      possibleTagIds.add(item.mapTag)
-    of yamlStartSeq:
-      if item.seqTag in [yTagQuestionMark, yTagExclamationMark]:
-        raise newException(YamlConstructionError,
-            "Complex value of implicit variant object type must have a tag.")
-      possibleTagIds.add(item.seqTag)
-    else: assert false
-    constructImplicitVariantObject(s, c, result, possibleTagIds, T)
-  else:
-    case item.kind
-    of yamlScalar:
-      if item.scalarTag notin [yTagQuestionMark, yTagExclamationMark,
-                               yamlTag(T)]:
-        raise newException(YamlConstructionError, "Wrong tag for " &
-                           typetraits.name(T))
-      elif item.scalarAnchor != yAnchorNone:
-        raise newException(YamlConstructionError, "Anchor on non-ref type")
-    of yamlStartMap:
-      if item.mapTag notin [yTagQuestionMark, yamlTag(T)]:
-        raise newException(YamlConstructionError, "Wrong tag for " &
-                           typetraits.name(T))
-      elif item.mapAnchor != yAnchorNone:
-        raise newException(YamlConstructionError, "Anchor on non-ref type")
-    of yamlStartSeq:
-      if item.seqTag notin [yTagQuestionMark, yamlTag(T)]:
-        raise newException(YamlConstructionError, "Wrong tag for " &
-                           typetraits.name(T))
-      elif item.seqAnchor != yAnchorNone:
-        raise newException(YamlConstructionError, "Anchor on non-ref type")
-    else: assert false
-    constructObject(s, c, result)
-
-proc constructChild*(s: var YamlStream, c: ConstructionContext,
-                     result: var string) =
-  let item = s.peek()
-  if item.kind == yamlScalar:
-    if item.scalarTag == yTagNimNilString:
-      discard s.next()
-      result = nil
-      return
-    elif item.scalarTag notin
-        [yTagQuestionMark, yTagExclamationMark, yamlTag(string)]:
-      raise newException(YamlConstructionError, "Wrong tag for string")
-    elif item.scalarAnchor != yAnchorNone:
-      raise newException(YamlConstructionError, "Anchor on non-ref type")
-  constructObject(s, c, result)
-
-proc constructChild*[T](s: var YamlStream, c: ConstructionContext,
-                        result: var seq[T]) =
-  let item = s.peek()
-  if item.kind == yamlScalar:
-    if item.scalarTag == yTagNimNilSeq:
-      discard s.next()
-      result = nil
-      return
-  elif item.kind == yamlStartSeq:
-    if item.seqTag notin [yTagQuestionMark, yamlTag(seq[T])]:
-      raise newException(YamlConstructionError, "Wrong tag for " &
-          typetraits.name(seq[T]))
-    elif item.seqAnchor != yAnchorNone:
-      raise newException(YamlConstructionError, "Anchor on non-ref type")
-  constructObject(s, c, result)
-
-proc constructChild*[O](s: var YamlStream, c: ConstructionContext,
-                        result: var ref O) =
-  var e = s.peek()
-  if e.kind == yamlScalar:
-    if e.scalarTag == yTagNull or (e.scalarTag == yTagQuestionMark and
-        guessType(e.scalarContent) == yTypeNull):
-      result = nil
-      discard s.next()
-      return
-  elif e.kind == yamlAlias:
-    try:
-      result = cast[ref O](c.refs[e.aliasTarget])
-      discard s.next()
-      return
-    except KeyError: assert(false)
-  new(result)
-  template removeAnchor(anchor: var AnchorId) {.dirty.} =
-    if anchor != yAnchorNone:
-      assert(not c.refs.hasKey(anchor))
-      c.refs[anchor] = cast[pointer](result)
-      anchor = yAnchorNone
-
-  case e.kind
-  of yamlScalar: removeAnchor(e.scalarAnchor)
-  of yamlStartMap: removeAnchor(e.mapAnchor)
-  of yamlStartSeq: removeAnchor(e.seqAnchor)
-  else: assert(false)
-  s.peek = e
-  try: constructChild(s, c, result[])
-  except YamlConstructionError, YamlStreamError, AssertionError: raise
-  except Exception:
-    var e = newException(YamlStreamError, getCurrentExceptionMsg())
-    e.parent = getCurrentException()
-    raise e
-
-proc representChild*(value: string, ts: TagStyle, c: SerializationContext):
-    RawYamlStream =
-  if isNil(value):
-    result = iterator(): YamlStreamEvent =
-      yield scalarEvent("", yTagNimNilString)
-  else: result = representObject(value, ts, c, presentTag(string, ts))
-
-proc representChild*[T](value: seq[T], ts: TagStyle, c: SerializationContext):
-    RawYamlStream =
-  if isNil(value):
-    result = iterator(): YamlStreamEvent =
-      yield scalarEvent("", yTagNimNilSeq)
-  else: result = representObject(value, ts, c, presentTag(seq[T], ts))
-
-proc representChild*[O](value: ref O, ts: TagStyle, c: SerializationContext):
-    RawYamlStream =
-  if isNil(value):
-    result = iterator(): YamlStreamEvent =
-      yield scalarEvent("~", yTagNull)
-  elif c.style == asNone: result = representChild(value[], ts, c)
-  else:
-    let p = cast[pointer](value)
-    if c.refs.hasKey(p):
-      try:
-        if c.refs[p] == yAnchorNone:
-          c.refs[p] = c.nextAnchorId
-          c.nextAnchorId = AnchorId(int(c.nextAnchorId) + 1)
-      except KeyError: assert false, "Can never happen"
-      result = iterator(): YamlStreamEvent {.raises: [].} =
-        var event: YamlStreamEvent
-        try: event = aliasEvent(c.refs[p])
-        except KeyError: assert false, "Can never happen"
-        yield event
-      return
-    try:
-      if c.style == asAlways:
-        c.refs[p] = c.nextAnchorId
-        c.nextAnchorId = AnchorId(int(c.nextAnchorId) + 1)
-      else: c.refs[p] = yAnchorNone
-      let
-        a = if c.style == asAlways: c.refs[p] else: cast[AnchorId](p)
-        childTagStyle = if ts == tsAll: tsAll else: tsRootOnly
-      result = iterator(): YamlStreamEvent =
-        var child = representChild(value[], childTagStyle, c)
-        var first = child()
-        assert(not finished(child))
-        case first.kind
-        of yamlStartMap:
-          first.mapAnchor = a
-          if ts == tsNone: first.mapTag = yTagQuestionMark
-        of yamlStartSeq:
-          first.seqAnchor = a
-          if ts == tsNone: first.seqTag = yTagQuestionMark
-        of yamlScalar:
-          first.scalarAnchor = a
-          if ts == tsNone and guessType(first.scalarContent) != yTypeNull:
-            first.scalarTag = yTagQuestionMark
-        else: discard
-        yield first
-        while true:
-          let event = child()
-          if finished(child): break
-          yield event
-    except KeyError: assert false, "Can never happen"
-
-proc representChild*[O](value: O, ts: TagStyle,
-                        c: SerializationContext): RawYamlStream =
-  when compiles(implicitVariantObject(value)):
-    # todo: this would probably be nicer if constructed with a macro
-    var count = 0
-    for name, field in fieldPairs(value):
-      if count > 0:
-        result =
-            representChild(field, if ts == tsAll: tsAll else: tsRootOnly, c)
-      inc(count)
-    if count == 1:
-      result = iterator(): YamlStreamEvent =
-        yield scalarEvent("~", yTagNull)
-  else:
-    result = representObject(value, ts, c, if ts == tsNone:
-        yTagQuestionMark else: yamlTag(O))
-
-proc construct*[T](s: var YamlStream, target: var T) =
-  var context = newConstructionContext()
-  try:
-    var e = s.next()
-    assert(e.kind == yamlStartDoc)
-
-    constructChild(s, context, target)
-    e = s.next()
-    assert(e.kind == yamlEndDoc)
-  except YamlConstructionError:
-    raise (ref YamlConstructionError)(getCurrentException())
-  except YamlStreamError:
-    raise (ref YamlStreamError)(getCurrentException())
-  except AssertionError:
-    raise (ref AssertionError)(getCurrentException())
-  except Exception:
-    # may occur while calling s()
-    var ex = newException(YamlStreamError, "")
-    ex.parent = getCurrentException()
-    raise ex
-
-proc load*[K](input: Stream, target: var K) =
-  var
-    parser = newYamlParser(serializationTagLibrary)
-    events = parser.parse(input)
-  try: construct(events, target)
-  except YamlConstructionError:
-    var e = (ref YamlConstructionError)(getCurrentException())
-    e.line = parser.getLineNumber()
-    e.column = parser.getColNumber()
-    e.lineContent = parser.getLineContent()
-    raise e
-  except YamlStreamError:
-    let e = (ref YamlStreamError)(getCurrentException())
-    if e.parent of IOError: raise (ref IOError)(e.parent)
-    elif e.parent of YamlParserError: raise (ref YamlParserError)(e.parent)
-    else: assert false
-
-proc setAnchor(a: var AnchorId, q: var Table[pointer, AnchorId])
-    {.inline.} =
-  if a != yAnchorNone:
-    try: a = q[cast[pointer](a)]
-    except KeyError: assert false, "Can never happen"
-
-proc represent*[T](value: T, ts: TagStyle = tsRootOnly,
-                   a: AnchorStyle = asTidy): YamlStream =
-  var
-    context = newSerializationContext(a)
-    objStream = iterator(): YamlStreamEvent =
-      yield startDocEvent()
-      var events = representChild(value, ts, context)
-      while true:
-        let e = events()
-        if finished(events): break
-        yield e
-      yield endDocEvent()
-  if a == asTidy:
-    var objQueue = newSeq[YamlStreamEvent]()
-    try:
-      for event in objStream(): objQueue.add(event)
-    except Exception:
-      assert(false)
-    var backend = iterator(): YamlStreamEvent =
-      for i in countup(0, objQueue.len - 1):
-        var event = objQueue[i]
-        case event.kind
-        of yamlStartMap: event.mapAnchor.setAnchor(context.refs)
-        of yamlStartSeq: event.seqAnchor.setAnchor(context.refs)
-        of yamlScalar: event.scalarAnchor.setAnchor(context.refs)
-        else: discard
-        yield event
-    result = initYamlStream(backend)
-  else: result = initYamlStream(objStream)
-
-proc dump*[K](value: K, target: Stream, tagStyle: TagStyle = tsRootOnly,
-              anchorStyle: AnchorStyle = asTidy,
-              options: PresentationOptions = defaultPresentationOptions) =
-  var events = represent(value,
-      if options.style == psCanonical: tsAll else: tagStyle,
-      if options.style == psJson: asNone else: anchorStyle)
-  try: present(events, target, serializationTagLibrary, options)
-  except YamlStreamError:
-    # serializing object does not raise any errors, so we can ignore this
-    assert false, "Can never happen"

private/streams.nim

@@ -1,61 +0,0 @@
-#            NimYAML - YAML implementation in Nim
-#        (c) Copyright 2015 Felix Krause
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-
-proc initYamlStream*(backend: iterator(): YamlStreamEvent): YamlStream =
-  result.peeked = false
-  result.backend = backend
-
-proc next*(s: var YamlStream): YamlStreamEvent =
-  if s.peeked:
-    s.peeked = false
-    shallowCopy(result, s.cached)
-  else:
-    try:
-      shallowCopy(result, s.backend())
-      assert(not finished(s.backend))
-    except AssertionError: raise
-    except YamlStreamError:
-      let cur = getCurrentException()
-      var e = newException(YamlStreamError, cur.msg)
-      e.parent = cur.parent
-      raise e
-    except Exception:
-      let cur = getCurrentException()
-      var e = newException(YamlStreamError, cur.msg)
-      e.parent = cur
-      raise e
-
-proc peek*(s: var YamlStream): YamlStreamEvent =
-  if not s.peeked:
-    s.cached = s.next()
-    s.peeked = true
-  shallowCopy(result, s.cached)
-
-proc `peek=`*(s: var YamlStream, value: YamlStreamEvent) =
-  s.cached = value
-  s.peeked = true
-    
-proc finished*(s: var YamlStream): bool =
-  if s.peeked: result = false
-  else:
-    try:
-      s.cached = s.backend()
-      if finished(s.backend): result = true
-      else:
-        s.peeked = true
-        result = false
-    except AssertionError: raise
-    except YamlStreamError:
-      let cur = getCurrentException()
-      var e = newException(YamlStreamError, cur.msg)
-      e.parent = cur.parent
-      raise e
-    except Exception:
-      let cur = getCurrentException()
-      echo cur.getStackTrace()
-      var e = newException(YamlStreamError, cur.msg)
-      e.parent = cur
-      raise e

private/tagLibrary.nim

@@ -1,83 +0,0 @@
-#            NimYAML - YAML implementation in Nim
-#        (c) Copyright 2015 Felix Krause
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-
-proc `$`*(id: TagId): string =
-  case id
-  of yTagQuestionMark: "?"
-  of yTagExclamationMark: "!"
-  of yTagString: "!!str"
-  of yTagSequence: "!!seq"
-  of yTagMapping: "!!map"
-  of yTagNull: "!!null"
-  of yTagBoolean: "!!bool"
-  of yTagInteger: "!!int"
-  of yTagFloat: "!!float"
-  of yTagOrderedMap: "!!omap"
-  of yTagPairs: "!!pairs"
-  of yTagSet: "!!set"
-  of yTagBinary: "!!binary"
-  of yTagMerge: "!!merge"
-  of yTagTimestamp: "!!timestamp"
-  of yTagValue: "!!value"
-  of yTagYaml: "!!yaml"
-  of yTagNimField: "!nim:field"
-  else: "<" & $cast[int](id) & ">"
-
-proc initTagLibrary*(): TagLibrary =
-  new(result)
-  result.tags = initTable[string, TagId]()
-  result.secondaryPrefix = yamlTagRepositoryPrefix
-  result.nextCustomTagId = yFirstCustomTagId
-
-proc registerUri*(tagLib: TagLibrary, uri: string): TagId =
-  tagLib.tags[uri] = tagLib.nextCustomTagId
-  result = tagLib.nextCustomTagId
-  tagLib.nextCustomTagId = cast[TagId](cast[int](tagLib.nextCustomTagId) + 1)
-    
-proc uri*(tagLib: TagLibrary, id: TagId): string =
-  for iUri, iId in tagLib.tags.pairs:
-    if iId == id: return iUri
-  raise newException(KeyError, "Unknown tag id: " & $id)
-
-proc initFailsafeTagLibrary(): TagLibrary =
-  result = initTagLibrary()
-  result.tags["!"] = yTagExclamationMark
-  result.tags["?"] = yTagQuestionMark
-  result.tags["tag:yaml.org,2002:str"] = yTagString
-  result.tags["tag:yaml.org,2002:seq"] = yTagSequence
-  result.tags["tag:yaml.org,2002:map"] = yTagMapping
-
-proc initCoreTagLibrary(): TagLibrary =
-  result = initFailsafeTagLibrary()
-  result.tags["tag:yaml.org,2002:null"]  = yTagNull
-  result.tags["tag:yaml.org,2002:bool"]  = yTagBoolean
-  result.tags["tag:yaml.org,2002:int"]   = yTagInteger
-  result.tags["tag:yaml.org,2002:float"] = yTagFloat
-
-proc initExtendedTagLibrary(): TagLibrary =
-  result = initCoreTagLibrary()
-  result.tags["tag:yaml.org,2002:omap"]      = yTagOrderedMap
-  result.tags["tag:yaml.org,2002:pairs"]     = yTagPairs
-  result.tags["tag:yaml.org,2002:binary"]    = yTagBinary
-  result.tags["tag:yaml.org,2002:merge"]     = yTagMerge
-  result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp
-  result.tags["tag:yaml.org,2002:value"]     = yTagValue
-  result.tags["tag:yaml.org,2002:yaml"]      = yTagYaml
-
-proc initSerializationTagLibrary(): TagLibrary =
-  result = initTagLibrary()
-  result.tags["!"] = yTagExclamationMark
-  result.tags["?"] = yTagQuestionMark
-  result.tags["tag:yaml.org,2002:str"]       = yTagString
-  result.tags["tag:yaml.org,2002:null"]      = yTagNull
-  result.tags["tag:yaml.org,2002:bool"]      = yTagBoolean
-  result.tags["tag:yaml.org,2002:float"]     = yTagFloat
-  result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp
-  result.tags["tag:yaml.org,2002:value"]     = yTagValue
-  result.tags["tag:yaml.org,2002:binary"]    = yTagBinary
-  result.tags["!nim:field"]                  = yTagNimField
-  result.tags["!nim:nil:string"]             = yTagNimNilString
-  result.tags["!nim:nil:seq"]                = yTagNimNilSeq

server/server.nim

@@ -6,7 +6,7 @@
 
 import jester, asyncdispatch, json, streams, strutils
 import packages.docutils.rstgen, packages.docutils.highlite
-import yaml
+import ../yaml
 
 routes:
   get "/":
@@ -16,7 +16,9 @@ routes:
     var
       style: PresentationStyle
       resultNode = newJObject()
-      tokens = false
+      msg: string
+      retStatus = Http200
+      contentType = "application/json"
     headers["Access-Control-Allow-Origin"] = "*"
     headers["Pragma"] = "no-cache"
     headers["Cache-Control"] = "no-cache"
@@ -36,14 +38,17 @@ routes:
         for event in events: output.add($event & "\n")
         resultNode["code"] = %0
         resultNode["output"] = %output
-        resp resultNode.pretty, "application/json"
-        tokens = true
-      if not tokens:
+        msg = resultNode.pretty
+      else:
+        retStatus = Http400
+        msg = "Invalid style: " & escape(@"style")
+        contentType = "text/plain;charset=utf8"
+      if isNil(msg):
         var
           output = newStringStream()
           highlighted = ""
         transform(newStringStream(@"input"), output, defineOptions(style))
-        
+
         # syntax highlighting (stolen and modified from stlib's rstgen)
         var g: GeneralTokenizer
         g.initGeneralTokenizer(output.data)
@@ -60,7 +65,7 @@ routes:
 
         resultNode["code"] = %0
         resultNode["output"] = %highlighted
-        resp resultNode.pretty, "application/json"
+        msg = resultNode.pretty
     except YamlParserError:
       let e = (ref YamlParserError)(getCurrentException())
       resultNode["code"] = %1
@@ -68,17 +73,17 @@ routes:
       resultNode["column"] = %e.column
       resultNode["message"] = %e.msg
       resultNode["detail"] = %e.lineContent
-      resp resultNode.pretty, "application/json"
+      msg = resultNode.pretty
     except YamlPresenterJsonError:
       let e = getCurrentException()
       resultNode["code"] = %2
       resultNode["message"] = %e.msg
-      headers["Content-Type"] = "application/json"
-      resp resultNode.pretty, "application/json"
+      msg = resultNode.pretty
     except:
       let e = getCurrentException()
-      let msg = "Name: " & $e.name & "\nMessage: " & e.msg &
-                "\nTrace:\n" & e.getStackTrace
-      resp Http500, msg, "text/plain;charset=utf-8"
-
+      msg = "Name: " & $e.name & "\nMessage: " & e.msg &
+          "\nTrace:\n" & e.getStackTrace
+      retStatus = Http500
+      contentType = "text/plain;charset=utf-8"
+    resp retStatus, msg, contentType
 runForever()

test/tdom.nim

@@ -5,21 +5,21 @@
 #    distribution, for details about the copyright.
 
 import "../yaml"
-import unittest, common
+import unittest, commonTestUtils, streams
 
 suite "DOM":
-  test "DOM: Composing simple Scalar":
+  test "Composing simple Scalar":
     let
       input  = newStringStream("scalar")
       result = loadDOM(input)
     assert result.root.kind == yScalar
     assert result.root.content == "scalar"
     assert result.root.tag == "?"
-  test "DOM: Serializing simple Scalar":
+  test "Serializing simple Scalar":
     let input = initYamlDoc(newYamlNode("scalar"))
     var result = serialize(input, initExtendedTagLibrary())
     ensure(result, startDocEvent(), scalarEvent("scalar"), endDocEvent())
-  test "DOM: Composing sequence":
+  test "Composing sequence":
     let
       input = newStringStream("- !!str a\n- !!bool no")
       result = loadDOM(input)
@@ -32,7 +32,7 @@ suite "DOM":
     assert result.root.children[1].kind == yScalar
     assert result.root.children[1].tag == "tag:yaml.org,2002:bool"
     assert result.root.children[1].content == "no"
-  test "DOM: Serializing sequence":
+  test "Serializing sequence":
     let input = initYamlDoc(newYamlNode([
         newYamlNode("a", "tag:yaml.org,2002:str"),
         newYamlNode("no", "tag:yaml.org,2002:bool")]))
@@ -40,7 +40,7 @@ suite "DOM":
     ensure(result, startDocEvent(), startSeqEvent(),
            scalarEvent("a", yTagString), scalarEvent("no", yTagBoolean),
            endSeqEvent(), endDocEvent())
-  test "DOM: Composing mapping":
+  test "Composing mapping":
     let
       input = newStringStream("--- !!map\n!foo bar: [a, b]")
       result = loadDOM(input)
@@ -52,7 +52,7 @@ suite "DOM":
     assert result.root.pairs[0].key.content == "bar"
     assert result.root.pairs[0].value.kind == ySequence
     assert result.root.pairs[0].value.children.len == 2
-  test "DOM: Serializing mapping":
+  test "Serializing mapping":
     let input = initYamlDoc(newYamlNode([
         (key: newYamlNode("bar"), value: newYamlNode([newYamlNode("a"),
                                                       newYamlNode("b")]))]))
@@ -60,7 +60,7 @@ suite "DOM":
     ensure(result, startDocEvent(), startMapEvent(), scalarEvent("bar"),
            startSeqEvent(), scalarEvent("a"), scalarEvent("b"),
            endSeqEvent(), endMapEvent(), endDocEvent())
-  test "DOM: Composing with anchors":
+  test "Composing with anchors":
     let
       input = newStringStream("- &a foo\n- &b bar\n- *a\n- *b")
       result = loadDOM(input)
@@ -72,7 +72,7 @@ suite "DOM":
     assert result.root.children[1].content == "bar"
     assert result.root.children[0] == result.root.children[2]
     assert result.root.children[1] == result.root.children[3]
-  test "DOM: Serializing with anchors":
+  test "Serializing with anchors":
     let
       a = newYamlNode("a")
       b = newYamlNode("b")
@@ -83,7 +83,7 @@ suite "DOM":
            scalarEvent("b", anchor=1.AnchorId), scalarEvent("c"),
            aliasEvent(0.AnchorId), aliasEvent(1.AnchorId), endSeqEvent(),
            endDocEvent())
-  test "DOM: Serializing with all anchors":
+  test "Serializing with all anchors":
     let
       a = newYamlNode("a")
       input = initYamlDoc(newYamlNode([a, newYamlNode("b"), a]))

test/testEventParser.nim

@@ -5,19 +5,19 @@
 #    distribution, for details about the copyright.
 
 import "../yaml"
-import lexbase
+import lexbase, streams, tables
 
 type
   LexerToken = enum
     plusStr, minusStr, plusDoc, minusDoc, plusMap, minusMap, plusSeq, minusSeq,
     eqVal, eqAli, chevTag, andAnchor, quotContent, colonContent, noToken
-    
+
   StreamPos = enum
     beforeStream, inStream, afterStream
 
   EventLexer = object of BaseLexer
     content: string
-    
+
   EventStreamError = object of Exception
 
 proc nextToken(lex: var EventLexer): LexerToken =
@@ -156,7 +156,7 @@ template setCurAnchor(val: AnchorId) =
   of yamlAlias: curEvent.aliasTarget = val
   else: raise newException(EventStreamError,
                            $curEvent.kind & " may not have an anchor")
- 
+
 template eventStart(k: YamlStreamEventKind) {.dirty.} =
   assertInStream()
   yieldEvent()

test/tests.nim

@@ -4,4 +4,4 @@
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 
-import constructingJson, serializing, dom
+import tlex, tjson, tserialization, tdom

test/tjson.nim

@@ -6,28 +6,36 @@
 
 import "../yaml"
 
-import unittest, json
+import unittest, json, streams
 
 proc wc(line, column: int, lineContent: string, message: string) =
   echo "Warning (", line, ",", column, "): ", message, "\n", lineContent
 
 proc ensureEqual(yamlIn, jsonIn: string) =
-  var
-    parser = newYamlParser(initCoreTagLibrary(), wc)
-    s = parser.parse(newStringStream(yamlIn))
-    yamlResult = constructJson(s)
-    jsonResult = parseJson(jsonIn)
-  assert yamlResult.len == 1
-  assert(jsonResult == yamlResult[0])
+  try:
+    var
+      parser = newYamlParser(initCoreTagLibrary(), wc)
+      s = parser.parse(newStringStream(yamlIn))
+      yamlResult = constructJson(s)
+      jsonResult = parseJson(jsonIn)
+    assert yamlResult.len == 1
+    assert(jsonResult == yamlResult[0], "Expected: " & $jsonResult & ", got: " &
+        $yamlResult[0])
+  except YamlStreamError:
+    let e = (ref YamlParserError)(getCurrentException().parent)
+    echo "error occurred: " & e.msg
+    echo "line: ", e.line, ", column: ", e.column
+    echo e.lineContent
+    raise e
 
 suite "Constructing JSON":
-  test "Constructing JSON: Simple Sequence":
+  test "Simple Sequence":
     ensureEqual("- 1\n- 2\n- 3", "[1, 2, 3]")
-    
-  test "Constructing JSON: Simple Map":
+
+  test "Simple Map":
     ensureEqual("a: b\nc: d", """{"a": "b", "c": "d"}""")
-    
-  test "Constructing JSON: Complex Structure":
+
+  test "Complex Structure":
     ensureEqual("""
 %YAML 1.2
 ---

test/tlex.nim

@@ -0,0 +1,262 @@
+import ../private/lex
+
+import unittest, strutils
+
+const tokensWithValue =
+    {ltScalarPart, ltQuotedScalar, ltYamlVersion, ltTagShorthand, ltTagUri,
+     ltUnknownDirective, ltUnknownDirectiveParams, ltLiteralTag, ltAnchor,
+     ltAlias, ltBlockScalar}
+
+type
+  TokenWithValue = object
+    case kind: LexerToken
+    of tokensWithValue:
+      value: string
+    of ltIndentation:
+      indentation: int
+    of ltTagHandle:
+      handle, suffix: string
+    else: discard
+
+proc actualRepr(lex: YamlLexer, t: LexerToken): string =
+  result = $t
+  case t
+  of tokensWithValue + {ltTagHandle}:
+    result.add("(" & escape(lex.buf) & ")")
+  of ltIndentation:
+    result.add("(" & $lex.indentation & ")")
+  else: discard
+
+proc assertEquals(input: string, expected: varargs[TokenWithValue]) =
+  let lex = newYamlLexer(input)
+  var
+    i = 0
+    blockScalarEnd = -1
+    flowDepth = 0
+  for expectedToken in expected:
+    inc(i)
+    try:
+      lex.next()
+      doAssert lex.cur == expectedToken.kind, "Wrong token kind at #" & $i &
+          ": Expected " & $expectedToken.kind & ", got " &
+          lex.actualRepr(lex.cur)
+      case expectedToken.kind
+      of tokensWithValue:
+        doAssert lex.buf == expectedToken.value, "Wrong token content at #" &
+            $i & ": Expected " & escape(expectedToken.value) &
+            ", got " & escape(lex.buf)
+        lex.buf = ""
+      of ltIndentation:
+        doAssert lex.indentation == expectedToken.indentation,
+            "Wrong indentation length at #" & $i & ": Expected " &
+            $expectedToken.indentation & ", got " & $lex.indentation
+        if lex.indentation <= blockScalarEnd:
+          lex.endBlockScalar()
+          blockScalarEnd = -1
+      of ltBraceOpen, ltBracketOpen:
+        inc(flowDepth)
+        if flowDepth == 1: lex.setFlow(true)
+      of ltBraceClose, ltBracketClose:
+        dec(flowDepth)
+        if flowDepth == 0: lex.setFlow(false)
+      of ltTagHandle:
+        let
+          handle = lex.buf.substr(0, lex.shorthandEnd)
+          suffix = lex.buf.substr(lex.shorthandEnd + 1)
+        doAssert handle == expectedToken.handle,
+            "Wrong handle at #" & $i & ": Expected " & expectedToken.handle &
+            ", got " & handle
+        doAssert suffix == expectedToken.suffix,
+            "Wrong suffix at #" & $i & ": Expected " & expectedToken.suffix &
+            ", got " & suffix
+        lex.buf = ""
+      else: discard
+    except YamlLexerError:
+      let e = (ref YamlLexerError)(getCurrentException())
+      echo "Error at line " & $e.line & ", column " & $e.column & ":"
+      echo e.lineContent
+      assert false
+
+proc assertLookahead(input: string, expected: bool, tokensBefore: int = 1) =
+  let lex = newYamlLexer(input)
+  var flowDepth = 0
+  for i in 0..tokensBefore:
+    lex.next()
+    case lex.cur
+    of ltBraceOpen, ltBracketOpen:
+      inc(flowDepth)
+      if flowDepth == 1: lex.setFlow(true)
+    of ltBraceClose, ltBracketClose:
+      dec(flowDepth)
+      if flowDepth == 0: lex.setFlow(false)
+    else: discard
+  doAssert lex.isImplicitKeyStart() == expected
+
+proc i(indent: int): TokenWithValue =
+  TokenWithValue(kind: ltIndentation, indentation: indent)
+proc sp(v: string): TokenWithValue =
+  TokenWithValue(kind: ltScalarPart, value: v)
+proc qs(v: string): TokenWithValue =
+  TokenWithValue(kind: ltQuotedScalar, value: v)
+proc se(): TokenWithValue = TokenWithValue(kind: ltStreamEnd)
+proc mk(): TokenWithValue = TokenWithValue(kind: ltMapKeyInd)
+proc mv(): TokenWithValue = TokenWithValue(kind: ltMapValInd)
+proc si(): TokenWithValue = TokenWithValue(kind: ltSeqItemInd)
+proc dy(): TokenWithValue = TokenWithValue(kind: ltYamlDirective)
+proc dt(): TokenWithValue = TokenWithValue(kind: ltTagDirective)
+proc du(v: string): TokenWithValue =
+  TokenWithValue(kind: ltUnknownDirective, value: v)
+proc dp(v: string): TokenWithValue =
+  TokenWithValue(kind: ltUnknownDirectiveParams, value: v)
+proc yv(v: string): TokenWithValue =
+  TokenWithValue(kind: ltYamlVersion, value: v)
+proc ts(v: string): TokenWithValue =
+  TokenWithValue(kind: ltTagShorthand, value: v)
+proc tu(v: string): TokenWithValue =
+  TokenWithValue(kind: ltTagUri, value: v)
+proc dirE(): TokenWithValue = TokenWithValue(kind: ltDirectivesEnd)
+proc docE(): TokenWithValue = TokenWithValue(kind: ltDocumentEnd)
+proc bsh(): TokenWithValue = TokenWithValue(kind: ltBlockScalarHeader)
+proc bs(v: string): TokenWithValue =
+  TokenWithValue(kind: ltBlockScalar, value: v)
+proc el(): TokenWithValue = TokenWithValue(kind: ltEmptyLine)
+proc ao(): TokenWithValue = TokenWithValue(kind: ltBracketOpen)
+proc ac(): TokenWithValue = TokenWithValue(kind: ltBracketClose)
+proc oo(): TokenWithValue = TokenWithValue(kind: ltBraceOpen)
+proc oc(): TokenWithValue = TokenWithValue(kind: ltBraceClose)
+proc c(): TokenWithValue = TokenWithValue(kind: ltComma)
+proc th(handle, suffix: string): TokenWithValue =
+  TokenWithValue(kind: ltTagHandle, handle: handle, suffix: suffix)
+proc lt(v: string): TokenWithValue =
+  TokenWithValue(kind: ltLiteralTag, value: v)
+proc an(v: string): TokenWithValue = TokenWithValue(kind: ltAnchor, value: v)
+proc al(v: string): TokenWithValue = TokenWithValue(kind: ltAlias, value: v)
+
+suite "Lexer":
+  test "Empty document":
+    assertEquals("", se())
+
+  test "Single-line scalar":
+    assertEquals("scalar", i(0), sp("scalar"), se())
+
+  test "Multiline scalar":
+    assertEquals("scalar\l  line two", i(0), sp("scalar"), i(2),
+        sp("line two"), se())
+
+  test "Single-line mapping":
+    assertEquals("key: value", i(0), sp("key"), mv(), sp("value"), se())
+
+  test "Multiline mapping":
+    assertEquals("key:\n  value", i(0), sp("key"), mv(), i(2), sp("value"),
+        se())
+
+  test "Explicit mapping":
+    assertEquals("? key\n: value", i(0), mk(), sp("key"), i(0), mv(),
+        sp("value"), se())
+
+  test "Sequence":
+    assertEquals("- a\n- b", i(0), si(), sp("a"), i(0), si(), sp("b"), se())
+
+  test "Single-line single-quoted scalar":
+    assertEquals("'quoted  scalar'", i(0), qs("quoted  scalar"), se())
+
+  test "Multiline single-quoted scalar":
+    assertEquals("'quoted\l  multi line  \l\lscalar'", i(0),
+    qs("quoted multi line\lscalar"), se())
+
+  test "Single-line double-quoted scalar":
+    assertEquals("\"quoted  scalar\"", i(0), qs("quoted  scalar"), se())
+
+  test "Multiline double-quoted scalar":
+    assertEquals("\"quoted\l  multi line  \l\lscalar\"", i(0),
+    qs("quoted multi line\lscalar"), se())
+
+  test "Escape sequences":
+    assertEquals(""""\n\x31\u0032\U00000033"""", i(0), qs("\l123"), se())
+
+  test "Directives":
+    assertEquals("%YAML 1.2\n---\n%TAG\n...\n\n%TAG ! example.html",
+        dy(), yv("1.2"), dirE(), i(0), sp("%TAG"), i(0), docE(), dt(),
+        ts("!"), tu("example.html"), se())
+
+  test "Markers and Unknown Directive":
+    assertEquals("---\n---\n...\n%UNKNOWN warbl", dirE(), dirE(), i(0),
+        docE(), du("UNKNOWN"), dp("warbl"), se())
+
+  test "Block scalar":
+    assertEquals("|\l  a\l\l  b\l # comment", i(0), bsh(), bs("a\l\lb\l"), se())
+
+  test "Block Scalars":
+    assertEquals("one : >2-\l   foo\l  bar\ltwo: |+\l bar\l  baz", i(0),
+        sp("one"), mv(), bsh(), bs(" foo\lbar"), i(0), sp("two"), mv(), bsh(),
+        bs("bar\l baz"), se())
+
+  test "Flow indicators":
+    assertEquals("bla]: {c: d, [e]: f}", i(0), sp("bla]"), mv(), oo(), sp("c"),
+        mv(), sp("d"), c(), ao(), sp("e"), ac(), mv(), sp("f"), oc(), se())
+
+  test "Adjacent map values in flow style":
+    assertEquals("{\"foo\":bar, [1]\l:egg}", i(0), oo(), qs("foo"), mv(),
+        sp("bar"), c(), ao(), sp("1"), ac(), mv(), sp("egg"), oc(), se())
+
+  test "Tag handles":
+    assertEquals("- !!str string\l- !local local\l- !e! e", i(0), si(),
+        th("!!", "str"), sp("string"), i(0), si(), th("!", "local"),
+        sp("local"), i(0), si(), th("!e!", ""), sp("e"), se())
+
+  test "Literal tag handle":
+    assertEquals("!<tag:yaml.org,2002:str> string", i(0),
+        lt("tag:yaml.org,2002:str"), sp("string"), se())
+
+  test "Anchors and aliases":
+    assertEquals("&a foo: {&b b: *a, *b : c}", i(0), an("a"), sp("foo"), mv(),
+        oo(), an("b"), sp("b"), mv(), al("a"), c(), al("b"), mv(), sp("c"),
+        oc(), se())
+
+  test "Empty lines":
+    assertEquals("""block: foo
+
+  bar
+
+    baz
+flow: {
+  foo
+
+  bar: baz
+
+
+  mi
+}""", i(0), sp("block"), mv(), sp("foo"), el(), i(2), sp("bar"), el(), i(4),
+    sp("baz"), i(0), sp("flow"), mv(), oo(), sp("foo"), el(), sp("bar"), mv(),
+    sp("baz"), el(), el(), sp("mi"), oc(), se())
+
+suite "Lookahead":
+  test "Simple Scalar":
+    assertLookahead("abcde", false)
+
+  test "Simple Mapping":
+    assertLookahead("a: b", true)
+
+  test "Colon inside plain scalar":
+    assertLookahead("abc:de", false)
+
+  test "Colon inside quoted scalar":
+    assertLookahead("'abc: de'", false)
+
+  test "Quotes inside plain scalar":
+    assertLookahead("abc\'\"de: foo", true)
+
+  test "Flow indicator inside plain scalar":
+    assertLookahead("abc}]: de", true)
+
+  test "Complex key":
+    assertLookahead("[1, 2, \"3\"]: foo", true)
+
+  test "Flow value":
+    assertLookahead("{a: b}", false)
+
+  test "In flow context":
+    assertLookahead("[ abcde]: foo", false, 2)
+
+  test "Adjacent value":
+    assertLookahead("[\"abc\":de]", true, 2)

test/tserialization.nim

@@ -5,31 +5,31 @@
 #    distribution, for details about the copyright.
 
 import "../yaml"
-import unittest, strutils
+import unittest, strutils, streams, tables
 
 type
   MyTuple = tuple
     str: string
     i: int32
     b: bool
-    
+
   TrafficLight = enum
     tlGreen, tlYellow, tlRed
-    
+
   Person = object
     firstnamechar: char
     surname: string
     age: int32
-    
+
   Node = object
     value: string
     next: ref Node
-    
+
   BetterInt = distinct int
-  
+
   AnimalKind = enum
     akCat, akDog
-  
+
   Animal = object
     name: string
     case kind: AnimalKind
@@ -39,22 +39,21 @@ type
       barkometer: int
 
 proc `$`(v: BetterInt): string {.borrow.}
-proc `==`(l, r: BetterInt): bool {.borrow.}
+proc `==`(left, right: BetterInt): bool {.borrow.}
 
 setTagUri(TrafficLight, "!tl")
 setTagUri(Node, "!example.net:Node")
 setTagUri(BetterInt, "!test:BetterInt")
 
 proc representObject*(value: BetterInt, ts: TagStyle = tsNone,
-    c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} =
-  result = iterator(): YamlStreamEvent =
-    var
-      val = $value
-      i = val.len - 3
-    while i > 0:
-      val.insert("_", i)
-      i -= 3
-    yield scalarEvent(val, tag, yAnchorNone)
+    c: SerializationContext, tag: TagId) {.raises: [].} =
+  var
+    val = $value
+    i = val.len - 3
+  while i > 0:
+    val.insert("_", i)
+    i -= 3
+  c.put(scalarEvent(val, tag, yAnchorNone))
 
 proc constructObject*(s: var YamlStream, c: ConstructionContext,
                       result: var BetterInt)
@@ -70,127 +69,179 @@ template assertStringEqual(expected, actual: string) =
       echo "expected:\n", expected, "\nactual:\n", actual
       assert(false)
 
+template expectConstructionError(li, co: int, message: string, body: typed) =
+  try:
+    body
+    echo "Expected YamlConstructionError, but none was raised!"
+    fail()
+  except YamlConstructionError:
+    let e = (ref YamlConstructionError)(getCurrentException())
+    doAssert li == e.line, "Expected error line " & $li & ", was " & $e.line
+    doAssert co == e.column, "Expected error column " & $co & ", was " & $e.column
+    doAssert message == e.msg, "Expected error message " & escape(message) &
+        ", got " & escape(e.msg)
+
 proc newNode(v: string): ref Node =
   new(result)
   result.value = v
   result.next = nil
 
-suite "Serialization":
-  setup:
-    let blockOnly = defineOptions(style=psBlockOnly)
+let blockOnly = defineOptions(style=psBlockOnly)
 
-  test "Serialization: Load integer without fixed length":
+suite "Serialization":
+  test "Load integer without fixed length":
     var input = newStringStream("-4247")
     var result: int
     load(input, result)
-    assert result == -4247, "result is " & $result 
-    
+    assert result == -4247, "result is " & $result
+
     input = newStringStream($(int64(int32.high) + 1'i64))
     var gotException = false
     try: load(input, result)
     except: gotException = true
     assert gotException, "Expected exception, got none."
 
-  test "Serialization: Dump integer without fixed length":
+  test "Dump integer without fixed length":
     var input = -4247
-    var output = newStringStream()
-    dump(input, output, tsNone, asTidy, blockOnly)
-    assertStringEqual "%YAML 1.2\n--- \n\"-4247\"", output.data
-    
+    var output = dump(input, tsNone, asTidy, blockOnly)
+    assertStringEqual "%YAML 1.2\n--- \n\"-4247\"", output
+
     when sizeof(int) == sizeof(int64):
       input = int(int32.high) + 1
       var gotException = false
-      try: dump(input, output, tsNone, asTidy, blockOnly)
+      try: output = dump(input, tsNone, asTidy, blockOnly)
       except: gotException = true
       assert gotException, "Expected exception, got none."
 
-  test "Serialization: Load nil string":
+  test "Load Hex byte (0xFF)":
+    let input = newStringStream("0xFF")
+    var result: byte
+    load(input, result)
+    assert(result == 255)
+
+  test "Load Hex byte (0xC)":
+    let input = newStringStream("0xC")
+    var result: byte
+    load(input, result)
+    assert(result == 12)
+
+  test "Load Octal byte (0o14)":
+    let input = newStringStream("0o14")
+    var result: byte
+    load(input, result)
+    assert(result == 12)
+
+  test "Load byte (14)":
+    let input = newStringStream("14")
+    var result: byte
+    load(input, result)
+    assert(result == 14)
+
+  test "Load Hex int (0xFF)":
+    let input = newStringStream("0xFF")
+    var result: int
+    load(input, result)
+    assert(result == 255)
+
+  test "Load Hex int (0xC)":
+    let input = newStringStream("0xC")
+    var result: int
+    load(input, result)
+    assert(result == 12)
+
+  test "Load Octal int (0o14)":
+    let input = newStringStream("0o14")
+    var result: int
+    load(input, result)
+    assert(result == 12)
+
+  test "Load int (14)":
+    let input = newStringStream("14")
+    var result: int
+    load(input, result)
+    assert(result == 14)
+
+  test "Load nil string":
     let input = newStringStream("!nim:nil:string \"\"")
     var result: string
     load(input, result)
     assert isNil(result)
-  
-  test "Serialization: Dump nil string":
-    let input: string = nil
-    var output = newStringStream()
-    dump(input, output, tsNone, asTidy, blockOnly)
-    assertStringEqual "%YAML 1.2\n--- \n!nim:nil:string \"\"", output.data
 
-  test "Serialization: Load string sequence":
+  test "Dump nil string":
+    let input: string = nil
+    var output = dump(input, tsNone, asTidy, blockOnly)
+    assertStringEqual "%YAML 1.2\n--- \n!nim:nil:string \"\"", output
+
+  test "Load string sequence":
     let input = newStringStream(" - a\n - b")
     var result: seq[string]
     load(input, result)
     assert result.len == 2
     assert result[0] == "a"
     assert result[1] == "b"
-    
-  test "Serialization: Dump string sequence":
+
+  test "Dump string sequence":
     var input = @["a", "b"]
-    var output = newStringStream()
-    dump(input, output, tsNone, asTidy, blockOnly)
-    assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output.data
-  
-  test "Serialization: Load nil seq":
+    var output = dump(input, tsNone, asTidy, blockOnly)
+    assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output
+
+  test "Load nil seq":
     let input = newStringStream("!nim:nil:seq \"\"")
     var result: seq[int]
     load(input, result)
     assert isNil(result)
-  
-  test "Serialization: Dump nil seq":
+
+  test "Dump nil seq":
     let input: seq[int] = nil
-    var output = newStringStream()
-    dump(input, output, tsNone, asTidy, blockOnly)
-    assertStringEqual "%YAML 1.2\n--- \n!nim:nil:seq \"\"", output.data
-  
-  test "Serialization: Load char set":
+    var output = dump(input, tsNone, asTidy, blockOnly)
+    assertStringEqual "%YAML 1.2\n--- \n!nim:nil:seq \"\"", output
+
+  test "Load char set":
     let input = newStringStream("- a\n- b")
     var result: set[char]
     load(input, result)
     assert result.card == 2
     assert 'a' in result
     assert 'b' in result
-  
-  test "Serialization: Dump char set":
+
+  test "Dump char set":
     var input = {'a', 'b'}
-    var output = newStringStream()
-    dump(input, output, tsNone, asTidy, blockOnly)
-    assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output.data
-  
-  test "Serialization: Load array":
+    var output = dump(input, tsNone, asTidy, blockOnly)
+    assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output
+
+  test "Load array":
     let input = newStringStream("- 23\n- 42\n- 47")
     var result: array[0..2, int32]
     load(input, result)
     assert result[0] == 23
     assert result[1] == 42
     assert result[2] == 47
-  
-  test "Serialization: Dump array":
+
+  test "Dump array":
     let input = [23'i32, 42'i32, 47'i32]
-    var output = newStringStream()
-    dump(input, output, tsNone, asTidy, blockOnly)
-    assertStringEqual "%YAML 1.2\n--- \n- 23\n- 42\n- 47", output.data
-    
-  test "Serialization: Load Table[int, string]":
+    var output = dump(input, tsNone, asTidy, blockOnly)
+    assertStringEqual "%YAML 1.2\n--- \n- 23\n- 42\n- 47", output
+
+  test "Load Table[int, string]":
     let input = newStringStream("23: dreiundzwanzig\n42: zweiundvierzig")
     var result: Table[int32, string]
     load(input, result)
     assert result.len == 2
     assert result[23] == "dreiundzwanzig"
     assert result[42] == "zweiundvierzig"
-    
-  test "Serialization: Dump Table[int, string]":
+
+  test "Dump Table[int, string]":
     var input = initTable[int32, string]()
     input[23] = "dreiundzwanzig"
     input[42] = "zweiundvierzig"
-    var output = newStringStream()
-    dump(input, output, tsNone, asTidy, blockOnly)
+    var output = dump(input, tsNone, asTidy, blockOnly)
     assertStringEqual("%YAML 1.2\n--- \n23: dreiundzwanzig\n42: zweiundvierzig",
-        output.data)
-    
-  test "Serialization: Load OrderedTable[tuple[int32, int32], string]":
+        output)
+
+  test "Load OrderedTable[tuple[int32, int32], string]":
     let input = newStringStream("- {a: 23, b: 42}: drzw\n- {a: 13, b: 47}: drsi")
     var result: OrderedTable[tuple[a, b: int32], string]
-    load(input, result) 
+    load(input, result)
     var i = 0
     for key, value in result.pairs:
       case i
@@ -202,13 +253,12 @@ suite "Serialization":
         assert value == "drsi"
       else: assert false
       i.inc()
-    
-  test "Serialization: Dump OrderedTable[tuple[int32, int32], string]":
+
+  test "Dump OrderedTable[tuple[int32, int32], string]":
     var input = initOrderedTable[tuple[a, b: int32], string]()
     input.add((a: 23'i32, b: 42'i32), "dreiundzwanzigzweiundvierzig")
     input.add((a: 13'i32, b: 47'i32), "dreizehnsiebenundvierzig")
-    var output = newStringStream()
-    dump(input, output, tsRootOnly, asTidy, blockOnly)
+    var output = dump(input, tsRootOnly, asTidy, blockOnly)
     assertStringEqual("""%YAML 1.2
 --- !nim:tables:OrderedTable(nim:tuple(nim:system:int32,nim:system:int32),tag:yaml.org,2002:str) 
 - 
@@ -220,9 +270,9 @@ suite "Serialization":
   ? 
     a: 13
     b: 47
-  : dreizehnsiebenundvierzig""", output.data)
-    
-  test "Serialization: Load Sequences in Sequence":
+  : dreizehnsiebenundvierzig""", output)
+
+  test "Load Sequences in Sequence":
     let input = newStringStream(" - [1, 2, 3]\n - [4, 5]\n - [6]")
     var result: seq[seq[int32]]
     load(input, result)
@@ -230,31 +280,28 @@ suite "Serialization":
     assert result[0] == @[1.int32, 2.int32, 3.int32]
     assert result[1] == @[4.int32, 5.int32]
     assert result[2] == @[6.int32]
-    
-  test "Serialization: Dump Sequences in Sequence":
+
+  test "Dump Sequences in Sequence":
     let input = @[@[1.int32, 2.int32, 3.int32], @[4.int32, 5.int32], @[6.int32]]
-    var output = newStringStream()
-    dump(input, output, tsNone)
-    assertStringEqual "%YAML 1.2\n--- \n- [1, 2, 3]\n- [4, 5]\n- [6]",
-                      output.data
-    
-  test "Serialization: Load Enum":
-    let input = newStringStream("!nim:system:seq(tl)\n- !tl tlRed\n- tlGreen\n- tlYellow")
+    var output = dump(input, tsNone)
+    assertStringEqual "%YAML 1.2\n--- \n- [1, 2, 3]\n- [4, 5]\n- [6]", output
+
+  test "Load Enum":
+    let input =
+      newStringStream("!nim:system:seq(tl)\n- !tl tlRed\n- tlGreen\n- tlYellow")
     var result: seq[TrafficLight]
     load(input, result)
     assert result.len == 3
     assert result[0] == tlRed
     assert result[1] == tlGreen
     assert result[2] == tlYellow
-    
-  test "Serialization: Dump Enum":
+
+  test "Dump Enum":
     let input = @[tlRed, tlGreen, tlYellow]
-    var output = newStringStream()
-    dump(input, output, tsNone, asTidy, blockOnly)
-    assertStringEqual "%YAML 1.2\n--- \n- tlRed\n- tlGreen\n- tlYellow",
-                      output.data
-    
-  test "Serialization: Load Tuple":
+    var output = dump(input, tsNone, asTidy, blockOnly)
+    assertStringEqual "%YAML 1.2\n--- \n- tlRed\n- tlGreen\n- tlYellow", output
+
+  test "Load Tuple":
     let input = newStringStream("str: value\ni: 42\nb: true")
     var result: MyTuple
     load(input, result)
@@ -262,43 +309,91 @@ suite "Serialization":
     assert result.i == 42
     assert result.b == true
 
-  test "Serialization: Dump Tuple":
+  test "Dump Tuple":
     let input = (str: "value", i: 42.int32, b: true)
-    var output = newStringStream()
-    dump(input, output, tsNone)
-    assertStringEqual "%YAML 1.2\n--- \nstr: value\ni: 42\nb: y", output.data
-    
-  test "Serialization: Load custom object":
+    var output = dump(input, tsNone)
+    assertStringEqual "%YAML 1.2\n--- \nstr: value\ni: 42\nb: y", output
+
+  test "Load Tuple - unknown field":
+    let input = "str: value\nfoo: bar\ni: 42\nb: true"
+    var result: MyTuple
+    expectConstructionError(2, 1, "While constructing MyTuple: Unknown field: \"foo\""):
+      load(input, result)
+
+  test "Load Tuple - missing field":
+    let input = "str: value\nb: true"
+    var result: MyTuple
+    expectConstructionError(2, 8, "While constructing MyTuple: Missing field: \"i\""):
+      load(input, result)
+
+  test "Load Tuple - duplicate field":
+    let input = "str: value\ni: 42\nb: true\nb: true"
+    var result: MyTuple
+    expectConstructionError(4, 1, "While constructing MyTuple: Duplicate field: \"b\""):
+      load(input, result)
+
+  test "Load Multiple Documents":
+    let input = newStringStream("1\n---\n2")
+    var result: seq[int]
+    loadMultiDoc(input, result)
+    assert(result.len == 2)
+    assert result[0] == 1
+    assert result[1] == 2
+
+  test "Load Multiple Documents (Single Doc)":
+    let input = newStringStream("1")
+    var result: seq[int]
+    loadMultiDoc(input, result)
+    assert(result.len == 1)
+    assert result[0] == 1
+
+  test "Load custom object":
     let input = newStringStream("firstnamechar: P\nsurname: Pan\nage: 12")
     var result: Person
     load(input, result)
     assert result.firstnamechar == 'P'
     assert result.surname   == "Pan"
     assert result.age == 12
-    
-  test "Serialization: Dump custom object":
+
+  test "Dump custom object":
     let input = Person(firstnamechar: 'P', surname: "Pan", age: 12)
-    var output = newStringStream()
-    dump(input, output, tsNone, asTidy, blockOnly)
+    var output = dump(input, tsNone, asTidy, blockOnly)
     assertStringEqual(
-        "%YAML 1.2\n--- \nfirstnamechar: P\nsurname: Pan\nage: 12", output.data)
-    
-  test "Serialization: Load sequence with explicit tags":
+        "%YAML 1.2\n--- \nfirstnamechar: P\nsurname: Pan\nage: 12", output)
+
+  test "Load custom object - unknown field":
+    let input = "  firstnamechar: P\n  surname: Pan\n  age: 12\n  occupation: free"
+    var result: Person
+    expectConstructionError(4, 3, "While constructing Person: Unknown field: \"occupation\""):
+      load(input, result)
+
+  test "Load custom object - missing field":
+    let input = "surname: Pan\nage: 12\n  "
+    var result: Person
+    expectConstructionError(3, 3, "While constructing Person: Missing field: \"firstnamechar\""):
+      load(input, result)
+
+  test "Load custom object - duplicate field":
+    let input = "firstnamechar: P\nsurname: Pan\nage: 12\nsurname: Pan"
+    var result: Person
+    expectConstructionError(4, 1, "While constructing Person: Duplicate field: \"surname\""):
+      load(input, result)
+
+  test "Load sequence with explicit tags":
     let input = newStringStream("--- !nim:system:seq(" &
         "tag:yaml.org,2002:str)\n- !!str one\n- !!str two")
     var result: seq[string]
     load(input, result)
     assert result[0] == "one"
     assert result[1] == "two"
-    
-  test "Serialization: Dump sequence with explicit tags":
+
+  test "Dump sequence with explicit tags":
     let input = @["one", "two"]
-    var output = newStringStream()
-    dump(input, output, tsAll, asTidy, blockOnly)
+    var output = dump(input, tsAll, asTidy, blockOnly)
     assertStringEqual("%YAML 1.2\n--- !nim:system:seq(" &
-        "tag:yaml.org,2002:str) \n- !!str one\n- !!str two", output.data)
-    
-  test "Serialization: Load custom object with explicit root tag":
+        "tag:yaml.org,2002:str) \n- !!str one\n- !!str two", output)
+
+  test "Load custom object with explicit root tag":
     let input = newStringStream(
         "--- !nim:custom:Person\nfirstnamechar: P\nsurname: Pan\nage: 12")
     var result: Person
@@ -306,16 +401,15 @@ suite "Serialization":
     assert result.firstnamechar == 'P'
     assert result.surname   == "Pan"
     assert result.age       == 12
-    
-  test "Serialization: Dump custom object with explicit root tag":
+
+  test "Dump custom object with explicit root tag":
     let input = Person(firstnamechar: 'P', surname: "Pan", age: 12)
-    var output = newStringStream()
-    dump(input, output, tsRootOnly, asTidy, blockOnly)
+    var output = dump(input, tsRootOnly, asTidy, blockOnly)
     assertStringEqual("%YAML 1.2\n" &
         "--- !nim:custom:Person \nfirstnamechar: P\nsurname: Pan\nage: 12",
-        output.data)
-  
-  test "Serialization: Load custom variant object":
+        output)
+
+  test "Load custom variant object":
     let input = newStringStream(
       "---\n- - name: Bastet\n  - kind: akCat\n  - purringIntensity: 7\n" &
       "- - name: Anubis\n  - kind: akDog\n  - barkometer: 13")
@@ -328,12 +422,11 @@ suite "Serialization":
     assert result[1].name == "Anubis"
     assert result[1].kind == akDog
     assert result[1].barkometer == 13
-  
-  test "Serialization: Dump custom variant object":
+
+  test "Dump custom variant object":
     let input = @[Animal(name: "Bastet", kind: akCat, purringIntensity: 7),
                   Animal(name: "Anubis", kind: akDog, barkometer: 13)]
-    var output = newStringStream()
-    dump(input, output, tsNone, asTidy, blockOnly)
+    var output = dump(input, tsNone, asTidy, blockOnly)
     assertStringEqual """%YAML 1.2
 --- 
 - 
@@ -349,9 +442,15 @@ suite "Serialization":
   - 
     kind: akDog
   - 
-    barkometer: 13""", output.data
-    
-  test "Serialization: Dump cyclic data structure":
+    barkometer: 13""", output
+
+  test "Load custom variant object - missing field":
+    let input = "[{name: Bastet}, {kind: akCat}]"
+    var result: Animal
+    expectConstructionError(1, 32, "While constructing Animal: Missing field: \"purringIntensity\""):
+      load(input, result)
+
+  test "Dump cyclic data structure":
     var
       a = newNode("a")
       b = newNode("b")
@@ -359,8 +458,7 @@ suite "Serialization":
     a.next = b
     b.next = c
     c.next = a
-    var output = newStringStream()
-    dump(a, output, tsRootOnly, asTidy, blockOnly)
+    var output = dump(a, tsRootOnly, asTidy, blockOnly)
     assertStringEqual """%YAML 1.2
 --- !example.net:Node &a 
 value: a
@@ -368,16 +466,16 @@ next:
   value: b
   next: 
     value: c
-    next: *a""", output.data
-    
-  test "Serialization: Load cyclic data structure":
+    next: *a""", output
+
+  test "Load cyclic data structure":
     let input = newStringStream("""%YAML 1.2
---- !nim:system:seq(example.net:Node)
-- &a
+--- !nim:system:seq(example.net:Node) 
+- &a 
   value: a
-  next: &b
+  next: &b 
     value: b
-    next: &c
+    next: &c 
       value: c
       next: *a
 - *b
@@ -398,8 +496,8 @@ next:
     assert(result[0].next == result[1])
     assert(result[1].next == result[2])
     assert(result[2].next == result[0])
-    
-  test "Serialization: Load nil values":
+
+  test "Load nil values":
     let input = newStringStream("- ~\n- !!str ~")
     var result: seq[ref string]
     try: load(input, result)
@@ -408,36 +506,34 @@ next:
       echo "line ", ex.line, ", column ", ex.column, ": ", ex.msg
       echo ex.lineContent
       raise ex
-        
+
     assert(result.len == 2)
     assert(result[0] == nil)
     assert(result[1][] == "~")
-    
-  test "Serialization: Dump nil values":
+
+  test "Dump nil values":
     var input = newSeq[ref string]()
     input.add(nil)
     input.add(new string)
     input[1][] = "~"
-    var output = newStringStream()
-    dump(input, output, tsRootOnly, asTidy, blockOnly)
+    var output = dump(input, tsRootOnly, asTidy, blockOnly)
     assertStringEqual(
         "%YAML 1.2\n--- !nim:system:seq(tag:yaml.org,2002:str) \n- !!null ~\n- !!str ~",
-        output.data)
-    
-  test "Serialization: Custom constructObject":
+        output)
+
+  test "Custom constructObject":
     let input = newStringStream("- 1\n- !test:BetterInt 2")
     var result: seq[BetterInt]
     load(input, result)
     assert(result.len == 2)
     assert(result[0] == 2.BetterInt)
     assert(result[1] == 3.BetterInt)
-    
-  test "Serialization: Custom representObject":
+
+  test "Custom representObject":
     let input = @[1.BetterInt, 9998887.BetterInt, 98312.BetterInt]
-    var output = newStringStream()
-    dump(input, output, tsAll, asTidy, blockOnly)
+    var output = dump(input, tsAll, asTidy, blockOnly)
     assertStringEqual """%YAML 1.2
 --- !nim:system:seq(test:BetterInt) 
 - !test:BetterInt 1
 - !test:BetterInt 9_998_887
-- !test:BetterInt 98_312""", output.data
+- !test:BetterInt 98_312""", output

test/yamlTestSuite.nim

@@ -4,8 +4,8 @@
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 
-import os, terminal, strutils
-import testEventParser, common
+import os, terminal, strutils, streams
+import testEventParser, commonTestUtils
 import "../yaml"
 
 const gitCmd =

yaml.nim

@@ -1,761 +1,44 @@
 #            NimYAML - YAML implementation in Nim
-#        (c) Copyright 2015 Felix Krause
+#        (c) Copyright 2016 Felix Krause
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 
-## This module provides facilities to generate and interpret
-## `YAML <http://yaml.org>`_ character streams. All primitive operations on
-## data objects use a `YamlStream <#YamlStream>`_ either as source or as
-## output. Because this stream is implemented as iterator, it is possible to
-## process YAML input and output sequentially, i.e. without loading the
-## processed data structure completely into RAM. This supports the processing of
-## large data structures.
+## This is the parent module of NimYAML, a package that provides facilities to
+## generate and interpret `YAML <http://yaml.org>`_ character streams. Importing
+## this package will import everything from all subpackages.
 ##
-## As YAML is a strict superset of `JSON <http://json.org>`_, JSON input is
-## automatically supported. While JSON is less readable than YAML,
-## this enhances interoperability with other languages.
+## There are three high-level APIs which are probably most useful:
+##
+## * The serialization API in `serialization <yaml/serialization.html>`_ enables
+##   you to load YAML data directly into native Nim types, and reversely dump
+##   native Nim types as YAML.
+## * The DOM API in `dom <yaml/dom.html>`_ parses YAML files in a tree structure
+##   which you can navigate.
+## * The JSON API in `tojson <yaml/tojson.html>`_ parses YAML files into the
+##   Nim stdlib's JSON structure, which may be useful if you have other modules
+##   which expect JSON input. Note that the serialization API is able to write
+##   and load JSON; you do not need the JSON API for that.
+##
+## Apart from those high-level APIs, NimYAML implements a low-level API which
+## enables you to process YAML input as data stream which does not need to be
+## loaded into RAM completely at once. It consists of the following modules:
+##
+## * The stream API in `stream <yaml/stream.html>`_ defines the central type for
+##   stream processing, ``YamlStream``. It also contains definitions and
+##   constructor procs for stream events.
+## * The parser API in `parser <yaml/parser.html>`_ gives you direct access to
+##   the YAML parser's output.
+## * The presenter API in `presenter <yaml/presenter.html>`_ gives you direct
+##   access to the presenter, i.e. the module that renders a YAML character
+##   stream.
+## * The taglib API in `taglib <yaml/taglib.html>`_ provides a data structure
+##   for keeping track of YAML tags that are generated by the parser or used in
+##   the presenter.
+## * The hints API in `hints <yaml/hints.html>`_ provides a simple proc for
+##   guessing the type of a scalar value.
 
-import streams, unicode, lexbase, tables, strutils, json, hashes, queues,
-       macros, typetraits, parseutils
-export streams, tables, json
-
-when defined(yamlDebug): import terminal
-
-type
-  TypeHint* = enum
-    ## A type hint can be computed from scalar content and tells you what
-    ## NimYAML thinks the scalar's type is. It is generated by
-    ## `guessType <#guessType,string>`_ The first matching RegEx
-    ## in the following table will be the type hint of a scalar string.
-    ##
-    ## You can use it to determine the type of YAML scalars that have a '?'
-    ## non-specific tag, but using this feature is completely optional.
-    ##
-    ## ================== =========================
-    ## Name               RegEx
-    ## ================== =========================
-    ## ``yTypeInteger``   ``0 | -? [1-9] [0-9]*``
-    ## ``yTypeFloat``     ``-? [1-9] ( \. [0-9]* [1-9] )? ( e [-+] [1-9] [0-9]* )?``
-    ## ``yTypeFloatInf``  ``-? \. (inf | Inf | INF)``
-    ## ``yTypeFloatNaN``  ``-? \. (nan | NaN | NAN)``
-    ## ``yTypeBoolTrue``  ``y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON``
-    ## ``yTypeBoolFalse`` ``n|N|no|No|NO|false|False|FALSE|off|Off|OFF``
-    ## ``yTypeNull``      ``~ | null | Null | NULL``
-    ## ``yTypeUnknown``   ``*``
-    ## ================== =========================
-    yTypeInteger, yTypeFloat, yTypeFloatInf, yTypeFloatNaN, yTypeBoolTrue,
-    yTypeBoolFalse, yTypeNull, yTypeUnknown
-    
-  YamlStreamEventKind* = enum
-    ## Kinds of YAML events that may occur in an ``YamlStream``. Event kinds
-    ## are discussed in `YamlStreamEvent <#YamlStreamEvent>`_.
-    yamlStartDoc, yamlEndDoc, yamlStartMap, yamlEndMap,
-    yamlStartSeq, yamlEndSeq, yamlScalar, yamlAlias
-    
-  TagId* = distinct int ## \
-    ## A ``TagId`` identifies a tag URI, like for example 
-    ## ``"tag:yaml.org,2002:str"``. The URI corresponding to a ``TagId`` can
-    ## be queried from the `TagLibrary <#TagLibrary>`_ which was
-    ## used to create this ``TagId``; e.g. when you parse a YAML character
-    ## stream, the ``TagLibrary`` of the parser is the one which generates
-    ## the resulting ``TagId`` s.
-    ##
-    ## URI strings are mapped to ``TagId`` s for efficiency  reasons (you
-    ## do not need to compare strings every time) and to be able to
-    ## discover unknown tag URIs early in the parsing process.
-    
-  AnchorId* = distinct int ## \
-    ## An ``AnchorId`` identifies an anchor in the current document. It
-    ## becomes invalid as soon as the current document scope is invalidated
-    ## (for example, because the parser yielded a ``yamlEndDocument``
-    ## event). ``AnchorId`` s exists because of efficiency, much like
-    ## ``TagId`` s. The actual anchor name is a presentation detail and
-    ## cannot be queried by the user.
-    
-  YamlStreamEvent* = object
-    ## An element from a `YamlStream <#YamlStream>`_. Events that start an
-    ## object (``yamlStartMap``, ``yamlStartSeq``, ``yamlScalar``) have
-    ## an optional anchor and a tag associated with them. The anchor will be
-    ## set to ``yAnchorNone`` if it doesn't exist.
-    ## 
-    ## A non-existing tag in the YAML character stream will be resolved to 
-    ## the non-specific tags ``?`` or ``!`` according to the YAML
-    ## specification. These are by convention mapped to the ``TagId`` s
-    ## ``yTagQuestionMark`` and ``yTagExclamationMark`` respectively.
-    ## Mapping is done by a `TagLibrary <#TagLibrary>`_.
-    case kind*: YamlStreamEventKind
-    of yamlStartMap:
-      mapAnchor* : AnchorId
-      mapTag*    : TagId
-    of yamlStartSeq:
-      seqAnchor* : AnchorId
-      seqTag*    : TagId
-    of yamlScalar:
-      scalarAnchor* : AnchorId
-      scalarTag*    : TagId
-      scalarContent*: string # may not be nil (but empty)
-    of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard
-    of yamlAlias:
-      aliasTarget* : AnchorId
-    
-  YamlStream* = object ## \
-    ## A ``YamlStream`` is an iterator-like object that yields a
-    ## well-formed stream of ``YamlStreamEvents``. Well-formed means that
-    ## every ``yamlStartMap`` is terminated by a ``yamlEndMap``, every
-    ## ``yamlStartSeq`` is terminated by a ``yamlEndSeq`` and every
-    ## ``yamlStartDoc`` is terminated by a ``yamlEndDoc``. Moreover, every
-    ## emitted mapping has an even number of children.
-    ##
-    ## The creator of a ``YamlStream`` is responsible for it being
-    ## well-formed. A user of the stream may assume that it is well-formed
-    ## and is not required to check for it. The procs in this module will
-    ## always yield a well-formed ``YamlStream`` and expect it to be
-    ## well-formed if they take it as input parameter.
-    ##
-    ## 
-    backend: iterator(): YamlStreamEvent
-    peeked: bool
-    cached: YamlStreamEvent
-        
-  TagLibrary* = ref object
-    ## A ``TagLibrary`` maps tag URIs to ``TagId`` s.
-    ##
-    ## When `YamlParser <#YamlParser>`_ encounters tags not existing in the
-    ## tag library, it will use
-    ## `registerUri <#registerUri,TagLibrary,string>`_ to add
-    ## the tag to the library.
-    ##
-    ## You can base your tag library on common tag libraries by initializing
-    ## them with `initFailsafeTagLibrary <#initFailsafeTagLibrary>`_,
-    ## `initCoreTagLibrary <#initCoreTagLibrary>`_ or
-    ## `initExtendedTagLibrary <#initExtendedTagLibrary>`_.
-    tags*: Table[string, TagId]
-    nextCustomTagId*: TagId
-    secondaryPrefix*: string
-    
-    
-  WarningCallback* = proc(line, column: int, lineContent: string,
-                          message: string)
-    ## Callback for parser warnings. Currently, this callback may be called
-    ## on two occasions while parsing a YAML document stream:
-    ##
-    ## - If the version number in the ``%YAML`` directive does not match
-    ##   ``1.2``.
-    ## - If there is an unknown directive encountered.
-    
-  FastParseLevelKind = enum
-    fplUnknown, fplSequence, fplMapKey, fplMapValue, fplSinglePairKey,
-    fplSinglePairValue, fplScalar, fplDocument
-  
-  FastParseLevel = object
-    kind: FastParseLevelKind
-    indentation: int
-  
-  YamlParser* = ref object
-    ## A parser object. Retains its ``TagLibrary`` across calls to
-    ## `parse <#parse,YamlParser,Stream>`_. Can be used
-    ## to access anchor names while parsing a YAML character stream, but
-    ## only until the document goes out of scope (i.e. until
-    ## ``yamlEndDocument`` is yielded).
-    tagLib: TagLibrary
-    callback: WarningCallback
-    lexer: BaseLexer
-    tokenstart: int
-    content, after: string
-    ancestry: seq[FastParseLevel]
-    level: FastParseLevel
-    tagUri: string
-    tag: TagId
-    anchor: AnchorId
-    shorthands: Table[string, string]
-    anchors: Table[string, AnchorId]
-    nextAnchorId: AnchorId
-    newlines: int
-    indentation: int
-        
-  PresentationStyle* = enum
-    ## Different styles for YAML character stream output.
-    ##
-    ## - ``ypsMinimal``: Single-line flow-only output which tries to
-    ##   use as few characters as possible.
-    ## - ``ypsCanonical``: Canonical YAML output. Writes all tags except
-    ##   for the non-specific tags ``?`` and ``!``, uses flow style, quotes
-    ##   all string scalars.
-    ## - ``ypsDefault``: Tries to be as human-readable as possible. Uses
-    ##   block style by default, but tries to condense mappings and
-    ##   sequences which only contain scalar nodes into a single line using
-    ##   flow style.
-    ## - ``ypsJson``: Omits the ``%YAML`` directive and the ``---``
-    ##   marker. Uses flow style. Flattens anchors and aliases, omits tags.
-    ##   Output will be parseable as JSON. ``YamlStream`` to dump may only
-    ##   contain one document.
-    ## - ``ypsBlockOnly``: Formats all output in block style, does not use
-    ##   flow style at all.
-    psMinimal, psCanonical, psDefault, psJson, psBlockOnly
-    
-  TagStyle* = enum
-    ## Whether object should be serialized with explicit tags.
-    ##
-    ## - ``tsNone``: No tags will be outputted unless necessary.
-    ## - ``tsRootOnly``: A tag will only be outputted for the root tag and
-    ##   where necessary.
-    ## - ``tsAll``: Tags will be outputted for every object.
-    tsNone, tsRootOnly, tsAll
-    
-  AnchorStyle* = enum
-    ## How ref object should be serialized.
-    ##
-    ## - ``asNone``: No anchors will be outputted. Values present at
-    ##   multiple places in the content that should be serialized will be
-    ##   fully serialized at every occurence. If the content is cyclic, this
-    ##   will lead to an endless loop!
-    ## - ``asTidy``: Anchors will only be generated for objects that
-    ##   actually occur more than once in the content to be serialized.
-    ##   This is a bit slower and needs more memory than ``asAlways``.
-    ## - ``asAlways``: Achors will be generated for every ref object in the
-    ##   content to be serialized, regardless of whether the object is
-    ##   referenced again afterwards
-    asNone, asTidy, asAlways
-    
-  NewLineStyle* = enum
-    ## What kind of newline sequence is used when presenting.
-    ##
-    ## - ``nlLF``: Use a single linefeed char as newline.
-    ## - ``nlCRLF``: Use a sequence of carriage return and linefeed as
-    ##   newline.
-    ## - ``nlOSDefault``: Use the target operation system's default newline
-    ##   sequence (CRLF on Windows, LF everywhere else).
-    nlLF, nlCRLF, nlOSDefault
-    
-  OutputYamlVersion* = enum
-    ## Specify which YAML version number the presenter shall emit. The
-    ## presenter will always emit content that is valid YAML 1.1, but by
-    ## default will write a directive ``%YAML 1.2``. For compatibility with
-    ## other YAML implementations, it is possible to change this here.
-    ##
-    ## It is also possible to specify that the presenter shall not emit any
-    ## YAML version. The generated content is then guaranteed to be valid
-    ## YAML 1.1 and 1.2 (but not 1.0 or any newer YAML version).
-    ov1_2, ov1_1, ovNone
-            
-  PresentationOptions* = object
-    ## Options for generating a YAML character stream
-    style*: PresentationStyle
-    indentationStep*: int
-    newlines*: NewLineStyle
-    outputVersion*: OutputYamlVersion
-        
-  ConstructionContext* = ref object
-    ## Context information for the process of constructing Nim values from YAML.
-    refs: Table[AnchorId, pointer]
-    
-  SerializationContext* = ref object
-    ## Context information for the process of serializing YAML from Nim values.
-    refs: Table[pointer, AnchorId]
-    style: AnchorStyle
-    nextAnchorId: AnchorId
-    
-  RawYamlStream* = iterator(): YamlStreamEvent {.raises: [].} ## \
-    ## Stream of ``YamlStreamEvent``s returned by ``representObject`` procs.
-    
-  YamlNodeKind* = enum
-    yScalar, yMapping, ySequence
-    
-  YamlNode* = ref YamlNodeObj not nil
-    ## Represents a node in a ``YamlDocument``.
-    
-  YamlNodeObj* = object
-    tag*: string
-    case kind*: YamlNodeKind
-    of yScalar: content*: string
-    of ySequence: children*: seq[YamlNode]
-    of yMapping: pairs*: seq[tuple[key, value: YamlNode]]
-    
-  YamlDocument* = object
-    ## Represents a YAML document.
-    root*: YamlNode
-    
-  YamlLoadingError* = object of Exception
-    ## Base class for all exceptions that may be raised during the process
-    ## of loading a YAML character stream. 
-    line*: int ## line number (1-based) where the error was encountered
-    column*: int ## column number (1-based) where the error was encountered
-    lineContent*: string ## \
-      ## content of the line where the error was encountered. Includes a
-      ## second line with a marker ``^`` at the position where the error
-      ## was encountered.
-    
-  YamlParserError* = object of YamlLoadingError
-    ## A parser error is raised if the character stream that is parsed is
-    ## not a valid YAML character stream. This stream cannot and will not be
-    ## parsed wholly nor partially and all events that have been emitted by
-    ## the YamlStream the parser provides should be discarded.
-    ##
-    ## A character stream is invalid YAML if and only if at least one of the
-    ## following conditions apply:
-    ##
-    ## - There are invalid characters in an element whose contents is
-    ##   restricted to a limited set of characters. For example, there are
-    ##   characters in a tag URI which are not valid URI characters.
-    ## - An element has invalid indentation. This can happen for example if
-    ##   a block list element indicated by ``"- "`` is less indented than
-    ##   the element in the previous line, but there is no block sequence
-    ##   list open at the same indentation level. 
-    ## - The YAML structure is invalid. For example, an explicit block map
-    ##   indicated by ``"? "`` and ``": "`` may not suddenly have a block
-    ##   sequence item (``"- "``) at the same indentation level. Another
-    ##   possible violation is closing a flow style object with the wrong
-    ##   closing character (``}``, ``]``) or not closing it at all.
-    ## - A custom tag shorthand is used that has not previously been 
-    ##   declared with a ``%TAG`` directive.
-    ## - Multiple tags or anchors are defined for the same node.
-    ## - An alias is used which does not map to any anchor that has
-    ##   previously been declared in the same document.
-    ## - An alias has a tag or anchor associated with it.
-    ##
-    ## Some elements in this list are vague. For a detailed description of a
-    ## valid YAML character stream, see the YAML specification.
-    
-  YamlPresenterJsonError* = object of Exception
-    ## Exception that may be raised by the YAML presenter when it is
-    ## instructed to output JSON, but is unable to do so. This may occur if:
-    ##
-    ## - The given `YamlStream <#YamlStream>`_ contains a map which has any
-    ##   non-scalar type as key.
-    ## - Any float scalar bears a ``NaN`` or positive/negative infinity value
-    
-  YamlPresenterOutputError* = object of Exception
-    ## Exception that may be raised by the YAML presenter. This occurs if
-    ## writing character data to the output stream raises any exception.
-    ## The error that has occurred is available from ``parent``.
-    
-  YamlStreamError* = object of Exception
-    ## Exception that may be raised by a ``YamlStream`` when the underlying
-    ## backend raises an exception. The error that has occurred is
-    ## available from ``parent``.
-    
-  YamlConstructionError* = object of YamlLoadingError
-    ## Exception that may be raised when constructing data objects from a
-    ## `YamlStream <#YamlStream>`_. The fields ``line``, ``column`` and
-    ## ``lineContent`` are only available if the costructing proc also does
-    ## parsing, because otherwise this information is not available to the
-    ## costruction proc.
-
-const
-  # failsafe schema
-
-  yTagExclamationMark*: TagId = 0.TagId ## ``!`` non-specific tag
-  yTagQuestionMark*   : TagId = 1.TagId ## ``?`` non-specific tag
-  yTagString*         : TagId = 2.TagId ## \
-    ## `!!str <http://yaml.org/type/str.html >`_ tag
-  yTagSequence*       : TagId = 3.TagId ## \
-    ## `!!seq <http://yaml.org/type/seq.html>`_ tag
-  yTagMapping*        : TagId = 4.TagId ## \
-    ## `!!map <http://yaml.org/type/map.html>`_ tag
-    
-  # json & core schema
-    
-  yTagNull*    : TagId = 5.TagId ## \
-    ## `!!null <http://yaml.org/type/null.html>`_ tag
-  yTagBoolean* : TagId = 6.TagId ## \
-    ## `!!bool <http://yaml.org/type/bool.html>`_ tag
-  yTagInteger* : TagId = 7.TagId ## \
-    ## `!!int <http://yaml.org/type/int.html>`_ tag
-  yTagFloat*   : TagId = 8.TagId ## \
-    ## `!!float <http://yaml.org/type/float.html>`_ tag
-    
-  # other language-independent YAML types (from http://yaml.org/type/ )
-    
-  yTagOrderedMap* : TagId = 9.TagId  ## \
-    ## `!!omap <http://yaml.org/type/omap.html>`_ tag
-  yTagPairs*      : TagId = 10.TagId ## \
-    ## `!!pairs <http://yaml.org/type/pairs.html>`_ tag
-  yTagSet*        : TagId = 11.TagId ## \
-    ## `!!set <http://yaml.org/type/set.html>`_ tag
-  yTagBinary*     : TagId = 12.TagId ## \
-    ## `!!binary <http://yaml.org/type/binary.html>`_ tag
-  yTagMerge*      : TagId = 13.TagId ## \
-    ## `!!merge <http://yaml.org/type/merge.html>`_ tag
-  yTagTimestamp*  : TagId = 14.TagId ## \
-    ## `!!timestamp <http://yaml.org/type/timestamp.html>`_ tag
-  yTagValue*      : TagId = 15.TagId ## \
-    ## `!!value <http://yaml.org/type/value.html>`_ tag
-  yTagYaml*       : TagId = 16.TagId ## \
-    ## `!!yaml <http://yaml.org/type/yaml.html>`_ tag
-    
-  yTagNimField*   : TagId = 100.TagId ## \
-    ## This tag is used in serialization for the name of a field of an
-    ## object. It may contain any string scalar that is a valid Nim symbol.
-  
-  yTagNimNilString* : TagId = 101.TagId ## for strings that are nil
-  yTagNimNilSeq*    : TagId = 102.TagId ## \
-    ## for seqs that are nil. This tag is used regardless of the seq's generic
-    ## type parameter.
-    
-  yFirstCustomTagId* : TagId = 1000.TagId ## \
-    ## The first ``TagId`` which should be assigned to an URI that does not
-    ## exist in the ``YamlTagLibrary`` which is used for parsing.
-    
-  yAnchorNone*: AnchorId = (-1).AnchorId ## \
-    ## yielded when no anchor was defined for a YAML node
-    
-  yamlTagRepositoryPrefix* = "tag:yaml.org,2002:"
-    
-  defaultPresentationOptions* =
-        PresentationOptions(style: psDefault, indentationStep: 2,
-                            newlines: nlOSDefault)
-    
-# interface
-
-proc `==`*(left: YamlStreamEvent, right: YamlStreamEvent): bool {.raises: [].}
-  ## compares all existing fields of the given items
-    
-proc `$`*(event: YamlStreamEvent): string {.raises: [].}
-  ## outputs a human-readable string describing the given event
-
-proc tag*(event: YamlStreamEvent): TagId {.raises: [FieldError].}
-
-proc startDocEvent*(): YamlStreamEvent {.inline, raises: [].}
-proc endDocEvent*(): YamlStreamEvent {.inline, raises: [].}
-proc startMapEvent*(tag: TagId = yTagQuestionMark,
-                    anchor: AnchorId = yAnchorNone):
-                    YamlStreamEvent {.inline, raises: [].}
-proc endMapEvent*(): YamlStreamEvent {.inline, raises: [].}
-proc startSeqEvent*(tag: TagId = yTagQuestionMark,
-                    anchor: AnchorId = yAnchorNone):
-                    YamlStreamEvent {.inline, raises: [].}
-proc endSeqEvent*(): YamlStreamEvent {.inline, raises: [].}
-proc scalarEvent*(content: string = "", tag: TagId = yTagQuestionMark,
-                  anchor: AnchorId = yAnchorNone):
-                  YamlStreamEvent {.inline, raises: [].}
-proc aliasEvent*(anchor: AnchorId): YamlStreamEvent {.inline, raises: [].}
-
-proc `==`*(left, right: TagId): bool {.borrow.}
-proc `$`*(id: TagId): string
-proc hash*(id: TagId): Hash {.borrow.}
-
-proc `==`*(left, right: AnchorId): bool {.borrow.}
-proc `$`*(id: AnchorId): string {.borrow.}
-proc hash*(id: AnchorId): Hash {.borrow.}
-
-proc initYamlStream*(backend: iterator(): YamlStreamEvent):
-    YamlStream {.raises: [].}
-  ## Creates a new ``YamlStream`` that uses the given iterator as backend.
-proc next*(s: var YamlStream): YamlStreamEvent {.raises: [YamlStreamError].}
-  ## Get the next item of the stream. Requires ``finished(s) == true``.
-  ## If the backend yields an exception, that exception will be encapsulated
-  ## into a ``YamlStreamError``, which will be raised. 
-proc peek*(s: var YamlStream): YamlStreamEvent {.raises: [YamlStreamError].}
-  ## Get the next item of the stream without advancing the stream.
-  ## Requires ``finished(s) == true``. Handles exceptions of the backend like
-  ## ``next()``.
-proc `peek=`*(s: var YamlStream, value: YamlStreamEvent) {.raises: [].}
-  ## Set the next item of the stream. Will replace a previously peeked item,
-  ## if one exists.
-proc finished*(s: var YamlStream): bool {.raises: [YamlStreamError].}
-  ## ``true`` if no more items are available in the stream. Handles exceptions
-  ## of the backend like ``next()``.
-iterator items*(s: var YamlStream): YamlStreamEvent
-    {.raises: [YamlStreamError].} =
-  ## Iterate over all items of the stream. You may not use ``peek()`` on the
-  ## stream while iterating.
-  if s.peeked:
-    s.peeked = false
-    yield s.cached
-  while true:
-    var event: YamlStreamEvent
-    try:
-      event = s.backend()
-      if finished(s.backend): break
-    except AssertionError: raise
-    except YamlStreamError:
-      let cur = getCurrentException()
-      var e = newException(YamlStreamError, cur.msg)
-      e.parent = cur.parent
-      raise e
-    except Exception:
-      var e = newException(YamlStreamError, getCurrentExceptionMsg())
-      e.parent = getCurrentException()
-      raise e
-    yield event
-
-proc initTagLibrary*(): TagLibrary {.raises: [].}
-  ## initializes the ``tags`` table and sets ``nextCustomTagId`` to
-  ## ``yFirstCustomTagId``.
-
-proc registerUri*(tagLib: TagLibrary, uri: string): TagId {.raises: [].}
-  ## registers a custom tag URI with a ``TagLibrary``. The URI will get
-  ## the ``TagId`` ``nextCustomTagId``, which will be incremented.
-    
-proc uri*(tagLib: TagLibrary, id: TagId): string {.raises: [KeyError].}
-  ## retrieve the URI a ``TagId`` maps to.
-
-proc initFailsafeTagLibrary*(): TagLibrary {.raises: [].}
-  ## Contains only:
-  ## - ``!``
-  ## - ``?``
-  ## - ``!!str``
-  ## - ``!!map``
-  ## - ``!!seq``
-proc initCoreTagLibrary*(): TagLibrary {.raises: [].}
-  ## Contains everything in ``initFailsafeTagLibrary`` plus:
-  ## - ``!!null``
-  ## - ``!!bool``
-  ## - ``!!int``
-  ## - ``!!float``
-proc initExtendedTagLibrary*(): TagLibrary {.raises: [].}
-  ## Contains everything from ``initCoreTagLibrary`` plus:
-  ## - ``!!omap``
-  ## - ``!!pairs``
-  ## - ``!!set``
-  ## - ``!!binary``
-  ## - ``!!merge``
-  ## - ``!!timestamp``
-  ## - ``!!value``
-  ## - ``!!yaml``
-
-proc initSerializationTagLibrary(): TagLibrary {.raises: [].}
-
-proc guessType*(scalar: string): TypeHint {.raises: [].}
-  ## Parse scalar string according to the RegEx table documented at
-  ## `TypeHint <#TypeHind>`_.
-
-proc newYamlParser*(tagLib: TagLibrary = initExtendedTagLibrary(),
-                    callback: WarningCallback = nil): YamlParser {.raises: [].}
-  ## Creates a YAML parser. if ``callback`` is not ``nil``, it will be called
-  ## whenever the parser yields a warning. 
-
-proc getLineNumber*(p: YamlParser): int {.raises: [].}
-  ## Get the line number (1-based) of the recently yielded parser token.
-  ## Useful for error reporting at later loading stages.
-
-proc getColNumber*(p: YamlParser): int {.raises: [].}
-  ## Get the column number (1-based) of the recently yielded parser token.
-  ## Useful for error reporting at later parsing stages.
-
-proc getLineContent*(p: YamlParser, marker: bool = true): string {.raises: [].}
-  ## Get the content of the input line containing the recently yielded parser
-  ## token. Useful for error reporting at later parsing stages. The line will
-  ## be terminated by ``"\n"``. If ``marker`` is ``true``, a second line will
-  ## be returned containing a ``^`` at the position of the recent parser
-  ## token.
-
-proc parse*(p: YamlParser, s: Stream): YamlStream {.raises: [].}
-  ## Parse the given stream as YAML character stream. 
-
-proc defineOptions*(style: PresentationStyle = psDefault,
-                    indentationStep: int = 2,
-                    newlines: NewLineStyle = nlOSDefault,
-                    outputVersion: OutputYamlVersion = ov1_2):
-    PresentationOptions {.raises: [].}
-  ## Define a set of options for presentation. Convenience proc that requires
-  ## you to only set those values that should not equal the default.
-
-proc constructJson*(s: var YamlStream): seq[JsonNode]
-    {.raises: [YamlConstructionError, YamlStreamError].}
-  ## Construct an in-memory JSON tree from a YAML event stream. The stream may
-  ## not contain any tags apart from those in ``coreTagLibrary``. Anchors and
-  ## aliases will be resolved. Maps in the input must not contain
-  ## non-scalars as keys. Each element of the result represents one document
-  ## in the YAML stream.
-  ##
-  ## **Warning:** The special float values ``[+-]Inf`` and ``NaN`` will be
-  ## parsed into Nim's JSON structure without error. However, they cannot be
-  ## rendered to a JSON character stream, because these values are not part
-  ## of the JSON specification. Nim's JSON implementation currently does not
-  ## check for these values and will output invalid JSON when rendering one
-  ## of these values into a JSON character stream.
-
-proc loadToJson*(s: Stream): seq[JsonNode] {.raises: [].}
-    ## Uses `YamlParser <#YamlParser>`_ and
-    ## `constructJson <#constructJson>`_ to construct an in-memory JSON tree
-    ## from a YAML character stream.
-    
-proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary,
-              options: PresentationOptions = defaultPresentationOptions)
-    {.raises: [YamlPresenterJsonError, YamlPresenterOutputError,
-               YamlStreamError].}
-  ## Convert ``s`` to a YAML character stream and write it to ``target``.
-    
-proc transform*(input: Stream, output: Stream,
-                options: PresentationOptions = defaultPresentationOptions)
-    {.raises: [IOError, YamlParserError, YamlPresenterJsonError,
-               YamlPresenterOutputError].}
-  ## Parser ``input`` as YAML character stream and then dump it to ``output``
-  ## while resolving non-specific tags to the ones in the YAML core tag
-  ## library.
-
-proc constructChild*[T](s: var YamlStream, c: ConstructionContext,
-                        result: var T)
-    {.raises: [YamlConstructionError, YamlStreamError].}
-  ## Constructs an arbitrary Nim value from a part of a YAML stream.
-  ## The stream will advance until after the finishing token that was used
-  ## for constructing the value. The ``ConstructionContext`` is needed for
-  ## potential child objects which may be refs.
-
-proc constructChild*(s: var YamlStream, c: ConstructionContext,
-                     result: var string)
-    {.raises: [YamlConstructionError, YamlStreamError].}
-  ## Constructs a Nim value that is a string from a part of a YAML stream.
-  ## This specialization takes care of possible nil strings.
-
-proc constructChild*[T](s: var YamlStream, c: ConstructionContext,
-                        result: var seq[T])
-    {.raises: [YamlConstructionError, YamlStreamError].}
-  ## Constructs a Nim value that is a string from a part of a YAML stream.
-  ## This specialization takes care of possible nil seqs.
-
-proc constructChild*[O](s: var YamlStream, c: ConstructionContext,
-                        result: var ref O)
-    {.raises: [YamlConstructionError, YamlStreamError].}
-  ## Constructs an arbitrary Nim value from a part of a YAML stream.
-  ## The stream will advance until after the finishing token that was used
-  ## for constructing the value. The object may be constructed from an alias
-  ## node which will be resolved using the ``ConstructionContext``.
-
-proc representChild*[O](value: ref O, ts: TagStyle, c: SerializationContext):
-    RawYamlStream {.raises: [].}
-  ## Represents an arbitrary Nim reference value as YAML object. The object
-  ## may be  represented as alias node if it is already present in the
-  ## ``SerializationContext``.
-
-proc representChild*(value: string, ts: TagStyle, c: SerializationContext):
-    RawYamlStream {.inline.}
-  ## Represents a Nim string. Supports nil strings.
-
-proc representChild*[O](value: O, ts: TagStyle,
-                        c: SerializationContext):
-    RawYamlStream {.raises: [].}
-  ## Represents an arbitrary Nim object as YAML object.
-
-proc construct*[T](s: var YamlStream, target: var T)
-    {.raises: [YamlConstructionError, YamlStreamError].}
-  ## Constructs a Nim value from a YAML stream.
-
-proc load*[K](input: Stream, target: var K)
-    {.raises: [YamlConstructionError, IOError, YamlParserError].}
-  ## Loads a Nim value from a YAML character stream.
-
-proc represent*[T](value: T, ts: TagStyle = tsRootOnly,
-                   a: AnchorStyle = asTidy): YamlStream {.raises: [].}
-  ## Represents a Nim value as ``YamlStream``
-
-proc dump*[K](value: K, target: Stream, tagStyle: TagStyle = tsRootOnly,
-              anchorStyle: AnchorStyle = asTidy,
-              options: PresentationOptions = defaultPresentationOptions)
-    {.raises: [YamlPresenterJsonError, YamlPresenterOutputError].}
-  ## Dump a Nim value as YAML character stream.
-
-var
-  serializationTagLibrary* = initSerializationTagLibrary() ## \
-    ## contains all local tags that are used for type serialization. Does
-    ## not contain any of the specific default tags for sequences or maps,
-    ## as those are not suited for Nim's static type system.
-    ##
-    ## Should not be modified manually. Will be extended by
-    ## `serializable <#serializable,stmt,stmt>`_.
-
-var
-  nextStaticTagId {.compileTime.} = 100.TagId ## \
-    ## used for generating unique TagIds with ``setTagUri``.
-  registeredUris {.compileTime.} = newSeq[string]() ## \
-    ## Since Table doesn't really work at compile time, we also store 
-    ## registered URIs here to be able to generate a static compiler error 
-    ## when the user tries to register an URI more than once.
-
-template setTagUri*(t: typedesc, uri: string): stmt =
-  ## Associate the given uri with a certain type. This uri is used as YAML tag
-  ## when loading and dumping values of this type.
-  when uri in registeredUris:
-    {. fatal: "[NimYAML] URI \"" & uri & "\" registered twice!" .}
-  const id {.genSym.} = nextStaticTagId
-  static:
-    registeredUris.add(uri)
-    nextStaticTagId = TagId(int(nextStaticTagId) + 1)
-  when nextStaticTagId == yFirstCustomTagId:
-    {.fatal: "Too many tags!".}
-  serializationTagLibrary.tags[uri] = id
-  proc yamlTag*(T: typedesc[t]): TagId {.inline, raises: [].} = id
-    ## autogenerated
-
-template setTagUri*(t: typedesc, uri: string, idName: expr): stmt =
-  ## Like `setTagUri <#setTagUri.t,typedesc,string>`_, but lets
-  ## you choose a symbol for the `TagId <#TagId>`_ of the uri. This is only
-  ## necessary if you want to implement serialization / construction yourself.
-  when uri in registeredUris:
-    {. fatal: "[NimYAML] URI \"" & uri & "\" registered twice!" .}
-  const idName* = nextStaticTagId
-  static:
-    registeredUris.add(uri)
-    nextStaticTagId = TagId(int(nextStaticTagId) + 1)
-  serializationTagLibrary.tags[uri] = idName
-  proc yamlTag*(T: typedesc[t]): TagId {.inline, raises: [].} = idName
-    ## autogenerated
-
-proc canBeImplicit(t: typedesc): bool {.compileTime.} =
-  let tDesc = getType(t)
-  if tDesc.kind != nnkObjectTy: return false
-  if tDesc[2].len != 1: return false
-  if tDesc[2][0].kind != nnkRecCase: return false
-  var foundEmptyBranch = false
-  for i in 1.. tDesc[2][0].len - 1:
-    case tDesc[2][0][i][1].len # branch contents
-    of 0:
-      if foundEmptyBranch: return false
-      else: foundEmptyBranch = true
-    of 1: discard
-    else: return false
-  return true
-
-template markAsImplicit*(t: typedesc): stmt =
-  ## Mark a variant object type as implicit. This requires the type to consist
-  ## of nothing but a case expression and each branch of the case expression
-  ## containing exactly one field - with the exception that one branch may
-  ## contain zero fields.
-  when canBeImplicit(t):
-    # this will be checked by means of compiles(implicitVariantObject(...))
-    proc implicitVariantObject*(unused: t) = discard
-  else:
-    {. fatal: "This type cannot be marked as implicit" .}
-
-static:
-  # standard YAML tags used by serialization
-  registeredUris.add("!")
-  registeredUris.add("?")
-  registeredUris.add("tag:yaml.org,2002:str")
-  registeredUris.add("tag:yaml.org,2002:null")
-  registeredUris.add("tag:yaml.org,2002:bool")
-  registeredUris.add("tag:yaml.org,2002:float")
-  registeredUris.add("tag:yaml.org,2002:timestamp")
-  registeredUris.add("tag:yaml.org,2002:value")
-  registeredUris.add("tag:yaml.org,2002:binary")
-  # special tags used by serialization
-  registeredUris.add("!nim:field")
-  registeredUris.add("!nim:nil:string")
-  registeredUris.add("!nim:nil:seq")
-
-# tags for Nim's standard types
-setTagUri(char, "!nim:system:char", yTagNimChar)
-setTagUri(int8, "!nim:system:int8", yTagNimInt8)
-setTagUri(int16, "!nim:system:int16", yTagNimInt16)
-setTagUri(int32, "!nim:system:int32", yTagNimInt32)
-setTagUri(int64, "!nim:system:int64", yTagNimInt64)
-setTagUri(uint8, "!nim:system:uint8", yTagNimUInt8)
-setTagUri(uint16, "!nim:system:uint16", yTagNimUInt16)
-setTagUri(uint32, "!nim:system:uint32", yTagNimUInt32)
-setTagUri(uint64, "!nim:system:uint64", yTagNimUInt64)
-setTagUri(float32, "!nim:system:float32", yTagNimFloat32)
-setTagUri(float64, "!nim:system:float64", yTagNimFloat64)
-
-# implementation
-
-include private.tagLibrary
-include private.events
-include private.json
-include private.presenter
-include private.hints
-include private.fastparse
-include private.streams
-include private.serialization
-include private.dom
+import yaml.dom, yaml.hints, yaml.parser, yaml.presenter,
+       yaml.serialization, yaml.stream, yaml.taglib, yaml.tojson
+export yaml.dom, yaml.hints, yaml.parser, yaml.presenter,
+       yaml.serialization, yaml.stream, yaml.taglib, yaml.tojson

yaml/dom.nim

@@ -0,0 +1,195 @@
+#            NimYAML - YAML implementation in Nim
+#        (c) Copyright 2016 Felix Krause
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+
+## ===============
+## Module yaml.dom
+## ===============
+##
+## This is the DOM API, which enables you to load YAML into a tree-like
+## structure. It can also dump the structure back to YAML. Formally, it
+## represents the *Representation Graph* as defined in the YAML specification.
+##
+## The main interface of this API are ``loadDOM`` and ``dumpDOM``. The other
+## exposed procs are low-level and useful if you want to load or generate parts
+## of a ``YamlStream``.
+
+import tables, streams
+import stream, taglib, serialization, ../private/internal, parser,
+       presenter
+
+type
+  YamlNodeKind* = enum
+    yScalar, yMapping, ySequence
+
+  YamlNode* = ref YamlNodeObj not nil
+    ## Represents a node in a ``YamlDocument``.
+
+  YamlNodeObj* = object
+    tag*: string
+    case kind*: YamlNodeKind
+    of yScalar: content*: string
+    of ySequence: children*: seq[YamlNode]
+    of yMapping: pairs*: seq[tuple[key, value: YamlNode]]
+
+  YamlDocument* = object
+    ## Represents a YAML document.
+    root*: YamlNode
+
+proc newYamlNode*(content: string, tag: string = "?"): YamlNode =
+  YamlNode(kind: yScalar, content: content, tag: tag)
+
+proc newYamlNode*(children: openarray[YamlNode], tag: string = "?"):
+    YamlNode =
+  YamlNode(kind: ySequence, children: @children, tag: tag)
+
+proc newYamlNode*(pairs: openarray[tuple[key, value: YamlNode]],
+                  tag: string = "?"): YamlNode =
+  YamlNode(kind: yMapping, pairs: @pairs, tag: tag)
+
+proc initYamlDoc*(root: YamlNode): YamlDocument = result.root = root
+
+proc composeNode(s: var YamlStream, tagLib: TagLibrary,
+                 c: ConstructionContext):
+    YamlNode {.raises: [YamlStreamError, YamlConstructionError].} =
+  var start: YamlStreamEvent
+  shallowCopy(start, s.next())
+  new(result)
+  try:
+    case start.kind
+    of yamlStartMap:
+      result.tag = tagLib.uri(start.mapTag)
+      result.kind = yMapping
+      result.pairs = newSeq[tuple[key, value: YamlNode]]()
+      while s.peek().kind != yamlEndMap:
+        let
+          key = composeNode(s, tagLib, c)
+          value = composeNode(s, tagLib, c)
+        result.pairs.add((key: key, value: value))
+      discard s.next()
+      if start.mapAnchor != yAnchorNone:
+        yAssert(not c.refs.hasKey(start.mapAnchor))
+        c.refs[start.mapAnchor] = cast[pointer](result)
+    of yamlStartSeq:
+      result.tag = tagLib.uri(start.seqTag)
+      result.kind = ySequence
+      result.children = newSeq[YamlNode]()
+      while s.peek().kind != yamlEndSeq:
+        result.children.add(composeNode(s, tagLib, c))
+      if start.seqAnchor != yAnchorNone:
+        yAssert(not c.refs.hasKey(start.seqAnchor))
+        c.refs[start.seqAnchor] = cast[pointer](result)
+      discard s.next()
+    of yamlScalar:
+      result.tag = tagLib.uri(start.scalarTag)
+      result.kind = yScalar
+      shallowCopy(result.content, start.scalarContent)
+      if start.scalarAnchor != yAnchorNone:
+        yAssert(not c.refs.hasKey(start.scalarAnchor))
+        c.refs[start.scalarAnchor] = cast[pointer](result)
+    of yamlAlias:
+      result = cast[YamlNode](c.refs[start.aliasTarget])
+    else: internalError("Malformed YamlStream")
+  except KeyError:
+    raise newException(YamlConstructionError,
+                       "Wrong tag library: TagId missing")
+
+proc compose*(s: var YamlStream, tagLib: TagLibrary): YamlDocument
+    {.raises: [YamlStreamError, YamlConstructionError].} =
+  var context = newConstructionContext()
+  var n: YamlStreamEvent
+  shallowCopy(n, s.next())
+  yAssert n.kind == yamlStartDoc
+  result.root = composeNode(s, tagLib, context)
+  n = s.next()
+  yAssert n.kind == yamlEndDoc
+
+proc loadDOM*(s: Stream | string): YamlDocument
+    {.raises: [IOError, YamlParserError, YamlConstructionError].} =
+  var
+    tagLib = initExtendedTagLibrary()
+    parser = newYamlParser(tagLib)
+    events = parser.parse(s)
+  try: result = compose(events, tagLib)
+  except YamlStreamError:
+    let e = getCurrentException()
+    if e.parent of YamlParserError:
+      raise (ref YamlParserError)(e.parent)
+    elif e.parent of IOError:
+      raise (ref IOError)(e.parent)
+    else: internalError("Unexpected exception: " & e.parent.repr)
+
+proc serializeNode(n: YamlNode, c: SerializationContext, a: AnchorStyle,
+                   tagLib: TagLibrary) {.raises: [].}=
+  let p = cast[pointer](n)
+  if a != asNone and c.refs.hasKey(p):
+    if c.refs.getOrDefault(p) == yAnchorNone:
+      c.refs[p] = c.nextAnchorId
+      c.nextAnchorId = AnchorId(int(c.nextAnchorId) + 1)
+    c.put(aliasEvent(c.refs.getOrDefault(p)))
+    return
+  var
+    tagId: TagId
+    anchor: AnchorId
+  if a == asAlways:
+    c.refs[p] = c.nextAnchorId
+    c.nextAnchorId = AnchorId(int(c.nextAnchorId) + 1)
+  else: c.refs[p] = yAnchorNone
+  tagId = if tagLib.tags.hasKey(n.tag): tagLib.tags.getOrDefault(n.tag) else:
+          tagLib.registerUri(n.tag)
+  case a
+  of asNone: anchor = yAnchorNone
+  of asTidy: anchor = cast[AnchorId](n)
+  of asAlways: anchor = c.refs.getOrDefault(p)
+
+  case n.kind
+  of yScalar: c.put(scalarEvent(n.content, tagId, anchor))
+  of ySequence:
+    c.put(startSeqEvent(tagId, anchor))
+    for item in n.children:
+      serializeNode(item, c, a, tagLib)
+    c.put(endSeqEvent())
+  of yMapping:
+    c.put(startMapEvent(tagId, anchor))
+    for i in n.pairs:
+      serializeNode(i.key, c, a, tagLib)
+      serializeNode(i.value, c, a, tagLib)
+    c.put(endMapEvent())
+
+template processAnchoredEvent(target: untyped, c: SerializationContext): typed =
+  let anchorId = c.refs.getOrDefault(cast[pointer](target))
+  if anchorId != yAnchorNone: target = anchorId
+  else: target = yAnchorNone
+
+proc serialize*(doc: YamlDocument, tagLib: TagLibrary, a: AnchorStyle = asTidy):
+    YamlStream {.raises: [].} =
+  var
+    bys = newBufferYamlStream()
+    c = newSerializationContext(a, proc(e: YamlStreamEvent) {.raises: [].} =
+      bys.put(e)
+    )
+  c.put(startDocEvent())
+  serializeNode(doc.root, c, a, tagLib)
+  c.put(endDocEvent())
+  if a == asTidy:
+    for event in bys.mitems():
+      case event.kind
+      of yamlScalar: processAnchoredEvent(event.scalarAnchor, c)
+      of yamlStartMap: processAnchoredEvent(event.mapAnchor, c)
+      of yamlStartSeq: processAnchoredEvent(event.seqAnchor, c)
+      else: discard
+  result = bys
+
+proc dumpDOM*(doc: YamlDocument, target: Stream,
+              anchorStyle: AnchorStyle = asTidy,
+              options: PresentationOptions = defaultPresentationOptions)
+    {.raises: [YamlPresenterJsonError, YamlPresenterOutputError,
+               YamlStreamError].} =
+  ## Dump a YamlDocument as YAML character stream.
+  var
+    tagLib = initExtendedTagLibrary()
+    events = serialize(doc, tagLib,
+                       if options.style == psJson: asNone else: anchorStyle)
+  present(events, target, tagLib, options)

yaml/hints.nim

@@ -1,10 +1,43 @@
 #            NimYAML - YAML implementation in Nim
-#        (c) Copyright 2015 Felix Krause
+#        (c) Copyright 2016 Felix Krause
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 
+## =================
+## Module yaml.hints
+## =================
+##
+## The hints API enables you to guess the type of YAML scalars.
+
+import macros
+import ../private/internal
+
 type
+  TypeHint* = enum
+    ## A type hint can be computed from scalar content and tells you what
+    ## NimYAML thinks the scalar's type is. It is generated by
+    ## `guessType <#guessType,string>`_ The first matching RegEx
+    ## in the following table will be the type hint of a scalar string.
+    ##
+    ## You can use it to determine the type of YAML scalars that have a '?'
+    ## non-specific tag, but using this feature is completely optional.
+    ##
+    ## ================== =========================
+    ## Name               RegEx
+    ## ================== =========================
+    ## ``yTypeInteger``   ``0 | -? [1-9] [0-9]*``
+    ## ``yTypeFloat``     ``-? [1-9] ( \. [0-9]* [1-9] )? ( e [-+] [1-9] [0-9]* )?``
+    ## ``yTypeFloatInf``  ``-? \. (inf | Inf | INF)``
+    ## ``yTypeFloatNaN``  ``-? \. (nan | NaN | NAN)``
+    ## ``yTypeBoolTrue``  ``y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON``
+    ## ``yTypeBoolFalse`` ``n|N|no|No|NO|false|False|FALSE|off|Off|OFF``
+    ## ``yTypeNull``      ``~ | null | Null | NULL``
+    ## ``yTypeUnknown``   ``*``
+    ## ================== =========================
+    yTypeInteger, yTypeFloat, yTypeFloatInf, yTypeFloatNaN, yTypeBoolTrue,
+    yTypeBoolFalse, yTypeNull, yTypeUnknown
+
   YamlTypeHintState = enum
     ythInitial,
     ythF, ythFA, ythFAL, ythFALS, ythFALSE,
@@ -28,11 +61,11 @@ type
 
     ythMinus, yth0, ythInt, ythDecimal, ythNumE, ythNumEPlusMinus, ythExponent
 
-macro typeHintStateMachine(c: untyped, content: untyped): stmt =
-  assert content.kind == nnkStmtList
+macro typeHintStateMachine(c: untyped, content: untyped): typed =
+  yAssert content.kind == nnkStmtList
   result = newNimNode(nnkCaseStmt, content).add(copyNimNode(c))
   for branch in content.children:
-    assert branch.kind == nnkOfBranch
+    yAssert branch.kind == nnkOfBranch
     var
       charBranch = newNimNode(nnkOfBranch, branch)
       i = 0
@@ -42,14 +75,14 @@ macro typeHintStateMachine(c: untyped, content: untyped): stmt =
       charBranch.add(copyNimTree(branch[i]))
       inc(i)
     for rule in branch[i].children:
-      assert rule.kind == nnkInfix
-      assert ($rule[0].ident == "=>")
+      yAssert rule.kind == nnkInfix
+      yAssert $rule[0].ident == "=>"
       var stateBranch = newNimNode(nnkOfBranch, rule)
       case rule[1].kind
       of nnkBracket:
         for item in rule[1].children: stateBranch.add(item)
       of nnkIdent: stateBranch.add(rule[1])
-      else: assert false
+      else: internalError("Invalid rule kind: " & $rule[1].kind)
       if rule[2].kind == nnkNilLit:
         stateBranch.add(newStmtList(newNimNode(nnkDiscardStmt).add(
                         newEmptyNode())))
@@ -150,7 +183,9 @@ template advanceTypeHint(ch: char) {.dirty.} =
     ythTR => ythTRU
   of 'y', 'Y': ythInitial => ythY
 
-proc guessType*(scalar: string): TypeHint =
+proc guessType*(scalar: string): TypeHint {.raises: [].} =
+  ## Parse scalar string according to the RegEx table documented at
+  ## `TypeHint <#TypeHind>`_.
   var typeHintState: YamlTypeHintState = ythInitial
   for c in scalar: advanceTypeHint(c)
   case typeHintState
@@ -161,4 +196,4 @@ proc guessType*(scalar: string): TypeHint =
   of ythDecimal, ythExponent: result = yTypeFloat
   of ythPointINF: result = yTypeFloatInf
   of ythPointNAN: result = yTypeFloatNaN
-  else: result = yTypeUnknown
+  else: result = yTypeUnknown

yaml/presenter.nim

@@ -1,23 +1,126 @@
 #            NimYAML - YAML implementation in Nim
-#        (c) Copyright 2015 Felix Krause
+#        (c) Copyright 2016 Felix Krause
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 
+## =====================
+## Module yaml.presenter
+## =====================
+##
+## This is the presenter API, used for generating YAML character streams.
+
+import streams, queues, strutils
+import taglib, stream, ../private/internal, hints, parser, stream
+
 type
+  PresentationStyle* = enum
+    ## Different styles for YAML character stream output.
+    ##
+    ## - ``ypsMinimal``: Single-line flow-only output which tries to
+    ##   use as few characters as possible.
+    ## - ``ypsCanonical``: Canonical YAML output. Writes all tags except
+    ##   for the non-specific tags ``?`` and ``!``, uses flow style, quotes
+    ##   all string scalars.
+    ## - ``ypsDefault``: Tries to be as human-readable as possible. Uses
+    ##   block style by default, but tries to condense mappings and
+    ##   sequences which only contain scalar nodes into a single line using
+    ##   flow style.
+    ## - ``ypsJson``: Omits the ``%YAML`` directive and the ``---``
+    ##   marker. Uses flow style. Flattens anchors and aliases, omits tags.
+    ##   Output will be parseable as JSON. ``YamlStream`` to dump may only
+    ##   contain one document.
+    ## - ``ypsBlockOnly``: Formats all output in block style, does not use
+    ##   flow style at all.
+    psMinimal, psCanonical, psDefault, psJson, psBlockOnly
+
+  TagStyle* = enum
+    ## Whether object should be serialized with explicit tags.
+    ##
+    ## - ``tsNone``: No tags will be outputted unless necessary.
+    ## - ``tsRootOnly``: A tag will only be outputted for the root tag and
+    ##   where necessary.
+    ## - ``tsAll``: Tags will be outputted for every object.
+    tsNone, tsRootOnly, tsAll
+
+  AnchorStyle* = enum
+    ## How ref object should be serialized.
+    ##
+    ## - ``asNone``: No anchors will be outputted. Values present at
+    ##   multiple places in the content that should be serialized will be
+    ##   fully serialized at every occurence. If the content is cyclic, this
+    ##   will lead to an endless loop!
+    ## - ``asTidy``: Anchors will only be generated for objects that
+    ##   actually occur more than once in the content to be serialized.
+    ##   This is a bit slower and needs more memory than ``asAlways``.
+    ## - ``asAlways``: Achors will be generated for every ref object in the
+    ##   content to be serialized, regardless of whether the object is
+    ##   referenced again afterwards
+    asNone, asTidy, asAlways
+
+  NewLineStyle* = enum
+    ## What kind of newline sequence is used when presenting.
+    ##
+    ## - ``nlLF``: Use a single linefeed char as newline.
+    ## - ``nlCRLF``: Use a sequence of carriage return and linefeed as
+    ##   newline.
+    ## - ``nlOSDefault``: Use the target operation system's default newline
+    ##   sequence (CRLF on Windows, LF everywhere else).
+    nlLF, nlCRLF, nlOSDefault
+
+  OutputYamlVersion* = enum
+    ## Specify which YAML version number the presenter shall emit. The
+    ## presenter will always emit content that is valid YAML 1.1, but by
+    ## default will write a directive ``%YAML 1.2``. For compatibility with
+    ## other YAML implementations, it is possible to change this here.
+    ##
+    ## It is also possible to specify that the presenter shall not emit any
+    ## YAML version. The generated content is then guaranteed to be valid
+    ## YAML 1.1 and 1.2 (but not 1.0 or any newer YAML version).
+    ov1_2, ov1_1, ovNone
+
+  PresentationOptions* = object
+    ## Options for generating a YAML character stream
+    style*: PresentationStyle
+    indentationStep*: int
+    newlines*: NewLineStyle
+    outputVersion*: OutputYamlVersion
+
+  YamlPresenterJsonError* = object of Exception
+    ## Exception that may be raised by the YAML presenter when it is
+    ## instructed to output JSON, but is unable to do so. This may occur if:
+    ##
+    ## - The given `YamlStream <#YamlStream>`_ contains a map which has any
+    ##   non-scalar type as key.
+    ## - Any float scalar bears a ``NaN`` or positive/negative infinity value
+
+  YamlPresenterOutputError* = object of Exception
+    ## Exception that may be raised by the YAML presenter. This occurs if
+    ## writing character data to the output stream raises any exception.
+    ## The error that has occurred is available from ``parent``.
+
   DumperState = enum
     dBlockExplicitMapKey, dBlockImplicitMapKey, dBlockMapValue,
     dBlockInlineMap, dBlockSequenceItem, dFlowImplicitMapKey, dFlowMapValue,
     dFlowExplicitMapKey, dFlowSequenceItem, dFlowMapStart, dFlowSequenceStart
-  
+
   ScalarStyle = enum
     sLiteral, sFolded, sPlain, sDoubleQuoted
 
+  PresenterTarget = Stream | ptr[string]
+
+const
+  defaultPresentationOptions* =
+        PresentationOptions(style: psDefault, indentationStep: 2,
+                            newlines: nlOSDefault)
+
 proc defineOptions*(style: PresentationStyle = psDefault,
                     indentationStep: int = 2,
                     newlines: NewLineStyle = nlOSDefault,
                     outputVersion: OutputYamlVersion = ov1_2):
-    PresentationOptions =
+    PresentationOptions {.raises: [].} =
+  ## Define a set of options for presentation. Convenience proc that requires
+  ## you to only set those values that should not equal the default.
   PresentationOptions(style: style, indentationStep: indentationStep,
                       newlines: newlines, outputVersion: outputVersion)
 
@@ -94,110 +197,118 @@ proc inspect(scalar: string, indentation: int,
   elif canUseFolded: result = sFolded
   elif canUsePlain: result = sPlain
   else: result = sDoubleQuoted
-    
-proc writeDoubleQuoted(scalar: string, s: Stream, indentation: int,
+
+template append(target: Stream, val: string | char) =
+  target.write(val)
+
+template append(target: ptr[string], val: string | char) =
+  target[].add(val)
+
+proc writeDoubleQuoted(scalar: string, s: PresenterTarget, indentation: int,
                        newline: string)
             {.raises: [YamlPresenterOutputError].} =
   var curPos = indentation
   try:
-    s.write('"')
+    s.append('"')
     curPos.inc()
     for c in scalar:
       if curPos == 79:
-        s.write('\\')
-        s.write(newline)
-        s.write(repeat(' ', indentation))
+        s.append('\\')
+        s.append(newline)
+        s.append(repeat(' ', indentation))
         curPos = indentation
         if c == ' ':
-          s.write('\\')
+          s.append('\\')
           curPos.inc()
       case c
       of '"':
-        s.write("\\\"")
+        s.append("\\\"")
         curPos.inc(2)
       of '\l':
-        s.write("\\n")
+        s.append("\\n")
         curPos.inc(2)
       of '\t':
-        s.write("\\t")
+        s.append("\\t")
         curPos.inc(2)
       of '\\':
-        s.write("\\\\")
+        s.append("\\\\")
         curPos.inc(2)
       else:
         if ord(c) < 32:
-          s.write("\\x" & toHex(ord(c), 2))
+          s.append("\\x" & toHex(ord(c), 2))
           curPos.inc(4)
         else:
-          s.write(c)
+          s.append(c)
           curPos.inc()
-    s.write('"')
+    s.append('"')
   except:
     var e = newException(YamlPresenterOutputError,
                          "Error while writing to output stream")
     e.parent = getCurrentException()
     raise e
 
-proc writeDoubleQuotedJson(scalar: string, s: Stream)
+proc writeDoubleQuotedJson(scalar: string, s: PresenterTarget)
     {.raises: [YamlPresenterOutputError].} =
   try:
-    s.write('"')
+    s.append('"')
     for c in scalar:
       case c
-      of '"': s.write("\\\"")
-      of '\\': s.write("\\\\")
-      of '\l': s.write("\\n")
-      of '\t': s.write("\\t")
-      of '\f': s.write("\\f")
-      of '\b': s.write("\\b")
+      of '"': s.append("\\\"")
+      of '\\': s.append("\\\\")
+      of '\l': s.append("\\n")
+      of '\t': s.append("\\t")
+      of '\f': s.append("\\f")
+      of '\b': s.append("\\b")
       else:
-        if ord(c) < 32: s.write("\\u" & toHex(ord(c), 4)) else: s.write(c)
-    s.write('"')
+        if ord(c) < 32: s.append("\\u" & toHex(ord(c), 4)) else: s.append(c)
+    s.append('"')
   except:
     var e = newException(YamlPresenterOutputError,
                          "Error while writing to output stream")
     e.parent = getCurrentException()
     raise e
 
-proc writeLiteral(scalar: string, indentation, indentStep: int, s: Stream,
-                  lines: seq[tuple[start, finish: int]], newline: string)
+proc writeLiteral(scalar: string, indentation, indentStep: int,
+                  s: PresenterTarget, lines: seq[tuple[start, finish: int]],
+                  newline: string)
     {.raises: [YamlPresenterOutputError].} =
   try:
-    s.write('|')
-    if scalar[^1] != '\l': s.write('-')
-    if scalar[0] in [' ', '\t']: s.write($indentStep)
+    s.append('|')
+    if scalar[^1] != '\l': s.append('-')
+    if scalar[0] in [' ', '\t']: s.append($indentStep)
     for line in lines:
-      s.write(newline)
-      s.write(repeat(' ', indentation + indentStep))
+      s.append(newline)
+      s.append(repeat(' ', indentation + indentStep))
       if line.finish >= line.start:
-        s.write(scalar[line.start .. line.finish])
+        s.append(scalar[line.start .. line.finish])
   except:
     var e = newException(YamlPresenterOutputError,
                          "Error while writing to output stream")
     e.parent = getCurrentException()
     raise e
 
-proc writeFolded(scalar: string, indentation, indentStep: int, s: Stream,
-                 words: seq[tuple[start, finish: int]], newline: string)
+proc writeFolded(scalar: string, indentation, indentStep: int,
+                 s: PresenterTarget, words: seq[tuple[start, finish: int]],
+                 newline: string)
     {.raises: [YamlPresenterOutputError].} =
   try:
-    s.write(">")
-    if scalar[^1] != '\l': s.write('-')
-    if scalar[0] in [' ', '\t']: s.write($indentStep)
+    s.append(">")
+    if scalar[^1] != '\l': s.append('-')
+    if scalar[0] in [' ', '\t']: s.append($indentStep)
     var curPos = 80
     for word in words:
       if word.start > 0 and scalar[word.start - 1] == '\l':
-        s.write(newline & newline)
-        s.write(repeat(' ', indentation + indentStep))
+        s.append(newline & newline)
+        s.append(repeat(' ', indentation + indentStep))
         curPos = indentation + indentStep
       elif curPos + (word.finish - word.start) > 80:
-        s.write(newline)
-        s.write(repeat(' ', indentation + indentStep))
+        s.append(newline)
+        s.append(repeat(' ', indentation + indentStep))
         curPos = indentation + indentStep
       else:
-        s.write(' ')
+        s.append(' ')
         curPos.inc()
-      s.write(scalar[word.start .. word.finish])
+      s.append(scalar[word.start .. word.finish])
       curPos += word.finish - word.start + 1
   except:
     var e = newException(YamlPresenterOutputError,
@@ -205,82 +316,82 @@ proc writeFolded(scalar: string, indentation, indentStep: int, s: Stream,
     e.parent = getCurrentException()
     raise e
 
-template safeWrite(s: string or char) {.dirty.} =
-  try: target.write(s)
+template safeWrite(target: PresenterTarget, s: string or char) =
+  try: target.append(s)
   except:
     var e = newException(YamlPresenterOutputError, "")
     e.parent = getCurrentException()
     raise e
 
-proc startItem(target: Stream, style: PresentationStyle, indentation: int,
-               state: var DumperState, isObject: bool, newline: string)
-    {.raises: [YamlPresenterOutputError].} =
+proc startItem(target: PresenterTarget, style: PresentationStyle,
+               indentation: int, state: var DumperState, isObject: bool,
+               newline: string) {.raises: [YamlPresenterOutputError].} =
   try:
     case state
     of dBlockMapValue:
-      target.write(newline)
-      target.write(repeat(' ', indentation))
+      target.append(newline)
+      target.append(repeat(' ', indentation))
       if isObject or style == psCanonical:
-        target.write("? ")
+        target.append("? ")
         state = dBlockExplicitMapKey
       else: state = dBlockImplicitMapKey
     of dBlockInlineMap: state = dBlockImplicitMapKey
     of dBlockExplicitMapKey:
-      target.write(newline)
-      target.write(repeat(' ', indentation))
-      target.write(": ")
+      target.append(newline)
+      target.append(repeat(' ', indentation))
+      target.append(": ")
       state = dBlockMapValue
     of dBlockImplicitMapKey:
-      target.write(": ")
+      target.append(": ")
       state = dBlockMapValue
     of dFlowExplicitMapKey:
       if style != psMinimal:
-        target.write(newline)
-        target.write(repeat(' ', indentation))
-      target.write(": ")
+        target.append(newline)
+        target.append(repeat(' ', indentation))
+      target.append(": ")
       state = dFlowMapValue
     of dFlowMapValue:
       if (isObject and style != psMinimal) or style in [psJson, psCanonical]:
-        target.write(',' & newline & repeat(' ', indentation))
+        target.append(',' & newline & repeat(' ', indentation))
         if style == psJson: state = dFlowImplicitMapKey
         else:
-          target.write("? ")
+          target.append("? ")
           state = dFlowExplicitMapKey
       elif isObject and style == psMinimal:
-        target.write(", ? ")
+        target.append(", ? ")
         state = dFlowExplicitMapKey
       else:
-        target.write(", ")
+        target.append(", ")
         state = dFlowImplicitMapKey
     of dFlowMapStart:
       if (isObject and style != psMinimal) or style in [psJson, psCanonical]:
-        target.write(newline & repeat(' ', indentation))
+        target.append(newline & repeat(' ', indentation))
         if style == psJson: state = dFlowImplicitMapKey
         else:
-          target.write("? ")
+          target.append("? ")
           state = dFlowExplicitMapKey
       else: state = dFlowImplicitMapKey
     of dFlowImplicitMapKey:
-      target.write(": ")
+      target.append(": ")
       state = dFlowMapValue
     of dBlockSequenceItem:
-      target.write(newline)
-      target.write(repeat(' ', indentation))
-      target.write("- ")
+      target.append(newline)
+      target.append(repeat(' ', indentation))
+      target.append("- ")
     of dFlowSequenceStart:
       case style
       of psMinimal, psDefault: discard
       of psCanonical, psJson:
-        target.write(newline)
-        target.write(repeat(' ', indentation))
+        target.append(newline)
+        target.append(repeat(' ', indentation))
       of psBlockOnly: discard # can never happen
       state = dFlowSequenceItem
     of dFlowSequenceItem:
       case style
-      of psMinimal, psDefault: target.write(", ")
+      of psMinimal, psDefault: target.append(", ")
       of psCanonical, psJson:
-        target.write(',' & newline)
-        target.write(repeat(' ', indentation))
+        target.append(',' & newline)
+        target.append(repeat(' ', indentation))
       of psBlockOnly: discard # can never happen
   except:
     var e = newException(YamlPresenterOutputError, "")
@@ -296,26 +407,27 @@ proc anchorName(a: AnchorId): string {.raises: [].} =
         else: result.add(char(j + ord('0') - 26))
         i -= 36
 
-proc writeTagAndAnchor(target: Stream, tag: TagId, tagLib: TagLibrary,
+proc writeTagAndAnchor(target: PresenterTarget, tag: TagId,
+                       tagLib: TagLibrary,
                        anchor: AnchorId) {.raises:[YamlPresenterOutputError].} =
   try:
     if tag notin [yTagQuestionMark, yTagExclamationMark]:
       let tagUri = tagLib.uri(tag)
       if tagUri.startsWith(tagLib.secondaryPrefix):
-        target.write("!!")
-        target.write(tagUri[18..tagUri.high])
-        target.write(' ')
+        target.append("!!")
+        target.append(tagUri[18..tagUri.high])
+        target.append(' ')
       elif tagUri.startsWith("!"):
-        target.write(tagUri)
-        target.write(' ')
+        target.append(tagUri)
+        target.append(' ')
       else:
-        target.write("!<")
-        target.write(tagUri)
-        target.write("> ")
+        target.append("!<")
+        target.append(tagUri)
+        target.append("> ")
     if anchor != yAnchorNone:
-      target.write("&")
-      target.write(anchorName(anchor))
-      target.write(' ')
+      target.append("&")
+      target.append(anchorName(anchor))
+      target.append(' ')
   except:
     var e = newException(YamlPresenterOutputError, "")
     e.parent = getCurrentException()
@@ -325,11 +437,12 @@ proc nextItem(c: var Queue, s: var YamlStream):
     YamlStreamEvent {.raises: [YamlStreamError].} =
   if c.len > 0:
     try: result = c.dequeue
-    except IndexError: assert false
+    except IndexError: internalError("Unexpected IndexError")
   else:
     result = s.next()
 
-proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary,
+proc doPresent(s: var YamlStream, target: PresenterTarget,
+              tagLib: TagLibrary,
               options: PresentationOptions = defaultPresentationOptions) =
   var
     indentation = 0
@@ -345,45 +458,45 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary,
         # TODO: tag directives
         try:
           case options.outputVersion
-          of ov1_2: target.write("%YAML 1.2" & newline)
-          of ov1_1: target.write("%YAML 1.1" & newLine)
+          of ov1_2: target.append("%YAML 1.2" & newline)
+          of ov1_1: target.append("%YAML 1.1" & newLine)
           of ovNone: discard
           if tagLib.secondaryPrefix != yamlTagRepositoryPrefix:
-            target.write("%TAG !! " & tagLib.secondaryPrefix & newline)
-          target.write("--- ")
+            target.append("%TAG !! " & tagLib.secondaryPrefix & newline)
+          target.append("--- ")
         except:
           var e = newException(YamlPresenterOutputError, "")
           e.parent = getCurrentException()
           raise e
     of yamlScalar:
       if levels.len == 0:
-        if options.style != psJson: safeWrite(newline)
+        if options.style != psJson: target.safeWrite(newline)
       else:
         startItem(target, options.style, indentation,
                   levels[levels.high], false, newline)
       if options.style != psJson:
         writeTagAndAnchor(target, item.scalarTag, tagLib, item.scalarAnchor)
-        
+
       if options.style == psJson:
         let hint = guessType(item.scalarContent)
         if item.scalarTag in [yTagQuestionMark, yTagBoolean] and
               hint in {yTypeBoolTrue, yTypeBoolFalse}:
-          safeWrite(if hint == yTypeBoolTrue: "true" else: "false")
+          target.safeWrite(if hint == yTypeBoolTrue: "true" else: "false")
         elif item.scalarTag in [yTagQuestionMark, yTagNull] and
             hint == yTypeNull:
-          safeWrite("null")
+          target.safeWrite("null")
         elif item.scalarTag in [yTagQuestionMark, yTagInteger,
             yTagNimInt8, yTagNimInt16, yTagNimInt32, yTagNimInt64,
             yTagNimUInt8, yTagNimUInt16, yTagNimUInt32, yTagNimUInt64] and
             hint == yTypeInteger:
-          safeWrite(item.scalarContent)
+          target.safeWrite(item.scalarContent)
         elif item.scalarTag in [yTagQuestionMark, yTagFloat, yTagNimFloat32,
             yTagNimFloat64] and hint in {yTypeFloatInf, yTypeFloatNaN}:
           raise newException(YamlPresenterJsonError,
               "Infinity and not-a-number values cannot be presented as JSON!")
         elif item.scalarTag in [yTagQuestionMark, yTagFloat] and
             hint == yTypeFloat:
-          safeWrite(item.scalarContent)
+          target.safeWrite(item.scalarContent)
         else: writeDoubleQuotedJson(item.scalarContent, target)
       elif options.style == psCanonical:
         writeDoubleQuoted(item.scalarContent, target,
@@ -396,19 +509,19 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary,
                         options.indentationStep, target, lines, newline)
         of sFolded: writeFolded(item.scalarContent, indentation,
                         options.indentationStep, target, words, newline)
-        of sPlain: safeWrite(item.scalarContent)
+        of sPlain: target.safeWrite(item.scalarContent)
         of sDoubleQuoted: writeDoubleQuoted(item.scalarContent, target,
                         indentation + options.indentationStep, newline)
     of yamlAlias:
       if options.style == psJson:
         raise newException(YamlPresenterJsonError,
                            "Alias not allowed in JSON output")
-      assert levels.len > 0
+      yAssert levels.len > 0
       startItem(target, options.style, indentation, levels[levels.high],
                 false, newline)
       try:
-        target.write('*')
-        target.write(cast[byte]('a') + cast[byte](item.aliasTarget))
+        target.append('*')
+        target.append(char(byte('a') + byte(item.aliasTarget)))
       except:
         var e = newException(YamlPresenterOutputError, "")
         e.parent = getCurrentException()
@@ -419,7 +532,7 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary,
       of psDefault:
         var length = 0
         while true:
-          assert(not(s.finished()))
+          yAssert(not s.finished())
           let next = s.next()
           cached.enqueue(next)
           case next.kind
@@ -437,27 +550,27 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary,
                              "Cannot have sequence as map key in JSON output!")
         nextState = dFlowSequenceStart
       of psMinimal, psCanonical: nextState = dFlowSequenceStart
-      of psBlockOnly: nextState = dBlockSequenceItem 
-      
+      of psBlockOnly: nextState = dBlockSequenceItem
+
       if levels.len == 0:
         case nextState
         of dBlockSequenceItem:
           if options.style != psJson:
             writeTagAndAnchor(target, item.seqTag, tagLib, item.seqAnchor)
         of dFlowSequenceStart:
-          safeWrite(newline)
+          target.safeWrite(newline)
           if options.style != psJson:
             writeTagAndAnchor(target, item.seqTag, tagLib, item.seqAnchor)
           indentation += options.indentationStep
-        else: assert false
+        else: internalError("Invalid nextState: " & $nextState)
       else:
         startItem(target, options.style, indentation,
                   levels[levels.high], true, newline)
         if options.style != psJson:
           writeTagAndAnchor(target, item.seqTag, tagLib, item.seqAnchor)
         indentation += options.indentationStep
-          
-      if nextState == dFlowSequenceStart: safeWrite('[')
+
+      if nextState == dFlowSequenceStart: target.safeWrite('[')
       if levels.len > 0 and options.style in [psJson, psCanonical] and
           levels[levels.high] in [dBlockExplicitMapKey, dBlockMapValue,
                                   dBlockImplicitMapKey, dBlockSequenceItem]:
@@ -496,16 +609,16 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary,
             writeTagAndAnchor(target, item.mapTag, tagLib, item.mapAnchor)
           else:
             if options.style != psJson:
-              safeWrite(newline)
+              target.safeWrite(newline)
               writeTagAndAnchor(target, item.mapTag, tagLib, item.mapAnchor)
             indentation += options.indentationStep
         of dFlowMapStart:
-          safeWrite(newline)
+          target.safeWrite(newline)
           if options.style != psJson:
             writeTagAndAnchor(target, item.mapTag, tagLib, item.mapAnchor)
           indentation += options.indentationStep
         of dBlockInlineMap: discard
-        else: assert false
+        else: internalError("Invalid nextState: " & $nextState)
       else:
         if nextState in [dBlockMapValue, dBlockImplicitMapKey]:
           startItem(target, options.style, indentation,
@@ -518,27 +631,27 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary,
           if options.style != psJson:
             writeTagAndAnchor(target, item.mapTag, tagLib, item.mapAnchor)
         indentation += options.indentationStep
-        
-      if nextState == dFlowMapStart: safeWrite('{')
+
+      if nextState == dFlowMapStart: target.safeWrite('{')
       if levels.len > 0 and options.style in [psJson, psCanonical] and
           levels[levels.high] in
           [dBlockExplicitMapKey, dBlockMapValue,
            dBlockImplicitMapKey, dBlockSequenceItem]:
         indentation += options.indentationStep
       levels.add(nextState)
-        
+
     of yamlEndSeq:
-      assert levels.len > 0
+      yAssert levels.len > 0
       case levels.pop()
       of dFlowSequenceItem:
         case options.style
-        of psDefault, psMinimal, psBlockOnly: safeWrite(']')
+        of psDefault, psMinimal, psBlockOnly: target.safeWrite(']')
         of psJson, psCanonical:
           indentation -= options.indentationStep
           try:
-            target.write(newline)
-            target.write(repeat(' ', indentation))
-            target.write(']')
+            target.append(newline)
+            target.append(repeat(' ', indentation))
+            target.append(']')
           except:
             var e = newException(YamlPresenterOutputError, "")
             e.parent = getCurrentException()
@@ -552,23 +665,23 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary,
             levels[levels.high] in [dBlockExplicitMapKey, dBlockMapValue,
                                     dBlockImplicitMapKey, dBlockSequenceItem]:
           indentation -= options.indentationStep
-        safeWrite(']')
+        target.safeWrite(']')
       of dBlockSequenceItem: discard
-      else: assert false
+      else: internalError("Invalid popped level")
       indentation -= options.indentationStep
     of yamlEndMap:
-      assert levels.len > 0
+      yAssert levels.len > 0
       let level = levels.pop()
       case level
       of dFlowMapValue:
         case options.style
-        of psDefault, psMinimal, psBlockOnly: safeWrite('}')
+        of psDefault, psMinimal, psBlockOnly: target.safeWrite('}')
         of psJson, psCanonical:
           indentation -= options.indentationStep
           try:
-            target.write(newline)
-            target.write(repeat(' ', indentation))
-            target.write('}')
+            target.append(newline)
+            target.append(repeat(' ', indentation))
+            target.append('}')
           except:
             var e = newException(YamlPresenterOutputError, "")
             e.parent = getCurrentException()
@@ -582,57 +695,89 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary,
             levels[levels.high] in [dBlockExplicitMapKey, dBlockMapValue,
                                     dBlockImplicitMapKey, dBlockSequenceItem]:
           indentation -= options.indentationStep
-        safeWrite('}')
+        target.safeWrite('}')
       of dBlockMapValue, dBlockInlineMap: discard
-      else: assert(false)
+      else: internalError("Invalid level: " & $level)
       indentation -= options.indentationStep
     of yamlEndDoc:
       if finished(s): break
       if options.style == psJson:
         raise newException(YamlPresenterJsonError,
             "Cannot output more than one document in JSON style")
-      safeWrite("..." & newline)
+      target.safeWrite("..." & newline)
 
-proc transform*(input: Stream, output: Stream,
-                options: PresentationOptions = defaultPresentationOptions) =
+proc present*(s: var YamlStream, target: Stream,
+              tagLib: TagLibrary,
+              options: PresentationOptions = defaultPresentationOptions)
+    {.raises: [YamlPresenterJsonError, YamlPresenterOutputError,
+               YamlStreamError].} =
+  ## Convert ``s`` to a YAML character stream and write it to ``target``.
+  doPresent(s, target, tagLib, options)
+
+proc present*(s: var YamlStream, tagLib: TagLibrary,
+              options: PresentationOptions = defaultPresentationOptions):
+    string {.raises: [YamlPresenterJsonError, YamlPresenterOutputError,
+                      YamlStreamError].} =
+  ## Convert ``s`` to a YAML character stream and return it as string.
+  result = ""
+  doPresent(s, addr result, tagLib, options)
+
+proc doTransform(input: Stream | string, output: PresenterTarget,
+                 options: PresentationOptions = defaultPresentationOptions) =
   var
     taglib = initExtendedTagLibrary()
     parser = newYamlParser(tagLib)
     events = parser.parse(input)
   try:
     if options.style == psCanonical:
-      var specificTagEvents = iterator(): YamlStreamEvent =
-        for e in events:
-          var event = e
-          case event.kind
-          of yamlStartDoc, yamlEndDoc, yamlEndMap, yamlAlias, yamlEndSeq:
-            discard
-          of yamlStartMap:
-            if event.mapTag in [yTagQuestionMark, yTagExclamationMark]:
-              event.mapTag = yTagMapping
-          of yamlStartSeq:
-            if event.seqTag in [yTagQuestionMark, yTagExclamationMark]:
-              event.seqTag = yTagSequence
-          of yamlScalar:
-            if event.scalarTag == yTagQuestionMark:
-              case guessType(event.scalarContent)
-              of yTypeInteger: event.scalarTag = yTagInteger
-              of yTypeFloat, yTypeFloatInf, yTypeFloatNaN:
-                event.scalarTag = yTagFloat
-              of yTypeBoolTrue, yTypeBoolFalse: event.scalarTag = yTagBoolean
-              of yTypeNull: event.scalarTag = yTagNull
-              of yTypeUnknown: event.scalarTag = yTagString
-            elif event.scalarTag == yTagExclamationMark:
-              event.scalarTag = yTagString
-          yield event
-      var s = initYamlStream(specificTagEvents)
-      present(s, output, tagLib, options)
+      var bys: YamlStream = newBufferYamlStream()
+      for e in events:
+        var event = e
+        case event.kind
+        of yamlStartDoc, yamlEndDoc, yamlEndMap, yamlAlias, yamlEndSeq:
+          discard
+        of yamlStartMap:
+          if event.mapTag in [yTagQuestionMark, yTagExclamationMark]:
+            event.mapTag = yTagMapping
+        of yamlStartSeq:
+          if event.seqTag in [yTagQuestionMark, yTagExclamationMark]:
+            event.seqTag = yTagSequence
+        of yamlScalar:
+          if event.scalarTag == yTagQuestionMark:
+            case guessType(event.scalarContent)
+            of yTypeInteger: event.scalarTag = yTagInteger
+            of yTypeFloat, yTypeFloatInf, yTypeFloatNaN:
+              event.scalarTag = yTagFloat
+            of yTypeBoolTrue, yTypeBoolFalse: event.scalarTag = yTagBoolean
+            of yTypeNull: event.scalarTag = yTagNull
+            of yTypeUnknown: event.scalarTag = yTagString
+          elif event.scalarTag == yTagExclamationMark:
+            event.scalarTag = yTagString
+        BufferYamlStream(bys).put(e)
+      present(bys, output, tagLib, options)
     else: present(events, output, tagLib, options)
   except YamlStreamError:
     var e = getCurrentException()
     while e.parent of YamlStreamError: e = e.parent
     if e.parent of IOError: raise (ref IOError)(e.parent)
     elif e.parent of YamlParserError: raise (ref YamlParserError)(e.parent)
-    else:
-      # never happens
-      assert(false)
+    else: internalError("Unexpected exception: " & e.parent.repr)
+
+proc transform*(input: Stream | string, output: Stream,
+                options: PresentationOptions = defaultPresentationOptions)
+    {.raises: [IOError, YamlParserError, YamlPresenterJsonError,
+               YamlPresenterOutputError].} =
+  ## Parser ``input`` as YAML character stream and then dump it to ``output``
+  ## while resolving non-specific tags to the ones in the YAML core tag
+  ## library.
+  doTransform(input, output, options)
+
+proc transform*(input: Stream | string,
+                options: PresentationOptions = defaultPresentationOptions):
+    string {.raises: [IOError, YamlParserError, YamlPresenterJsonError,
+                      YamlPresenterOutputError].} =
+  ## Parser ``input`` as YAML character stream, resolves non-specific tags to
+  ## the ones in the YAML core tag library, and then returns a serialized
+  ## YAML string that represents the stream.
+  result = ""
+  doTransform(input, addr result, options)

yaml/stream.nim

@@ -0,0 +1,305 @@
+#            NimYAML - YAML implementation in Nim
+#        (c) Copyright 2016 Felix Krause
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+
+## ==================
+## Module yaml.stream
+## ==================
+##
+## The stream API provides the basic data structure on which all low-level APIs
+## operate. It is not named ``streams`` to not confuse it with the modle in the
+## stdlib with that name.
+
+import hashes
+import ../private/internal, taglib
+
+type
+  AnchorId* = distinct int ## \
+    ## An ``AnchorId`` identifies an anchor in the current document. It
+    ## becomes invalid as soon as the current document scope is invalidated
+    ## (for example, because the parser yielded a ``yamlEndDocument``
+    ## event). ``AnchorId`` s exists because of efficiency, much like
+    ## ``TagId`` s. The actual anchor name is a presentation detail and
+    ## cannot be queried by the user.
+
+  YamlStreamEventKind* = enum
+    ## Kinds of YAML events that may occur in an ``YamlStream``. Event kinds
+    ## are discussed in `YamlStreamEvent <#YamlStreamEvent>`_.
+    yamlStartDoc, yamlEndDoc, yamlStartMap, yamlEndMap,
+    yamlStartSeq, yamlEndSeq, yamlScalar, yamlAlias
+
+  YamlStreamEvent* = object
+    ## An element from a `YamlStream <#YamlStream>`_. Events that start an
+    ## object (``yamlStartMap``, ``yamlStartSeq``, ``yamlScalar``) have
+    ## an optional anchor and a tag associated with them. The anchor will be
+    ## set to ``yAnchorNone`` if it doesn't exist.
+    ##
+    ## A non-existing tag in the YAML character stream will be resolved to
+    ## the non-specific tags ``?`` or ``!`` according to the YAML
+    ## specification. These are by convention mapped to the ``TagId`` s
+    ## ``yTagQuestionMark`` and ``yTagExclamationMark`` respectively.
+    ## Mapping is done by a `TagLibrary <#TagLibrary>`_.
+    case kind*: YamlStreamEventKind
+    of yamlStartMap:
+      mapAnchor* : AnchorId
+      mapTag*    : TagId
+    of yamlStartSeq:
+      seqAnchor* : AnchorId
+      seqTag*    : TagId
+    of yamlScalar:
+      scalarAnchor* : AnchorId
+      scalarTag*    : TagId
+      scalarContent*: string # may not be nil (but empty)
+    of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard
+    of yamlAlias:
+      aliasTarget* : AnchorId
+
+  YamlStream* = ref object of RootObj ## \
+    ## A ``YamlStream`` is an iterator-like object that yields a
+    ## well-formed stream of ``YamlStreamEvents``. Well-formed means that
+    ## every ``yamlStartMap`` is terminated by a ``yamlEndMap``, every
+    ## ``yamlStartSeq`` is terminated by a ``yamlEndSeq`` and every
+    ## ``yamlStartDoc`` is terminated by a ``yamlEndDoc``. Moreover, every
+    ## emitted mapping has an even number of children.
+    ##
+    ## The creator of a ``YamlStream`` is responsible for it being
+    ## well-formed. A user of the stream may assume that it is well-formed
+    ## and is not required to check for it. The procs in this module will
+    ## always yield a well-formed ``YamlStream`` and expect it to be
+    ## well-formed if they take it as input parameter.
+    nextImpl*: proc(s: YamlStream, e: var YamlStreamEvent): bool
+    lastTokenContextImpl*:
+        proc(s: YamlStream, line, column: var int,
+             lineContent: var string): bool {.raises: [].}
+    isFinished*: bool
+    peeked: bool
+    cached: YamlStreamEvent
+
+  YamlStreamError* = object of Exception
+    ## Exception that may be raised by a ``YamlStream`` when the underlying
+    ## backend raises an exception. The error that has occurred is
+    ## available from ``parent``.
+
+const
+  yAnchorNone*: AnchorId = (-1).AnchorId ## \
+    ## yielded when no anchor was defined for a YAML node
+
+proc `==`*(left, right: AnchorId): bool {.borrow.}
+proc `$`*(id: AnchorId): string {.borrow.}
+proc hash*(id: AnchorId): Hash {.borrow.}
+
+proc noLastContext(s: YamlStream, line, column: var int,
+    lineContent: var string): bool {.raises: [].} =
+  (line, column, lineContent) = (-1, -1, "")
+  result = false
+
+proc basicInit*(s: YamlStream, lastTokenContextImpl:
+    proc(s: YamlStream, line, column: var int, lineContent: var string): bool
+    {.raises: [].} = noLastContext)
+    {.raises: [].} =
+  ## initialize basic values of the YamlStream. Call this in your constructor
+  ## if you subclass YamlStream.
+  s.peeked = false
+  s.isFinished = false
+  s.lastTokenContextImpl = lastTokenContextImpl
+
+when not defined(JS):
+  type IteratorYamlStream = ref object of YamlStream
+    backend: iterator(): YamlStreamEvent
+
+  proc initYamlStream*(backend: iterator(): YamlStreamEvent): YamlStream
+      {.raises: [].} =
+    ## Creates a new ``YamlStream`` that uses the given iterator as backend.
+    result = new(IteratorYamlStream)
+    result.basicInit()
+    IteratorYamlStream(result).backend = backend
+    result.nextImpl = proc(s: YamlStream, e: var YamlStreamEvent): bool =
+      e = IteratorYamlStream(s).backend()
+      if finished(IteratorYamlStream(s).backend):
+        s.isFinished = true
+        result = false
+      else: result = true
+
+type
+  BufferYamlStream* = ref object of YamlStream
+    pos: int
+    buf: seq[YamlStreamEvent] not nil
+
+proc newBufferYamlStream*(): BufferYamlStream not nil =
+  result = cast[BufferYamlStream not nil](new(BufferYamlStream))
+  result.basicInit()
+  result.buf = @[]
+  result.pos = 0
+  result.nextImpl = proc(s: YamlStream, e: var YamlStreamEvent): bool =
+    let bys = BufferYamlStream(s)
+    if bys.pos == bys.buf.len:
+      result = false
+      s.isFinished = true
+    else:
+      e = bys.buf[bys.pos]
+      inc(bys.pos)
+      result = true
+
+proc put*(bys: BufferYamlStream, e: YamlStreamEvent) {.raises: [].} =
+  bys.buf.add(e)
+
+proc next*(s: YamlStream): YamlStreamEvent {.raises: [YamlStreamError].} =
+  ## Get the next item of the stream. Requires ``finished(s) == true``.
+  ## If the backend yields an exception, that exception will be encapsulated
+  ## into a ``YamlStreamError``, which will be raised.
+  if s.peeked:
+    s.peeked = false
+    shallowCopy(result, s.cached)
+    return
+  else:
+    yAssert(not s.isFinished)
+    try:
+      while true:
+        if s.nextImpl(s, result): break
+        yAssert(not s.isFinished)
+    except YamlStreamError:
+      let cur = getCurrentException()
+      var e = newException(YamlStreamError, cur.msg)
+      e.parent = cur.parent
+      raise e
+    except Exception:
+      let cur = getCurrentException()
+      var e = newException(YamlStreamError, cur.msg)
+      e.parent = cur
+      raise e
+
+proc peek*(s: YamlStream): YamlStreamEvent {.raises: [YamlStreamError].} =
+  ## Get the next item of the stream without advancing the stream.
+  ## Requires ``finished(s) == true``. Handles exceptions of the backend like
+  ## ``next()``.
+  if not s.peeked:
+    shallowCopy(s.cached, s.next())
+    s.peeked = true
+  shallowCopy(result, s.cached)
+
+proc `peek=`*(s: YamlStream, value: YamlStreamEvent) {.raises: [].} =
+  ## Set the next item of the stream. Will replace a previously peeked item,
+  ## if one exists.
+  s.cached = value
+  s.peeked = true
+
+proc finished*(s: YamlStream): bool {.raises: [YamlStreamError].} =
+  ## ``true`` if no more items are available in the stream. Handles exceptions
+  ## of the backend like ``next()``.
+  if s.peeked: result = false
+  else:
+    try:
+      while true:
+        if s.isFinished: return true
+        if s.nextImpl(s, s.cached):
+          s.peeked = true
+          return false
+    except YamlStreamError:
+      let cur = getCurrentException()
+      var e = newException(YamlStreamError, cur.msg)
+      e.parent = cur.parent
+      raise e
+    except Exception:
+      let cur = getCurrentException()
+      var e = newException(YamlStreamError, cur.msg)
+      e.parent = cur
+      raise e
+
+proc getLastTokenContext*(s: YamlStream, line, column: var int,
+    lineContent: var string): bool =
+  ## ``true`` if source context information is available about the last returned
+  ## token. If ``true``, line, column and lineContent are set to position and
+  ## line content where the last token has been read from.
+  result = s.lastTokenContextImpl(s, line, column, lineContent)
+
+iterator items*(s: YamlStream): YamlStreamEvent
+    {.raises: [YamlStreamError].} =
+  ## Iterate over all items of the stream. You may not use ``peek()`` on the
+  ## stream while iterating.
+  while not s.finished(): yield s.next()
+
+iterator mitems*(bys: BufferYamlStream): var YamlStreamEvent {.raises: [].} =
+  ## Iterate over all items of the stream. You may not use ``peek()`` on the
+  ## stream while iterating.
+  for e in bys.buf.mitems(): yield e
+
+proc `==`*(left: YamlStreamEvent, right: YamlStreamEvent): bool {.raises: [].} =
+  ## compares all existing fields of the given items
+  if left.kind != right.kind: return false
+  case left.kind
+  of yamlStartDoc, yamlEndDoc, yamlEndMap, yamlEndSeq: result = true
+  of yamlStartMap:
+    result = left.mapAnchor == right.mapAnchor and left.mapTag == right.mapTag
+  of yamlStartSeq:
+    result = left.seqAnchor == right.seqAnchor and left.seqTag == right.seqTag
+  of yamlScalar:
+    result = left.scalarAnchor == right.scalarAnchor and
+             left.scalarTag == right.scalarTag and
+             left.scalarContent == right.scalarContent
+  of yamlAlias: result = left.aliasTarget == right.aliasTarget
+
+proc `$`*(event: YamlStreamEvent): string {.raises: [].} =
+  ## outputs a human-readable string describing the given event
+  result = $event.kind & '('
+  case event.kind
+  of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard
+  of yamlStartMap:
+    result &= "tag=" & $event.mapTag
+    if event.mapAnchor != yAnchorNone: result &= ", anchor=" & $event.mapAnchor
+  of yamlStartSeq:
+    result &= "tag=" & $event.seqTag
+    if event.seqAnchor != yAnchorNone: result &= ", anchor=" & $event.seqAnchor
+  of yamlScalar:
+    result &= "tag=" & $event.scalarTag
+    if event.scalarAnchor != yAnchorNone:
+      result &= ", anchor=" & $event.scalarAnchor
+    result &= ", content=\"" & event.scalarContent & '\"'
+  of yamlAlias:
+    result &= "aliasTarget=" & $event.aliasTarget
+  result &= ")"
+
+proc tag*(event: YamlStreamEvent): TagId {.raises: [FieldError].} =
+  ## returns the tag of the given event
+  case event.kind
+  of yamlStartMap: result = event.mapTag
+  of yamlStartSeq: result = event.seqTag
+  of yamlScalar: result = event.scalarTag
+  else: raise newException(FieldError, "Event " & $event.kind & " has no tag")
+
+proc startDocEvent*(): YamlStreamEvent {.inline, raises: [].} =
+  ## creates a new event that marks the start of a YAML document
+  result = YamlStreamEvent(kind: yamlStartDoc)
+
+proc endDocEvent*(): YamlStreamEvent {.inline, raises: [].} =
+  ## creates a new event that marks the end of a YAML document
+  result = YamlStreamEvent(kind: yamlEndDoc)
+
+proc startMapEvent*(tag: TagId = yTagQuestionMark,
+    anchor: AnchorId = yAnchorNone): YamlStreamEvent {.inline, raises: [].} =
+  ## creates a new event that marks the start of a YAML mapping
+  result = YamlStreamEvent(kind: yamlStartMap, mapTag: tag, mapAnchor: anchor)
+
+proc endMapEvent*(): YamlStreamEvent {.inline, raises: [].} =
+  ## creates a new event that marks the end of a YAML mapping
+  result = YamlStreamEvent(kind: yamlEndMap)
+
+proc startSeqEvent*(tag: TagId = yTagQuestionMark,
+    anchor: AnchorId = yAnchorNone): YamlStreamEvent {.inline, raises: [].} =
+  ## creates a new event that marks the beginning of a YAML sequence
+  result = YamlStreamEvent(kind: yamlStartSeq, seqTag: tag, seqAnchor: anchor)
+
+proc endSeqEvent*(): YamlStreamEvent {.inline, raises: [].} =
+  ## creates a new event that marks the end of a YAML sequence
+  result = YamlStreamEvent(kind: yamlEndSeq)
+
+proc scalarEvent*(content: string = "", tag: TagId = yTagQuestionMark,
+    anchor: AnchorId = yAnchorNone): YamlStreamEvent {.inline, raises: [].} =
+  ## creates a new event that represents a YAML scalar
+  result = YamlStreamEvent(kind: yamlScalar, scalarTag: tag,
+                           scalarAnchor: anchor, scalarContent: content)
+
+proc aliasEvent*(anchor: AnchorId): YamlStreamEvent {.inline, raises: [].} =
+  ## creates a new event that represents a YAML alias
+  result = YamlStreamEvent(kind: yamlAlias, aliasTarget: anchor)

yaml/taglib.nim

@@ -0,0 +1,309 @@
+#            NimYAML - YAML implementation in Nim
+#        (c) Copyright 2016 Felix Krause
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+
+## ==================
+## Module yaml.taglib
+## ==================
+##
+## The taglib API enables you to query real names of tags emitted by the parser
+## and create own tags. It also enables you to define tags for types used with
+## the serialization API.
+
+import tables, macros, hashes
+
+type
+  TagId* = distinct int ## \
+    ## A ``TagId`` identifies a tag URI, like for example
+    ## ``"tag:yaml.org,2002:str"``. The URI corresponding to a ``TagId`` can
+    ## be queried from the `TagLibrary <#TagLibrary>`_ which was
+    ## used to create this ``TagId``; e.g. when you parse a YAML character
+    ## stream, the ``TagLibrary`` of the parser is the one which generates
+    ## the resulting ``TagId`` s.
+    ##
+    ## URI strings are mapped to ``TagId`` s for efficiency  reasons (you
+    ## do not need to compare strings every time) and to be able to
+    ## discover unknown tag URIs early in the parsing process.
+
+  TagLibrary* = ref object
+    ## A ``TagLibrary`` maps tag URIs to ``TagId`` s.
+    ##
+    ## When `YamlParser <#YamlParser>`_ encounters tags not existing in the
+    ## tag library, it will use
+    ## `registerUri <#registerUri,TagLibrary,string>`_ to add
+    ## the tag to the library.
+    ##
+    ## You can base your tag library on common tag libraries by initializing
+    ## them with `initFailsafeTagLibrary <#initFailsafeTagLibrary>`_,
+    ## `initCoreTagLibrary <#initCoreTagLibrary>`_ or
+    ## `initExtendedTagLibrary <#initExtendedTagLibrary>`_.
+    tags*: Table[string, TagId]
+    nextCustomTagId*: TagId
+    secondaryPrefix*: string
+
+const
+  # failsafe schema
+
+  yTagExclamationMark*: TagId = 0.TagId ## ``!`` non-specific tag
+  yTagQuestionMark*   : TagId = 1.TagId ## ``?`` non-specific tag
+  yTagString*         : TagId = 2.TagId ## \
+    ## `!!str <http://yaml.org/type/str.html >`_ tag
+  yTagSequence*       : TagId = 3.TagId ## \
+    ## `!!seq <http://yaml.org/type/seq.html>`_ tag
+  yTagMapping*        : TagId = 4.TagId ## \
+    ## `!!map <http://yaml.org/type/map.html>`_ tag
+
+  # json & core schema
+
+  yTagNull*    : TagId = 5.TagId ## \
+    ## `!!null <http://yaml.org/type/null.html>`_ tag
+  yTagBoolean* : TagId = 6.TagId ## \
+    ## `!!bool <http://yaml.org/type/bool.html>`_ tag
+  yTagInteger* : TagId = 7.TagId ## \
+    ## `!!int <http://yaml.org/type/int.html>`_ tag
+  yTagFloat*   : TagId = 8.TagId ## \
+    ## `!!float <http://yaml.org/type/float.html>`_ tag
+
+  # other language-independent YAML types (from http://yaml.org/type/ )
+
+  yTagOrderedMap* : TagId = 9.TagId  ## \
+    ## `!!omap <http://yaml.org/type/omap.html>`_ tag
+  yTagPairs*      : TagId = 10.TagId ## \
+    ## `!!pairs <http://yaml.org/type/pairs.html>`_ tag
+  yTagSet*        : TagId = 11.TagId ## \
+    ## `!!set <http://yaml.org/type/set.html>`_ tag
+  yTagBinary*     : TagId = 12.TagId ## \
+    ## `!!binary <http://yaml.org/type/binary.html>`_ tag
+  yTagMerge*      : TagId = 13.TagId ## \
+    ## `!!merge <http://yaml.org/type/merge.html>`_ tag
+  yTagTimestamp*  : TagId = 14.TagId ## \
+    ## `!!timestamp <http://yaml.org/type/timestamp.html>`_ tag
+  yTagValue*      : TagId = 15.TagId ## \
+    ## `!!value <http://yaml.org/type/value.html>`_ tag
+  yTagYaml*       : TagId = 16.TagId ## \
+    ## `!!yaml <http://yaml.org/type/yaml.html>`_ tag
+
+  yTagNimField*   : TagId = 100.TagId ## \
+    ## This tag is used in serialization for the name of a field of an
+    ## object. It may contain any string scalar that is a valid Nim symbol.
+
+  yTagNimNilString* : TagId = 101.TagId ## for strings that are nil
+  yTagNimNilSeq*    : TagId = 102.TagId ## \
+    ## for seqs that are nil. This tag is used regardless of the seq's generic
+    ## type parameter.
+
+  yFirstCustomTagId* : TagId = 1000.TagId ## \
+    ## The first ``TagId`` which should be assigned to an URI that does not
+    ## exist in the ``YamlTagLibrary`` which is used for parsing.
+
+  yamlTagRepositoryPrefix* = "tag:yaml.org,2002:"
+
+proc `==`*(left, right: TagId): bool {.borrow.}
+proc hash*(id: TagId): Hash {.borrow.}
+
+proc `$`*(id: TagId): string {.raises: [].} =
+  case id
+  of yTagQuestionMark: "?"
+  of yTagExclamationMark: "!"
+  of yTagString: "!!str"
+  of yTagSequence: "!!seq"
+  of yTagMapping: "!!map"
+  of yTagNull: "!!null"
+  of yTagBoolean: "!!bool"
+  of yTagInteger: "!!int"
+  of yTagFloat: "!!float"
+  of yTagOrderedMap: "!!omap"
+  of yTagPairs: "!!pairs"
+  of yTagSet: "!!set"
+  of yTagBinary: "!!binary"
+  of yTagMerge: "!!merge"
+  of yTagTimestamp: "!!timestamp"
+  of yTagValue: "!!value"
+  of yTagYaml: "!!yaml"
+  of yTagNimField: "!nim:field"
+  else: "<" & $int(id) & ">"
+
+proc initTagLibrary*(): TagLibrary {.raises: [].} =
+  ## initializes the ``tags`` table and sets ``nextCustomTagId`` to
+  ## ``yFirstCustomTagId``.
+  new(result)
+  result.tags = initTable[string, TagId]()
+  result.secondaryPrefix = yamlTagRepositoryPrefix
+  result.nextCustomTagId = yFirstCustomTagId
+
+proc registerUri*(tagLib: TagLibrary, uri: string): TagId {.raises: [].} =
+  ## registers a custom tag URI with a ``TagLibrary``. The URI will get
+  ## the ``TagId`` ``nextCustomTagId``, which will be incremented.
+  tagLib.tags[uri] = tagLib.nextCustomTagId
+  result = tagLib.nextCustomTagId
+  tagLib.nextCustomTagId = cast[TagId](cast[int](tagLib.nextCustomTagId) + 1)
+
+proc uri*(tagLib: TagLibrary, id: TagId): string {.raises: [KeyError].} =
+  ## retrieve the URI a ``TagId`` maps to.
+  for iUri, iId in tagLib.tags.pairs:
+    if iId == id: return iUri
+  raise newException(KeyError, "Unknown tag id: " & $id)
+
+proc initFailsafeTagLibrary*(): TagLibrary {.raises: [].} =
+  ## Contains only:
+  ## - ``!``
+  ## - ``?``
+  ## - ``!!str``
+  ## - ``!!map``
+  ## - ``!!seq``
+  result = initTagLibrary()
+  result.tags["!"] = yTagExclamationMark
+  result.tags["?"] = yTagQuestionMark
+  result.tags["tag:yaml.org,2002:str"] = yTagString
+  result.tags["tag:yaml.org,2002:seq"] = yTagSequence
+  result.tags["tag:yaml.org,2002:map"] = yTagMapping
+
+proc initCoreTagLibrary*(): TagLibrary {.raises: [].} =
+  ## Contains everything in ``initFailsafeTagLibrary`` plus:
+  ## - ``!!null``
+  ## - ``!!bool``
+  ## - ``!!int``
+  ## - ``!!float``
+  result = initFailsafeTagLibrary()
+  result.tags["tag:yaml.org,2002:null"]  = yTagNull
+  result.tags["tag:yaml.org,2002:bool"]  = yTagBoolean
+  result.tags["tag:yaml.org,2002:int"]   = yTagInteger
+  result.tags["tag:yaml.org,2002:float"] = yTagFloat
+
+proc initExtendedTagLibrary*(): TagLibrary {.raises: [].} =
+  ## Contains everything from ``initCoreTagLibrary`` plus:
+  ## - ``!!omap``
+  ## - ``!!pairs``
+  ## - ``!!set``
+  ## - ``!!binary``
+  ## - ``!!merge``
+  ## - ``!!timestamp``
+  ## - ``!!value``
+  ## - ``!!yaml``
+  result = initCoreTagLibrary()
+  result.tags["tag:yaml.org,2002:omap"]      = yTagOrderedMap
+  result.tags["tag:yaml.org,2002:pairs"]     = yTagPairs
+  result.tags["tag:yaml.org,2002:binary"]    = yTagBinary
+  result.tags["tag:yaml.org,2002:merge"]     = yTagMerge
+  result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp
+  result.tags["tag:yaml.org,2002:value"]     = yTagValue
+  result.tags["tag:yaml.org,2002:yaml"]      = yTagYaml
+
+
+proc initSerializationTagLibrary*(): TagLibrary =
+  result = initTagLibrary()
+  result.tags["!"] = yTagExclamationMark
+  result.tags["?"] = yTagQuestionMark
+  result.tags["tag:yaml.org,2002:str"]       = yTagString
+  result.tags["tag:yaml.org,2002:null"]      = yTagNull
+  result.tags["tag:yaml.org,2002:bool"]      = yTagBoolean
+  result.tags["tag:yaml.org,2002:float"]     = yTagFloat
+  result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp
+  result.tags["tag:yaml.org,2002:value"]     = yTagValue
+  result.tags["tag:yaml.org,2002:binary"]    = yTagBinary
+  result.tags["!nim:field"]                  = yTagNimField
+  result.tags["!nim:nil:string"]             = yTagNimNilString
+  result.tags["!nim:nil:seq"]                = yTagNimNilSeq
+
+var
+  serializationTagLibrary* = initSerializationTagLibrary() ## \
+    ## contains all local tags that are used for type serialization. Does
+    ## not contain any of the specific default tags for sequences or maps,
+    ## as those are not suited for Nim's static type system.
+    ##
+    ## Should not be modified manually. Will be extended by
+    ## `serializable <#serializable,stmt,stmt>`_.
+
+var
+  nextStaticTagId {.compileTime.} = 100.TagId ## \
+    ## used for generating unique TagIds with ``setTagUri``.
+  registeredUris {.compileTime.} = newSeq[string]() ## \
+    ## Since Table doesn't really work at compile time, we also store
+    ## registered URIs here to be able to generate a static compiler error
+    ## when the user tries to register an URI more than once.
+
+template setTagUri*(t: typedesc, uri: string): typed =
+  ## Associate the given uri with a certain type. This uri is used as YAML tag
+  ## when loading and dumping values of this type.
+  when uri in registeredUris:
+    {. fatal: "[NimYAML] URI \"" & uri & "\" registered twice!" .}
+  const id {.genSym.} = nextStaticTagId
+  static:
+    registeredUris.add(uri)
+    nextStaticTagId = TagId(int(nextStaticTagId) + 1)
+  when nextStaticTagId == yFirstCustomTagId:
+    {.fatal: "Too many tags!".}
+  serializationTagLibrary.tags[uri] = id
+  proc yamlTag*(T: typedesc[t]): TagId {.inline, raises: [].} = id
+    ## autogenerated
+
+template setTagUri*(t: typedesc, uri: string, idName: untyped): typed =
+  ## Like `setTagUri <#setTagUri.t,typedesc,string>`_, but lets
+  ## you choose a symbol for the `TagId <#TagId>`_ of the uri. This is only
+  ## necessary if you want to implement serialization / construction yourself.
+  when uri in registeredUris:
+    {. fatal: "[NimYAML] URI \"" & uri & "\" registered twice!" .}
+  const idName* = nextStaticTagId
+  static:
+    registeredUris.add(uri)
+    nextStaticTagId = TagId(int(nextStaticTagId) + 1)
+  serializationTagLibrary.tags[uri] = idName
+  proc yamlTag*(T: typedesc[t]): TagId {.inline, raises: [].} = idName
+    ## autogenerated
+
+proc canBeImplicit(t: typedesc): bool {.compileTime.} =
+  let tDesc = getType(t)
+  if tDesc.kind != nnkObjectTy: return false
+  if tDesc[2].len != 1: return false
+  if tDesc[2][0].kind != nnkRecCase: return false
+  var foundEmptyBranch = false
+  for i in 1.. tDesc[2][0].len - 1:
+    case tDesc[2][0][i][1].len # branch contents
+    of 0:
+      if foundEmptyBranch: return false
+      else: foundEmptyBranch = true
+    of 1: discard
+    else: return false
+  return true
+
+template markAsImplicit*(t: typedesc): typed =
+  ## Mark a variant object type as implicit. This requires the type to consist
+  ## of nothing but a case expression and each branch of the case expression
+  ## containing exactly one field - with the exception that one branch may
+  ## contain zero fields.
+  when canBeImplicit(t):
+    # this will be checked by means of compiles(implicitVariantObject(...))
+    proc implicitVariantObject*(unused: t) = discard
+  else:
+    {. fatal: "This type cannot be marked as implicit" .}
+
+static:
+  # standard YAML tags used by serialization
+  registeredUris.add("!")
+  registeredUris.add("?")
+  registeredUris.add("tag:yaml.org,2002:str")
+  registeredUris.add("tag:yaml.org,2002:null")
+  registeredUris.add("tag:yaml.org,2002:bool")
+  registeredUris.add("tag:yaml.org,2002:float")
+  registeredUris.add("tag:yaml.org,2002:timestamp")
+  registeredUris.add("tag:yaml.org,2002:value")
+  registeredUris.add("tag:yaml.org,2002:binary")
+  # special tags used by serialization
+  registeredUris.add("!nim:field")
+  registeredUris.add("!nim:nil:string")
+  registeredUris.add("!nim:nil:seq")
+
+# tags for Nim's standard types
+setTagUri(char, "!nim:system:char", yTagNimChar)
+setTagUri(int8, "!nim:system:int8", yTagNimInt8)
+setTagUri(int16, "!nim:system:int16", yTagNimInt16)
+setTagUri(int32, "!nim:system:int32", yTagNimInt32)
+setTagUri(int64, "!nim:system:int64", yTagNimInt64)
+setTagUri(uint8, "!nim:system:uint8", yTagNimUInt8)
+setTagUri(uint16, "!nim:system:uint16", yTagNimUInt16)
+setTagUri(uint32, "!nim:system:uint32", yTagNimUInt32)
+setTagUri(uint64, "!nim:system:uint64", yTagNimUInt64)
+setTagUri(float32, "!nim:system:float32", yTagNimFloat32)
+setTagUri(float64, "!nim:system:float64", yTagNimFloat64)

yaml/tojson.nim

@@ -1,9 +1,19 @@
 #            NimYAML - YAML implementation in Nim
-#        (c) Copyright 2015 Felix Krause
+#        (c) Copyright 2016 Felix Krause
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 
+## ==================
+## Module yaml.tojson
+## ==================
+##
+## The tojson API enables you to parser a YAML character stream into the JSON
+## structures provided by Nim's stdlib.
+
+import json, streams, strutils, tables
+import taglib, hints, serialization, stream, ../private/internal, parser
+
 type Level = tuple[node: JsonNode, key: string]
 
 proc initLevel(node: JsonNode): Level {.raises: [].} =
@@ -13,7 +23,7 @@ proc jsonFromScalar(content: string, tag: TagId): JsonNode
    {.raises: [YamlConstructionError].}=
   new(result)
   var mappedType: TypeHint
-    
+
   case tag
   of yTagQuestionMark: mappedType = guessType(content)
   of yTagExclamationMark, yTagString: mappedType = yTypeUnknown
@@ -35,7 +45,7 @@ proc jsonFromScalar(content: string, tag: TagId): JsonNode
       raise newException(YamlConstructionError,
                          "Invalid float value: " & content)
   else: mappedType = yTypeUnknown
-  
+
   try:
     case mappedType
     of yTypeInteger:
@@ -66,9 +76,22 @@ proc jsonFromScalar(content: string, tag: TagId): JsonNode
     e.parent = getCurrentException()
     raise e
 
-proc constructJson*(s: var YamlStream): seq[JsonNode] =
+proc constructJson*(s: var YamlStream): seq[JsonNode]
+    {.raises: [YamlConstructionError, YamlStreamError].} =
+  ## Construct an in-memory JSON tree from a YAML event stream. The stream may
+  ## not contain any tags apart from those in ``coreTagLibrary``. Anchors and
+  ## aliases will be resolved. Maps in the input must not contain
+  ## non-scalars as keys. Each element of the result represents one document
+  ## in the YAML stream.
+  ##
+  ## **Warning:** The special float values ``[+-]Inf`` and ``NaN`` will be
+  ## parsed into Nim's JSON structure without error. However, they cannot be
+  ## rendered to a JSON character stream, because these values are not part
+  ## of the JSON specification. Nim's JSON implementation currently does not
+  ## check for these values and will output invalid JSON when rendering one
+  ## of these values into a JSON character stream.
   newSeq(result, 0)
-    
+
   var
     levels  = newSeq[Level]()
     anchors = initTable[AnchorId, JsonNode]()
@@ -95,7 +118,7 @@ proc constructJson*(s: var YamlStream): seq[JsonNode] =
         levels.add((node: jsonFromScalar(event.scalarContent,
                                          event.scalarTag), key: nil))
         continue
-  
+
       case levels[levels.high].node.kind
       of JArray:
         let jsonScalar = jsonFromScalar(event.scalarContent,
@@ -117,7 +140,8 @@ proc constructJson*(s: var YamlStream): seq[JsonNode] =
           levels[levels.high].key = nil
           if event.scalarAnchor != yAnchorNone:
             anchors[event.scalarAnchor] = jsonScalar
-      else: discard # will never happen
+      else:
+        internalError("Unexpected node kind: " & $levels[levels.high].node.kind)
     of yamlEndSeq, yamlEndMap:
       if levels.len > 1:
         let level = levels.pop()
@@ -130,56 +154,57 @@ proc constructJson*(s: var YamlStream): seq[JsonNode] =
           else:
             levels[levels.high].node[levels[levels.high].key] = level.node
             levels[levels.high].key = nil
-        else: discard # will never happen
+        else:
+          internalError("Unexpected node kind: " &
+                        $levels[levels.high].node.kind)
       else: discard # wait for yamlEndDocument
     of yamlAlias:
       # we can savely assume that the alias exists in anchors
       # (else the parser would have already thrown an exception)
       case levels[levels.high].node.kind
       of JArray:
-        try:
-          levels[levels.high].node.elems.add(anchors[event.aliasTarget])
-        except KeyError:
-          # we can safely assume that this doesn't happen. It would
-          # have resulted in a parser error earlier.
-          assert(false)
+        levels[levels.high].node.elems.add(
+            anchors.getOrDefault(event.aliasTarget))
       of JObject:
         if isNil(levels[levels.high].key):
           raise newException(YamlConstructionError,
               "cannot use alias node as key in JSON")
         else:
-          try:
-            levels[levels.high].node.fields.add(
-                levels[levels.high].key, anchors[event.aliasTarget])
-          except KeyError:
-            # we can safely assume that this doesn't happen. It would
-            # have resulted in a parser error earlier.
-            assert(false)
+          levels[levels.high].node.fields.add(
+              levels[levels.high].key, anchors.getOrDefault(event.aliasTarget))
           levels[levels.high].key = nil
-      else: discard # will never happen
+      else:
+        internalError("Unexpected node kind: " & $levels[levels.high].node.kind)
 
-proc loadToJson*(s: Stream): seq[JsonNode] =
+proc loadToJson*(s: Stream): seq[JsonNode]
+    {.raises: [YamlParserError, YamlConstructionError, IOError].} =
+  ## Uses `YamlParser <#YamlParser>`_ and
+  ## `constructJson <#constructJson>`_ to construct an in-memory JSON tree
+  ## from a YAML character stream.
   var
     parser = newYamlParser(initCoreTagLibrary())
     events = parser.parse(s)
   try:
     return constructJson(events)
-  except YamlConstructionError:
-    var e = cast[ref YamlConstructionError](getCurrentException())
-    e.line = parser.getLineNumber()
-    e.column = parser.getColNumber()
-    e.lineContent = parser.getLineContent()
-    raise e
   except YamlStreamError:
     let e = getCurrentException()
     if e.parent of IOError:
-      raise cast[ref IOError](e.parent)
+      raise (ref IOError)(e.parent)
     elif e.parent of YamlParserError:
-      raise cast[ref YamlParserError](e.parent)
-    else:
-      # can never happen
-      assert(false)
-  except AssertionError: raise
-  except Exception:
-    # compiler bug: https://github.com/nim-lang/Nim/issues/3772
-    assert(false)
+      raise (ref YamlParserError)(e.parent)
+    else: internalError("Unexpected exception: " & e.parent.repr)
+
+proc loadToJson*(str: string): seq[JsonNode]
+    {.raises: [YamlParserError, YamlConstructionError].} =
+  ## Uses `YamlParser <#YamlParser>`_ and
+  ## `constructJson <#constructJson>`_ to construct an in-memory JSON tree
+  ## from a YAML character stream.
+  var
+    parser = newYamlParser(initCoreTagLibrary())
+    events = parser.parse(str)
+  try: return constructJson(events)
+  except YamlStreamError:
+    let e = getCurrentException()
+    if e.parent of YamlParserError:
+      raise (ref YamlParserError)(e.parent)
+    else: internalError("Unexpected exception: " & e.parent.repr)