all tests green again

doc/snippets/quickstart/08/01/00-code.nim

@@ -1,4 +1,4 @@
-import yaml, streams
+import yaml, yaml/data, streams
 type Person = object
   name: string
 
@@ -7,14 +7,15 @@ setTagUri(Person, nimTag("demo:Person"), yTagPerson)
 var
   s = newFileStream("in.yaml", fmRead)
   context = newConstructionContext()
-  parser = newYamlParser(serializationTagLibrary)
+  parser = initYamlParser(serializationTagLibrary)
   events = parser.parse(s)
 
+assert events.next().kind == yamlStartStream
 assert events.next().kind == yamlStartDoc
 assert events.next().kind == yamlStartSeq
 var nextEvent = events.peek()
 while nextEvent.kind != yamlEndSeq:
-  var curTag = nextEvent.tag()
+  var curTag = nextEvent.properties().tag
   if curTag == yTagQuestionMark:
     # we only support implicitly tagged scalars
     assert nextEvent.kind == yamlScalar
@@ -47,5 +48,5 @@ while nextEvent.kind != yamlEndSeq:
   nextEvent = events.peek()
 assert events.next().kind == yamlEndSeq
 assert events.next().kind == yamlEndDoc
-assert events.finished()
+assert events.next().kind == yamlEndStream
 s.close()

test/tdom.nim

@@ -18,7 +18,8 @@ suite "DOM":
   test "Serializing simple Scalar":
     let input = initYamlDoc(newYamlNode("scalar"))
     var result = serialize(input, initExtendedTagLibrary())
-    ensure(result, startDocEvent(), scalarEvent("scalar"), endDocEvent())
+    ensure(result, startStreamEvent(), startDocEvent(), scalarEvent("scalar"),
+        endDocEvent(), endStreamEvent())
   test "Composing sequence":
     let
       input = newStringStream("- !!str a\n- !!bool no")
@@ -37,9 +38,9 @@ suite "DOM":
         newYamlNode("a", "tag:yaml.org,2002:str"),
         newYamlNode("no", "tag:yaml.org,2002:bool")]))
     var result = serialize(input, initExtendedTagLibrary())
-    ensure(result, startDocEvent(), startSeqEvent(),
+    ensure(result, startStreamEvent(), startDocEvent(), startSeqEvent(),
            scalarEvent("a", yTagString), scalarEvent("no", yTagBoolean),
-           endSeqEvent(), endDocEvent())
+           endSeqEvent(), endDocEvent(), endStreamEvent())
   test "Composing mapping":
     let
       input = newStringStream("--- !!map\n!foo bar: [a, b]")
@@ -58,9 +59,9 @@ suite "DOM":
         (key: newYamlNode("bar"), value: newYamlNode([newYamlNode("a"),
                                                       newYamlNode("b")]))]))
     var result = serialize(input, initExtendedTagLibrary())
-    ensure(result, startDocEvent(), startMapEvent(), scalarEvent("bar"),
+    ensure(result, startStreamEvent(), startDocEvent(), startMapEvent(),
-           startSeqEvent(), scalarEvent("a"), scalarEvent("b"),
+        scalarEvent("bar"), startSeqEvent(), scalarEvent("a"), scalarEvent("b"),
-           endSeqEvent(), endMapEvent(), endDocEvent())
+        endSeqEvent(), endMapEvent(), endDocEvent(), endStreamEvent())
   test "Composing with anchors":
     let
       input = newStringStream("- &a foo\n- &b bar\n- *a\n- *b")
@@ -79,17 +80,18 @@ suite "DOM":
       b = newYamlNode("b")
       input = initYamlDoc(newYamlNode([a, b, newYamlNode("c"), a, b]))
     var result = serialize(input, initExtendedTagLibrary())
-    ensure(result, startDocEvent(), startSeqEvent(),
+    ensure(result, startStreamEvent(), startDocEvent(), startSeqEvent(),
            scalarEvent("a", anchor="a".Anchor),
            scalarEvent("b", anchor="b".Anchor), scalarEvent("c"),
            aliasEvent("a".Anchor), aliasEvent("b".Anchor), endSeqEvent(),
-           endDocEvent())
+           endDocEvent(), endStreamEvent())
   test "Serializing with all anchors":
     let
       a = newYamlNode("a")
       input = initYamlDoc(newYamlNode([a, newYamlNode("b"), a]))
     var result = serialize(input, initExtendedTagLibrary(), asAlways)
-    ensure(result, startDocEvent(), startSeqEvent(anchor="a".Anchor),
+    ensure(result, startStreamEvent(), startDocEvent(),
+           startSeqEvent(anchor="a".Anchor),
            scalarEvent("a", anchor = "b".Anchor),
            scalarEvent("b", anchor="c".Anchor), aliasEvent("b".Anchor),
-           endSeqEvent(), endDocEvent())
+           endSeqEvent(), endDocEvent(), endStreamEvent())

yaml/data.nim

@@ -1,5 +1,5 @@
 import hashes
-import private/internal
+import private/escaping
 
 type
   Anchor* = distinct string ## \

yaml/dom.nim

@@ -192,42 +192,61 @@ proc loadDom*(s: Stream | string): YamlDocument
     {.raises: [IOError, OSError, YamlParserError, YamlConstructionError].} =
   var
     tagLib = initExtendedTagLibrary()
-    parser: YamlParser
+    parser = initYamlParser(tagLib)
-  parser.init(tagLib)
+    events = parser.parse(s)
-  var events = parser.parse(s)
+    e: Event
-  try: result = compose(events, tagLib)
+  try:
+    e = events.next()
+    yAssert(e.kind == yamlStartStream)
+    result = compose(events, tagLib)
+    e = events.next()
+    if e.kind != yamlEndStream:
+      raise newYamlConstructionError(events, e.startPos, "stream contains multiple documents")
   except YamlStreamError:
-    let e = getCurrentException()
+    let ex = getCurrentException()
-    if e.parent of YamlParserError:
+    if ex.parent of YamlParserError:
-      raise (ref YamlParserError)(e.parent)
+      raise (ref YamlParserError)(ex.parent)
-    elif e.parent of IOError:
+    elif ex.parent of IOError:
-      raise (ref IOError)(e.parent)
+      raise (ref IOError)(ex.parent)
-    else: internalError("Unexpected exception: " & e.parent.repr)
+    else: internalError("Unexpected exception: " & ex.parent.repr)
+
+proc loadMultiDom*(s: Stream | string): seq[YamlDocument]
+    {.raises: [IOError, OSError, YamlParserError, YamlConstructionError].} =
+  var
+    tagLib = initExtendedTagLibrary()
+    parser = initYamlParser(tagLib)
+    events = parser.parse(s)
+    e: Event
+  try:
+    e = events.next()
+    yAssert(e.kind == yamlStartStream)
+    while events.peek().kind == yamlStartDoc:
+      result.add(compose(events, tagLib))
+    e = events.next()
+    yAssert(e.kind != yamlEndStream)
+  except YamlStreamError:
+    let ex = getCurrentException()
+    if ex.parent of YamlParserError:
+      raise (ref YamlParserError)(ex.parent)
+    elif ex.parent of IOError:
+      raise (ref IOError)(ex.parent)
+    else: internalError("Unexpected exception: " & ex.parent.repr)
 
 proc serializeNode(n: YamlNode, c: SerializationContext, a: AnchorStyle,
                    tagLib: TagLibrary) {.raises: [].}=
-  var val = yAnchorNone
+  var anchor = yAnchorNone
   let p = cast[pointer](n)
   if a != asNone and c.refs.hasKey(p):
-    val = c.refs.getOrDefault(p).a
+    anchor = c.refs.getOrDefault(p).a
-    if val == yAnchorNone:
+    c.refs[p] = (anchor, true)
-      val = c.nextAnchorId.Anchor
+    c.put(aliasEvent(anchor))
-      c.refs[p] = (val, false)
-      nextAnchor(c.nextAnchorId, len(c.nextAnchorId) - 1)
-    c.put(aliasEvent(val))
     return
-  var
-    anchor: Anchor
   if a != asNone:
-    val = c.nextAnchorId.Anchor
+    anchor = c.nextAnchorId.Anchor
     c.refs[p] = (c.nextAnchorId.Anchor, false)
     nextAnchor(c.nextAnchorId, len(c.nextAnchorId) - 1)
   let tag = if tagLib.tags.hasKey(n.tag): tagLib.tags.getOrDefault(n.tag) else:
           tagLib.registerUri(n.tag)
-  case a
-  of asNone: anchor = yAnchorNone
-  of asTidy: anchor = cast[Anchor](n)
-  of asAlways: anchor = val
 
   case n.kind
   of yScalar: c.put(scalarEvent(n.content, tag, anchor))
@@ -243,13 +262,6 @@ proc serializeNode(n: YamlNode, c: SerializationContext, a: AnchorStyle,
       serializeNode(value, c, a, tagLib)
     c.put(endMapEvent())
 
-proc processAnchoredEvent(target: var Properties, c: SerializationContext) =
-  for key, val in c.refs:
-    if val.a == target.anchor:
-      if not val.referenced:
-        target.anchor = yAnchorNone
-      break
-
 proc serialize*(doc: YamlDocument, tagLib: TagLibrary, a: AnchorStyle = asTidy):
     YamlStream {.raises: [].} =
   var
@@ -257,15 +269,20 @@ proc serialize*(doc: YamlDocument, tagLib: TagLibrary, a: AnchorStyle = asTidy):
     c = newSerializationContext(a, proc(e: Event) {.raises: [].} =
       bys.put(e)
     )
+  c.put(startStreamEvent())
   c.put(startDocEvent())
   serializeNode(doc.root, c, a, tagLib)
   c.put(endDocEvent())
+  c.put(endStreamEvent())
   if a == asTidy:
+    var ctx = initAnchorContext()
     for event in bys.mitems():
       case event.kind
-      of yamlScalar: processAnchoredEvent(event.scalarProperties, c)
+      of yamlScalar: ctx.process(event.scalarProperties, c.refs)
-      of yamlStartMap: processAnchoredEvent(event.mapProperties, c)
+      of yamlStartMap: ctx.process(event.mapProperties, c.refs)
-      of yamlStartSeq: processAnchoredEvent(event.seqProperties, c)
+      of yamlStartSeq: ctx.process(event.seqProperties, c.refs)
+      of yamlAlias:
+        event.aliasTarget = ctx.map(event.aliasTarget)
       else: discard
   result = bys
 

yaml/parser.nim

@@ -12,7 +12,7 @@
 ## non-nil string or Stream object as YAML character stream.
 
 import tables, strutils, macros, streams
-import taglib, stream, private/lex, private/internal, data
+import taglib, stream, private/lex, private/internal, private/escaping, data
 
 when defined(nimNoNil):
     {.experimental: "notnil".}

yaml/private/escaping.nim

@@ -0,0 +1,10 @@
+proc yamlTestSuiteEscape*(s: string): string =
+  result = ""
+  for c in s:
+    case c
+    of '\l': result.add("\\n")
+    of '\c': result.add("\\r")
+    of '\\': result.add("\\\\")
+    of '\b': result.add("\\b")
+    of '\t': result.add("\\t")
+    else: result.add(c)

yaml/private/internal.nim

@@ -4,6 +4,9 @@
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 
+import tables
+import ../data
+
 template internalError*(s: string) =
   # Note: to get the internal stacktrace that caused the error
   # compile with the `d:debug` flag.
@@ -41,17 +44,6 @@ template yAssert*(e: typed) =
       echo "[NimYAML] Please report this bug."
       quit 1
 
-proc yamlTestSuiteEscape*(s: string): string =
-  result = ""
-  for c in s:
-    case c
-    of '\l': result.add("\\n")
-    of '\c': result.add("\\r")
-    of '\\': result.add("\\\\")
-    of '\b': result.add("\\b")
-    of '\t': result.add("\\t")
-    else: result.add(c)
-
 proc nextAnchor*(s: var string, i: int) =
   if s[i] == 'z':
     s[i] = 'a'
@@ -74,4 +66,32 @@ proc registerHandle*(handles: var seq[tuple[handle, uriPrefix: string]], handle,
       handles[i].uriPrefix = uriPrefix
       return false
   handles.add((handle, uriPrefix))
-  return false
+  return false
+
+type
+  AnchorContext* = object
+    nextAnchorId: string
+    mapping: Table[Anchor, Anchor]
+
+proc initAnchorContext*(): AnchorContext =
+  return AnchorContext(nextAnchorId: "a", mapping: initTable[Anchor, Anchor]())
+
+proc process*(context: var AnchorContext,
+    target: var Properties, refs: Table[pointer, tuple[a: Anchor, referenced: bool]]) =
+  if target.anchor == yAnchorNone: return
+  for key, val in refs:
+    if val.a == target.anchor:
+      if not val.referenced:
+        target.anchor = yAnchorNone
+        return
+      break
+  if context.mapping.hasKey(target.anchor):
+    target.anchor = context.mapping.getOrDefault(target.anchor)
+  else:
+    let old = move(target.anchor)
+    target.anchor = context.nextAnchorId.Anchor
+    nextAnchor(context.nextAnchorId, len(context.nextAnchorId)-1)
+    context.mapping[old] = target.anchor
+
+proc map*(context: AnchorContext, anchor: Anchor): Anchor =
+  return context.mapping.getOrDefault(anchor)

yaml/serialization.nim

@@ -116,12 +116,15 @@ proc safeTagUri(tag: Tag): string {.raises: [].} =
   except KeyError:
     internalError("Unexpected KeyError for Tag " & $tag)
 
-proc constructionError(s: YamlStream, mark: Mark, msg: string): ref YamlConstructionError =
+proc newYamlConstructionError*(s: YamlStream, mark: Mark, msg: string): ref YamlConstructionError =
   result = newException(YamlConstructionError, msg)
   result.mark = mark
   if not s.getLastTokenContext(result.lineContent):
     result.lineContent = ""
 
+proc constructionError(s: YamlStream, mark: Mark, msg: string): ref YamlConstructionError =
+  return newYamlConstructionError(s, mark, msg)
+
 template constructScalarItem*(s: var YamlStream, i: untyped,
                               t: typedesc, content: untyped) =
   ## Helper template for implementing ``constructObject`` for types that
@@ -1351,7 +1354,7 @@ proc construct*[T](s: var YamlStream, target: var T)
     raise ex
 
 proc load*[K](input: Stream | string, target: var K)
-    {.raises: [YamlConstructionError, IOError, YamlParserError].} =
+    {.raises: [YamlConstructionError, IOError, OSError, YamlParserError].} =
   ## Loads a Nim value from a YAML character stream.
   var
     parser = initYamlParser(serializationTagLibrary)
@@ -1363,7 +1366,7 @@ proc load*[K](input: Stream | string, target: var K)
     e = events.next()
     if e.kind != yamlEndStream:
       var ex = (ref YamlConstructionError)(
-        mark: e.startPos, msg: "stream contains multiple document")
+        mark: e.startPos, msg: "stream contains multiple documents")
       discard events.getLastTokenContext(ex.lineContent)
       raise ex
   except YamlStreamError:
@@ -1399,14 +1402,6 @@ proc loadMultiDoc*[K](input: Stream | string, target: var seq[K]) =
     elif e.parent of YamlParserError: raise (ref YamlParserError)(e.parent)
     else: internalError("Unexpected exception: " & $e.parent.name)
 
-proc setAnchor(a: var Anchor, c: var SerializationContext)
-    {.inline.} =
-  if a != yAnchorNone:
-    for key, val in c.refs:
-      if val.a == a:
-        if not val.referenced: a = yAnchorNone
-        return
-
 proc represent*[T](value: T, ts: TagStyle = tsRootOnly,
                    a: AnchorStyle = asTidy): YamlStream =
   ## Represents a Nim value as ``YamlStream``
@@ -1420,11 +1415,13 @@ proc represent*[T](value: T, ts: TagStyle = tsRootOnly,
   bys.put(endDocEvent())
   bys.put(endStreamEvent())
   if a == asTidy:
+    var ctx = initAnchorContext()
     for item in bys.mitems():
       case item.kind
-      of yamlStartMap: setAnchor(item.mapProperties.anchor, context)
+      of yamlStartMap: ctx.process(item.mapProperties, context.refs)
-      of yamlStartSeq: setAnchor(item.seqProperties.anchor, context)
+      of yamlStartSeq: ctx.process(item.seqProperties, context.refs)
-      of yamlScalar: setAnchor(item.scalarProperties.anchor, context)
+      of yamlScalar: ctx.process(item.scalarProperties, context.refs)
+      of yamlAlias: item.aliasTarget = ctx.map(item.aliasTarget)
       else: discard
   result = bys