parser tests working, other tests compiling

test/tdom.nim

@@ -80,16 +80,16 @@ suite "DOM":
       input = initYamlDoc(newYamlNode([a, b, newYamlNode("c"), a, b]))
     var result = serialize(input, initExtendedTagLibrary())
     ensure(result, startDocEvent(), startSeqEvent(),
-           scalarEvent("a", anchor=0.AnchorId),
+           scalarEvent("a", anchor="a".Anchor),
-           scalarEvent("b", anchor=1.AnchorId), scalarEvent("c"),
+           scalarEvent("b", anchor="b".Anchor), scalarEvent("c"),
-           aliasEvent(0.AnchorId), aliasEvent(1.AnchorId), endSeqEvent(),
+           aliasEvent("a".Anchor), aliasEvent("b".Anchor), endSeqEvent(),
            endDocEvent())
   test "Serializing with all anchors":
     let
       a = newYamlNode("a")
       input = initYamlDoc(newYamlNode([a, newYamlNode("b"), a]))
     var result = serialize(input, initExtendedTagLibrary(), asAlways)
-    ensure(result, startDocEvent(), startSeqEvent(anchor=0.AnchorId),
+    ensure(result, startDocEvent(), startSeqEvent(anchor="a".Anchor),
-           scalarEvent("a", anchor=1.AnchorId),
+           scalarEvent("a", anchor = "b".Anchor),
-           scalarEvent("b", anchor=2.AnchorId), aliasEvent(1.AnchorId),
+           scalarEvent("b", anchor="c".Anchor), aliasEvent("b".Anchor),
            endSeqEvent(), endDocEvent())

test/tjson.nim

@@ -8,13 +8,10 @@ import "../yaml"
 
 import unittest, json
 
-proc wc(line, column: int, lineContent: string, message: string) =
-  echo "Warning (", line, ",", column, "): ", message, "\n", lineContent
-
 proc ensureEqual(yamlIn, jsonIn: string) =
   try:
     var
-      parser = newYamlParser(initCoreTagLibrary(), wc)
+      parser = initYamlParser(initCoreTagLibrary(), true)
       s = parser.parse(yamlIn)
       yamlResult = constructJson(s)
       jsonResult = parseJson(jsonIn)
@@ -24,7 +21,7 @@ proc ensureEqual(yamlIn, jsonIn: string) =
   except YamlStreamError:
     let e = (ref YamlParserError)(getCurrentException().parent)
     echo "error occurred: " & e.msg
-    echo "line: ", e.line, ", column: ", e.column
+    echo "line: ", e.mark.line, ", column: ", e.mark.column
     echo e.lineContent
     raise e
 

test/tlex.nim

@@ -161,7 +161,7 @@ suite "Lexer":
 
   test "Block Scalars":
     assertEquals("one : >2-\l   foo\l  bar\ltwo: |+\l bar\l  baz", i(0),
-        pl("one"), mv(), fs(" foo bar"), i(0), pl("two"), mv(),
+        pl("one"), mv(), fs(" foo\nbar"), i(0), pl("two"), mv(),
         ls("bar\l baz"), e())
 
   test "Flow indicators":

test/tparser.nim

@@ -75,14 +75,6 @@ macro genTests(): untyped =
   let errorTests = toHashSet(staticExec("cd " & (absolutePath / "tags" / "error") &
                          " && ls -1d *").splitLines())
   var ignored = toHashSet([".git", "name", "tags", "meta"])
-  #-----------------------------------------------------------------------------
-  # THE FOLLOWING TESTS WOULD FAIL FOR THE DOCUMENTED REASONS
-  ignored.incl("W5VH")
-    # YAML allows the colon as part of an anchor or alias name.
-    # For aliases, this leads to confusion becaues `*a:` looks like an implicit
-    # mapping key (but is not).
-    # Therefore, NimYAML disallows colons in anchor names.
-  #-----------------------------------------------------------------------------
 
   result = newStmtList()
   # walkDir for some crude reason does not work with travis build

test/tserialization.nim

@@ -119,8 +119,8 @@ template expectConstructionError(li, co: int, message: string, body: typed) =
     fail()
   except YamlConstructionError:
     let e = (ref YamlConstructionError)(getCurrentException())
-    doAssert li == e.line, "Expected error line " & $li & ", was " & $e.line
+    doAssert li == e.mark.line, "Expected error line " & $li & ", was " & $e.mark.line
-    doAssert co == e.column, "Expected error column " & $co & ", was " & $e.column
+    doAssert co == e.mark.column, "Expected error column " & $co & ", was " & $e.mark.column
     doAssert message == e.msg, "Expected error message \n" & escape(message) &
         ", got \n" & escape(e.msg)
 
@@ -307,18 +307,18 @@ suite "Serialization":
 
   test "Dump OrderedTable[tuple[int32, int32], string]":
     var input = initOrderedTable[tuple[a, b: int32], string]()
-    input.add((a: 23'i32, b: 42'i32), "dreiundzwanzigzweiundvierzig")
+    input[(a: 23'i32, b: 42'i32)] = "dreiundzwanzigzweiundvierzig"
-    input.add((a: 13'i32, b: 47'i32), "dreizehnsiebenundvierzig")
+    input[(a: 13'i32, b: 47'i32)] = "dreizehnsiebenundvierzig"
     var output = dump(input, tsRootOnly, asTidy, blockOnly)
     assertStringEqual(yamlDirs &
-        """!n!tables:OrderedTable(tag:nimyaml.org;2016:tuple(tag:nimyaml.org;2016:system:int32;tag:nimyaml.org;2016:system:int32);tag:yaml.org;2002:str) 
+        """!n!tables:OrderedTable(tag:nimyaml.org;2016:tuple(tag:nimyaml.org;2016:system:int32;tag:nimyaml.org;2016:system:int32);tag:yaml.org;2002:str)
-- 
+-
-  ? 
+  ?
     a: 23
     b: 42
   : dreiundzwanzigzweiundvierzig
-- 
+-
-  ? 
+  ?
     a: 13
     b: 47
   : dreizehnsiebenundvierzig""", output)
@@ -479,19 +479,19 @@ suite "Serialization":
     var output = dump(input, tsNone, asTidy, blockOnly)
     assertStringEqual yamlDirs & """
 
-- 
+-
-  - 
+  -
     name: Bastet
-  - 
+  -
     kind: akCat
-  - 
+  -
     purringIntensity: 7
-- 
+-
-  - 
+  -
     name: Anubis
-  - 
+  -
     kind: akDog
-  - 
+  -
     barkometer: 13""", output
 
   test "Load custom variant object - missing field":
@@ -545,17 +545,17 @@ suite "Serialization":
     let output = dump(input, tsNone, asTidy, blockOnly)
     assertStringEqual yamlDirs & """
 
-- 
+-
-  - 
+  -
     gStorable: gs
-  - 
+  -
     kind: deA
-  - 
+  -
     cStorable: cs
-- 
+-
-  - 
+  -
     gStorable: a
-  - 
+  -
     kind: deC""", output
 
   test "Load object with ignored key":
@@ -586,17 +586,17 @@ suite "Serialization":
       b.next = c
       c.next = a
       var output = dump(a, tsRootOnly, asTidy, blockOnly)
-      assertStringEqual yamlDirs & """!example.net:Node &a 
+      assertStringEqual yamlDirs & """!example.net:Node &a
 value: a
-next: 
+next:
   value: b
-  next: 
+  next:
     value: c
     next: *a""", output
 
     test "Load cyclic data structure":
-      let input = yamlDirs & """!n!system:seq(example.net:Node) 
+      let input = yamlDirs & """!n!system:seq(example.net:Node)
-  - &a 
+  - &a
     value: a
     next: &b
       value: b
@@ -610,7 +610,7 @@ next:
       try: load(input, result)
       except YamlConstructionError:
         let ex = (ref YamlConstructionError)(getCurrentException())
-        echo "line ", ex.line, ", column ", ex.column, ": ", ex.msg
+        echo "line ", ex.mark.line, ", column ", ex.mark.column, ": ", ex.msg
         echo ex.lineContent
         raise ex
 
@@ -651,7 +651,7 @@ next:
   test "Custom representObject":
     let input = @[1.BetterInt, 9998887.BetterInt, 98312.BetterInt]
     var output = dump(input, tsAll, asTidy, blockOnly)
-    assertStringEqual yamlDirs & """!n!system:seq(test:BetterInt) 
+    assertStringEqual yamlDirs & """!n!system:seq(test:BetterInt)
 - !test:BetterInt 1
 - !test:BetterInt 9_998_887
 - !test:BetterInt 98_312""", output

yaml/data.nim

@@ -28,7 +28,7 @@ type
     ssAny, ssPlain, ssSingleQuoted, ssDoubleQuoted, ssLiteral, ssFolded
 
   CollectionStyle* = enum
-    csBlock, csFlow
+    csAny, csBlock, csFlow, csPair
 
   EventKind* = enum
     ## Kinds of YAML events that may occur in an ``YamlStream``. Event kinds
@@ -156,6 +156,12 @@ proc collectionStyle*(event: Event): CollectionStyle =
   of yamlStartSeq: result = event.seqStyle
   else: raise (ref FieldDefect)(msg: "Event " & $event.kind & " has no collectionStyle")
 
+proc startStreamEvent*(): Event =
+  return Event(startPos: defaultMark, endPos: defaultMark, kind: yamlStartStream)
+
+proc endStreamEvent*(): Event =
+  return Event(startPos: defaultMark, endPos: defaultMark, kind: yamlEndStream)
+
 proc startDocEvent*(explicit: bool = false, version: string = "", startPos, endPos: Mark = defaultMark): Event
     {.inline, raises: [].} =
   ## creates a new event that marks the start of a YAML document
@@ -176,10 +182,10 @@ proc startMapEvent*(style: CollectionStyle, props: Properties,
                  kind: yamlStartMap, mapProperties: props,
                  mapStyle: style)
 
-proc startMapEvent*(style: CollectionStyle,
+proc startMapEvent*(style: CollectionStyle = csAny,
                     tag: TagId = yTagQuestionMark,
                     anchor: Anchor = yAnchorNone,
-                    startPos, endPos: Mark): Event {.inline.} =
+                    startPos, endPos: Mark = defaultMark): Event {.inline.} =
   return startMapEvent(style, (anchor, tag), startPos, endPos)
 
 proc endMapEvent*(startPos, endPos: Mark = defaultMark): Event {.inline, raises: [].} =
@@ -194,7 +200,7 @@ proc startSeqEvent*(style: CollectionStyle,
                  kind: yamlStartSeq, seqProperties: props,
                  seqStyle: style)
 
-proc startSeqEvent*(style: CollectionStyle,
+proc startSeqEvent*(style: CollectionStyle = csAny,
                     tag: TagId = yTagQuestionMark,
                     anchor: Anchor = yAnchorNone,
                     startPos, endPos: Mark = defaultMark): Event {.inline.} =
@@ -222,12 +228,12 @@ proc aliasEvent*(target: Anchor, startPos, endPos: Mark = defaultMark): Event {.
   ## creates a new event that represents a YAML alias
   result = Event(startPos: startPos, endPos: endPos, kind: yamlAlias, aliasTarget: target)
 
-proc `==`*(left, right: Anchor): bool {.borrow.}
+proc `==`*(left, right: Anchor): bool {.borrow, locks: 0.}
-proc `$`*(id: Anchor): string {.borrow.}
+proc `$`*(id: Anchor): string {.borrow, locks: 0.}
-proc hash*(id: Anchor): Hash {.borrow.}
+proc hash*(id: Anchor): Hash {.borrow, locks: 0.}
 
-proc `==`*(left, right: TagId): bool {.borrow.}
+proc `==`*(left, right: TagId): bool {.borrow, locks: 0.}
-proc hash*(id: TagId): Hash {.borrow.}
+proc hash*(id: TagId): Hash {.borrow, locks: 0.}
 
 proc `$`*(id: TagId): string {.raises: [].} =
   case id

yaml/dom.nim

@@ -195,7 +195,7 @@ proc compose*(s: var YamlStream, tagLib: TagLibrary): YamlDocument
   yAssert n.kind == yamlEndDoc
 
 proc loadDom*(s: Stream | string): YamlDocument
-    {.raises: [IOError, YamlParserError, YamlConstructionError].} =
+    {.raises: [IOError, OSError, YamlParserError, YamlConstructionError].} =
   var
     tagLib = initExtendedTagLibrary()
     parser: YamlParser

yaml/parser.nim

@@ -27,7 +27,7 @@ type
     tagLib: TagLibrary
     issueWarnings: bool
 
-  State = proc(c: Context, e: var Event): bool {.locks: 0, gcSafe.}
+  State = proc(c: Context, e: var Event): bool {.gcSafe.}
 
   Level = object
     state: State
@@ -38,6 +38,9 @@ type
     issueWarnings: bool
     lex: Lexer
     levels: seq[Level]
+    keyCache: seq[Event]
+    keyCachePos: int
+    caching: bool
 
     headerProps, inlineProps: Properties
     headerStart, inlineStart: Mark
@@ -87,7 +90,7 @@ const defaultProperties = (yAnchorNone, yTagQuestionMark)
 
 # parser states
 
-{.push gcSafe, locks: 0.}
+{.push gcSafe, .}
 proc atStreamStart(c: Context, e: var Event): bool
 proc atStreamEnd(c: Context, e : var Event): bool
 proc beforeDoc(c: Context, e: var Event): bool
@@ -97,14 +100,14 @@ proc beforeImplicitRoot(c: Context, e: var Event): bool
 proc atBlockIndentation(c: Context, e: var Event): bool
 proc beforeBlockIndentation(c: Context, e: var Event): bool
 proc beforeNodeProperties(c: Context, e: var Event): bool
-proc requireImplicitMapStart(c: Context, e: var Event): bool
 proc afterCompactParent(c: Context, e: var Event): bool
 proc afterCompactParentProps(c: Context, e: var Event): bool
-proc requireInlineBlockItem(c: Context, e: var Event): bool
+proc mergePropsOnNewline(c: Context, e: var Event): bool
 proc beforeFlowItemProps(c: Context, e: var Event): bool
 proc inBlockSeq(c: Context, e: var Event): bool
 proc beforeBlockMapValue(c: Context, e: var Event): bool
 proc atBlockIndentationProps(c: Context, e: var Event): bool
+proc beforeFlowItem(c: Context, e: var Event): bool
 proc afterFlowSeqSep(c: Context, e: var Event): bool
 proc afterFlowMapSep(c: Context, e: var Event): bool
 proc atBlockMapKeyProps(c: Context, e: var Event): bool
@@ -118,6 +121,7 @@ proc afterFlowMapValue(c: Context, e: var Event): bool
 proc afterFlowSeqSepProps(c: Context, e: var Event): bool
 proc afterFlowSeqItem(c: Context, e: var Event): bool
 proc afterPairValue(c: Context, e: var Event): bool
+proc emitCollectionKey(c: Context, e: var Event): bool
 {.pop.}
 
 template debug(message: string) {.dirty.} =
@@ -162,15 +166,23 @@ proc init[T](c: Context, p: YamlParser, source: T) {.inline.} =
   c.tagLib = p.tagLib
   c.issueWarnings = p.issueWarnings
   c.lex.init(source)
+  c.keyCachePos = 0
+  c.caching = false
 
 # interface
 
 proc init*(p: var YamlParser, tagLib: TagLibrary = initExtendedTagLibrary(),
             issueWarnings: bool = false) =
-  ## Creates a YAML parser.
+  ## Initializes a YAML parser.
   p.tagLib = tagLib
   p.issueWarnings = issueWarnings
 
+proc initYamlParser*(tagLib: TagLibrary = initExtendedTagLibrary(),
+                     issueWarnings: bool = false): YamlParser =
+  ## Creates an initializes YAML parser and returns it
+  result.tagLib = tagLib
+  result.issueWarnings = issueWarnings
+
 proc parse*(p: YamlParser, s: Stream): YamlStream =
   let c = new(Context)
   c.init(p, s)
@@ -188,7 +200,7 @@ proc isEmpty(props: Properties): bool =
            props.tag == yTagQuestionMark
 
 proc generateError(c: Context, message: string):
-    ref YamlParserError {.raises: [].} =
+    ref YamlParserError {.raises: [], .} =
   result = (ref YamlParserError)(
     msg: message, parent: nil, mark: c.lex.curStartPos,
     lineContent: c.lex.currentLine())
@@ -216,6 +228,18 @@ proc toStyle(t: Token): ScalarStyle =
     of Folded: ssFolded
     else: ssAny)
 
+proc mergeProps(c: Context, src, target: var Properties) =
+  if src.tag != yTagQuestionMark:
+    if target.tag != yTagQuestionMark:
+      raise c.generateError("Only one tag allowed per node")
+    target.tag = src.tag
+    src.tag = yTagQuestionMark
+  if src.anchor != yAnchorNone:
+    if target.anchor != yAnchorNone:
+      raise c.generateError("Only one anchor allowed per node")
+    target.anchor = src.anchor
+    src.anchor = yAnchorNone
+
 proc autoScalarTag(props: Properties, t: Token): Properties =
   result = props
   if t in {Token.SingleQuoted, Token.DoubleQuoted} and
@@ -278,7 +302,7 @@ proc beforeDoc(c: Context, e: var Event): bool =
       c.lex.next()
       if c.lex.cur != Token.Suffix:
         raise c.generateError("Invalid token (expected tag URI): " & $c.lex.cur)
-      discard c.tagLib.registerHandle(c.lex.fullLexeme(), tagHandle)
+      discard c.tagLib.registerHandle(c.lex.evaluated, tagHandle)
       c.lex.next()
     of UnknownDirective:
       seenDirectives = true
@@ -291,7 +315,7 @@ proc beforeDoc(c: Context, e: var Event): bool =
 
 proc afterDirectivesEnd(c: Context, e: var Event): bool =
   case c.lex.cur
-  of TagHandle, VerbatimTag, Token.Anchor:
+  of nodePropertyKind:
     c.inlineStart = c.lex.curStartPos
     c.pushLevel(beforeNodeProperties)
     return false
@@ -304,10 +328,9 @@ proc afterDirectivesEnd(c: Context, e: var Event): bool =
     e = scalarEvent("", c.inlineProps, ssPlain, c.lex.curStartPos, c.lex.curEndPos)
     c.popLevel()
     return true
-  of Folded, Literal:
+  of scalarTokenKind:
-    e = scalarEvent(c.lex.evaluated, c.inlineProps,
+    e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur),
-                    if c.lex.cur == Token.Folded: ssFolded else: ssLiteral,
+                    toStyle(c.lex.cur), c.lex.curStartPos, c.lex.curEndPos)
-                    c.lex.curStartPos, c.lex.curEndPos)
     c.popLevel()
     c.lex.next()
     return true
@@ -324,67 +347,15 @@ proc beforeImplicitRoot(c: Context, e: var Event): bool =
   of SeqItemInd, MapKeyInd, MapValueInd:
     c.transition(afterCompactParent)
     return false
-  of scalarTokenKind:
+  of scalarTokenKind, MapStart, SeqStart:
-    c.transition(requireImplicitMapStart)
+    c.transition(atBlockIndentationProps)
     return false
   of nodePropertyKind:
-    c.transition(requireImplicitMapStart)
+    c.transition(atBlockIndentationProps)
     c.pushLevel(beforeNodeProperties)
-  of MapStart, SeqStart:
-    c.transition(afterCompactParentProps)
-    return false
   else:
     raise c.generateError("Unexpected token (expected collection start): " & $c.lex.cur)
 
-proc requireImplicitMapStart(c: Context, e: var Event): bool =
-  c.updateIndentation(c.lex.recentIndentation())
-  case c.lex.cur
-  of Alias:
-    e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos)
-    let headerEnd = c.lex.curStartPos
-    c.lex.next()
-    if c.lex.cur == Token.MapValueInd:
-      c.peek = e
-      e = startMapEvent(csBlock, c.headerProps, c.headerStart, headerEnd)
-      c.headerProps = defaultProperties
-      c.transition(afterImplicitKey)
-    else:
-      if not isEmpty(c.headerProps):
-        raise c.generateError("Alias may not have properties")
-      c.popLevel()
-    return true
-  of Plain, SingleQuoted, DoubleQuoted:
-    e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur),
-                    toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos)
-    c.inlineProps = defaultProperties
-    let headerEnd = c.lex.curStartPos
-    c.lex.next()
-    case c.lex.cur
-    of Token.MapValueInd:
-      if c.lex.lastScalarWasMultiline():
-        raise c.generateError("Implicit mapping key may not be multiline")
-      c.peek = move(e)
-      e = startMapEvent(csBlock, c.headerProps,
-                        c.headerStart, headerEnd)
-      c.headerProps = defaultProperties
-      c.transition(afterImplicitKey)
-    else: c.popLevel()
-    return true
-  of Literal, Folded:
-    e = scalarEvent(c.lex.evaluated, c.inlineProps, toStyle(c.lex.cur),
-                    c.inlineStart, c.lex.curEndPos)
-    c.inlineProps = defaultProperties
-    c.lex.next()
-    c.popLevel()
-    return true
-  of MapStart, SeqStart:
-    c.transition(beforeFlowItemProps)
-    return false
-  of Indentation:
-    raise c.generateError("Standalone node properties not allowed on non-header line")
-  else:
-    raise c.generateError("Unexpected token (expected implicit mapping key): " & $c.lex.cur)
-
 proc atBlockIndentation(c: Context, e: var Event): bool =
   if c.blockIndentation == c.levels[^1].indentation and
       (c.lex.cur != Token.SeqItemInd or
@@ -400,9 +371,9 @@ proc atBlockIndentation(c: Context, e: var Event): bool =
   case c.lex.cur
   of nodePropertyKind:
     if isEmpty(c.headerProps):
-      c.transition(requireInlineBlockItem)
+      c.transition(mergePropsOnNewline)
     else:
-      c.transition(requireImplicitMapStart)
+      c.transition(atBlockIndentationProps)
     c.pushLevel(beforeNodeProperties)
     return false
   of SeqItemInd:
@@ -486,20 +457,41 @@ proc atBlockIndentationProps(c: Context, e: var Event): bool =
       c.headerProps = defaultProperties
       c.transition(afterImplicitKey)
     else:
+      c.mergeProps(c.headerProps, e.scalarProperties)
       c.popLevel()
     return true
-  of MapStart:
+  of MapStart, SeqStart:
-    e = startMapEvent(csFlow, c.headerProps, c.headerStart, c.lex.curEndPos)
+    let
+      startPos = c.lex.curStartPos
+      indent = c.levels[^1].indentation
+    c.transition(beforeFlowItemProps)
+    c.caching = true
+    while c.lex.flowDepth > 0:
+      c.keyCache.add(c.next())
+    c.keyCache.add(c.next())
+    c.caching = false
+    if c.lex.cur == Token.MapValueInd:
+      c.pushLevel(afterImplicitKey, indent)
+      c.pushLevel(emitCollectionKey)
+      if c.lex.curStartPos.line != startPos.line:
+        raise c.generateError("Implicit mapping key may not be multiline")
+      e = startMapEvent(csBlock, c.headerProps, c.headerStart, startPos)
+      c.headerProps = defaultProperties
+      return true
+    else:
+      c.pushLevel(emitCollectionKey)
+      return false
+  of Literal, Folded:
+    c.mergeProps(c.inlineProps, c.headerProps)
+    e = scalarEvent(c.lex.evaluated, c.headerProps, toStyle(c.lex.cur),
+                    c.inlineStart, c.lex.curEndPos)
     c.headerProps = defaultProperties
-    c.transition(afterFlowMapSep)
     c.lex.next()
+    c.popLevel()
     return true
-  of SeqStart:
+  of Indentation:
-    e = startSeqEvent(csFlow, c.headerProps, c.headerStart, c.lex.curEndPos)
-    c.headerProps = defaultProperties
-    c.transition(afterFlowSeqSep)
     c.lex.next()
-    return true
+    return false
   else:
     raise c.generateError("Unexpected token (expected block content): " & $c.lex.cur)
 
@@ -521,8 +513,7 @@ proc beforeNodeProperties(c: Context, e: var Event): bool =
       raise c.generateError("Only one anchor allowed per node")
     c.inlineProps.anchor = c.lex.shortLexeme().Anchor
   of Indentation:
-    c.headerProps = c.inlineProps
+    c.mergeProps(c.inlineProps, c.headerProps)
-    c.inlineProps = defaultProperties
     c.popLevel()
     return false
   of Alias:
@@ -656,19 +647,10 @@ proc afterBlockParentProps(c: Context, e: var Event): bool =
     c.transition(afterCompactParentProps)
     return false
 
-proc requireInlineBlockItem(c: Context, e: var Event): bool =
+proc mergePropsOnNewline(c: Context, e: var Event): bool =
   c.updateIndentation(c.lex.recentIndentation())
   if c.lex.cur == Token.Indentation:
-    if c.inlineProps.tag != yTagQuestionMark:
+    c.mergeProps(c.inlineProps, c.headerProps)
-      if c.headerProps.tag != yTagQuestionMark:
-        raise c.generateError("Only one tag allowed per node")
-      c.headerProps.tag = c.inlineProps.tag
-      c.inlineProps.tag = yTagQuestionMark
-    if c.inlineProps.anchor != yAnchorNone:
-      if c.headerProps.anchor != yAnchorNone:
-        raise c.generateError("Only one anchor allowed per node")
-      c.headerProps.anchor = c.inlineProps.anchor
-      c.inlineProps.anchor = yAnchorNone
   c.transition(afterCompactParentProps)
   return false
 
@@ -826,7 +808,6 @@ proc beforeBlockIndentation(c: Context, e: var Event): bool =
     raise c.generateError("Unexpected content after node in block context (expected newline): " & $c.lex.cur)
 
 proc beforeFlowItem(c: Context, e: var Event): bool =
-  debug("parse: beforeFlowItem")
   c.inlineStart = c.lex.curStartPos
   case c.lex.cur
   of nodePropertyKind:
@@ -931,7 +912,7 @@ proc afterFlowMapSep(c: Context, e: var Event): bool =
   c.pushLevel(beforeFlowItem)
   return false
 
-proc possibleNextSequenceItem(c: Context, e: var Event, endToken: Token, afterProps, afterItem: State): bool =
+proc afterFlowSeqSep(c: Context, e: var Event): bool =
   c.inlineStart = c.lex.curStartPos
   case c.lex.cur
   of SeqSep:
@@ -939,41 +920,42 @@ proc possibleNextSequenceItem(c: Context, e: var Event, endToken: Token, afterPr
     c.lex.next()
     return true
   of nodePropertyKind:
-    c.transition(afterProps)
+    c.transition(afterFlowSeqSepProps)
     c.pushLevel(beforeNodeProperties)
     return false
-  of Plain, SingleQuoted, DoubleQuoted:
+  of Plain, SingleQuoted, DoubleQuoted, MapStart, SeqStart:
-    c.transition(afterProps)
+    c.transition(afterFlowSeqSepProps)
     return false
   of MapKeyInd:
-    c.transition(afterItem)
+    c.transition(afterFlowSeqSepProps)
     e = startMapEvent(csFlow, defaultProperties, c.lex.curStartPos, c.lex.curEndPos)
     c.lex.next()
+    c.transition(afterFlowSeqItem)
     c.pushLevel(beforePairValue)
     c.pushLevel(beforeFlowItem)
     return true
   of MapValueInd:
-    c.transition(afterItem)
+    c.transition(afterFlowSeqItem)
     e = startMapEvent(csFlow, defaultProperties, c.lex.curStartPos, c.lex.curEndPos)
     c.pushLevel(atEmptyPairKey)
     return true
+  of SeqEnd:
+    e = endSeqEvent(c.lex.curStartPos, c.lex.curEndPos)
+    c.lex.next()
+    c.popLevel()
+    return true
   else:
-    if c.lex.cur == endToken:
+    c.transition(afterFlowSeqItem)
-      e = endSeqEvent(c.lex.curStartPos, c.lex.curEndPos)
+    c.pushLevel(beforeFlowItem)
-      c.lex.next()
+    return false
-      c.popLevel()
-      return true
-    else:
-      c.transition(afterItem)
-      c.pushLevel(beforeFlowItem)
-      return false
 
-proc afterFlowSeqSep(c: Context, e: var Event): bool =
+proc afterFlowSeqSepProps(c: Context, e: var Event): bool =
-  return possibleNextSequenceItem(c, e, Token.SeqEnd, afterFlowSeqSepProps, afterFlowSeqItem)
+  # here we handle potential implicit single pairs within flow sequences.
-
+  c.transition(afterFlowSeqItem)
-proc forcedNextSequenceItem(c: Context, e: var Event): bool =
+  case c.lex.cur
-  if c.lex.cur in {Token.Plain, Token.SingleQuoted, Token.DoubleQuoted}:
+  of Plain, SingleQuoted, DoubleQuoted:
-    e = scalarEvent(c.lex.evaluated, c.inlineProps, toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos)
+    e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur),
+                    toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos)
     c.inlineProps = defaultProperties
     c.lex.next()
     if c.lex.cur == Token.MapValueInd:
@@ -981,14 +963,40 @@ proc forcedNextSequenceItem(c: Context, e: var Event): bool =
       e = startMapEvent(csFlow, defaultProperties, c.lex.curStartPos, c.lex.curStartPos)
       c.pushLevel(afterImplicitPairStart)
     return true
+  of MapStart, SeqStart:
+    let
+      startPos = c.lex.curStartPos
+      indent = c.levels[^1].indentation
+      cacheStart = c.keyCache.len
+      targetFlowDepth = c.lex.flowDepth - 1
+      alreadyCaching = c.caching
+    c.pushLevel(beforeFlowItemProps)
+    c.caching = true
+    while c.lex.flowDepth > targetFlowDepth:
+      c.keyCache.add(c.next())
+    c.keyCache.add(c.next())
+    c.caching = alreadyCaching
+    if c.lex.cur == Token.MapValueInd:
+      c.pushLevel(afterImplicitPairStart, indent)
+      if c.lex.curStartPos.line != startPos.line:
+        raise c.generateError("Implicit mapping key may not be multiline")
+      if not alreadyCaching:
+        c.pushLevel(emitCollectionKey)
+        e = startMapEvent(csPair, defaultProperties, startPos, startPos)
+        return true
+      else:
+        # we are already filling a cache.
+        # so we just squeeze the map start in.
+        c.keyCache.insert(startMapEvent(csPair, defaultProperties, startPos, startPos), cacheStart)
+        return false
+    else:
+      if not alreadyCaching:
+        c.pushLevel(emitCollectionKey)
+      return false
   else:
     c.pushLevel(beforeFlowItem)
     return false
 
-proc afterFlowSeqSepProps(c: Context, e: var Event): bool =
-  c.transition(afterFlowSeqItem)
-  return forcedNextSequenceItem(c, e)
-
 proc atEmptyPairKey(c: Context, e: var Event): bool =
   c.transition(beforePairValue)
   e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curStartPos)
@@ -1017,6 +1025,17 @@ proc afterPairValue(c: Context, e: var Event): bool =
   c.popLevel()
   return true
 
+proc emitCollectionKey(c: Context, e: var Event): bool =
+  debug("emitCollection key: pos = " & $c.keyCachePos & ", len = " & $c.keyCache.len)
+  yAssert(c.keyCachePos < c.keyCache.len)
+  e = move(c.keyCache[c.keyCachePos])
+  inc(c.keyCachePos)
+  if c.keyCachePos == len(c.keyCache):
+    c.keyCache.setLen(0)
+    c.keyCachePos = 0
+    c.popLevel()
+  return true
+
 proc display*(p: YamlParser, event: Event): string =
   ## Generate a representation of the given event with proper visualization of
   ## anchor and tag (if any). The generated representation is conformant to the

yaml/presenter.nim

@@ -457,15 +457,15 @@ proc doPresent(s: var YamlStream, target: PresenterTarget,
           of ov1_2: target.append("%YAML 1.2" & newline)
           of ov1_1: target.append("%YAML 1.1" & newLine)
           of ovNone: discard
-          for prefix, handle in tagLib.handles():
+          for prefix, uri in tagLib.handles():
-            if handle == "!":
+            if prefix == "!":
-              if prefix != "!":
+              if uri != "!":
-                target.append("%TAG ! " & prefix & newline)
+                target.append("%TAG ! " & uri & newline)
-            elif handle == "!!":
+            elif prefix == "!!":
-              if prefix != yamlTagRepositoryPrefix:
+              if uri != yamlTagRepositoryPrefix:
-                target.append("%TAG !! " & prefix & newline)
+                target.append("%TAG !! " & uri & newline)
             else:
-              target.append("%TAG " & handle & ' ' & prefix & newline)
+              target.append("%TAG " & prefix & ' ' & uri & newline)
           target.append("--- ")
         except:
           var e = newException(YamlPresenterOutputError, "")

yaml/private/lex.nim

@@ -14,13 +14,13 @@ type
   Lexer* = object
     cur*: Token
     curStartPos*, curEndPos*: Mark
+    flowDepth*: int
     # recently read scalar or URI, if any
     evaluated*: string
     # internals
     indentation: int
     source: BaseLexer
     tokenStart: int
-    flowDepth: int
     state, lineStartState, jsonEnablingState: State
     c: char
     seenMultiline: bool
@@ -90,10 +90,10 @@ const
 
   UnknownIndentation* = int.low
 
-proc currentIndentation*(lex: Lexer): int =
+proc currentIndentation*(lex: Lexer): int {.locks: 0.} =
   return lex.source.getColNumber(lex.source.bufpos) - 1
 
-proc recentIndentation*(lex: Lexer): int =
+proc recentIndentation*(lex: Lexer): int {.locks: 0.} =
   return lex.indentation
 
 # lexer source handling
@@ -163,7 +163,7 @@ proc afterJsonEnablingToken(lex: var Lexer): bool {.raises: LexerError.}
 proc lineIndentation(lex: var Lexer): bool {.raises: [].}
 proc lineDirEnd(lex: var Lexer): bool {.raises: [].}
 proc lineDocEnd(lex: var Lexer): bool {.raises: [].}
-proc atSuffix(lex: var Lexer): bool {.raises: [].}
+proc atSuffix(lex: var Lexer): bool {.raises: [LexerError].}
 proc streamEnd(lex: var Lexer): bool {.raises: [].}
 {.pop.}
 
@@ -333,7 +333,7 @@ proc readPlainScalar(lex: var Lexer) =
         while true:
           lex.advance()
           case lex.c
-          of ' ':
+          of space:
             lex.endToken()
             let spaceStart = lex.source.bufpos - 2
             block spaceLoop:
@@ -363,7 +363,7 @@ proc readPlainScalar(lex: var Lexer) =
                     lex.state = insideLine
                     break multilineLoop
                   break spaceLoop
-                of ' ': discard
+                of space: discard
                 else: break spaceLoop
           of ':':
             if not lex.isPlainSafe():
@@ -412,7 +412,7 @@ proc readPlainScalar(lex: var Lexer) =
             break multilineLoop
           of lsNewline: lex.endLine()
           newlines += 1
-      while lex.c == ' ': lex.advance()
+      while lex.c in space: lex.advance()
       if (lex.c == ':' and not lex.isPlainSafe()) or
          lex.c == '#' or (lex.c in flowIndicators and
          lex.flowDepth > 0):
@@ -478,7 +478,9 @@ proc readBlockScalar(lex: var Lexer) =
 
   block body:
     # determining indentation and leading empty lines
-    var maxLeadingSpaces = 0
+    var
+      maxLeadingSpaces = 0
+      moreIndented = false
     while true:
       if indent == 0:
         while lex.c == ' ': lex.advance()
@@ -506,16 +508,18 @@ proc readBlockScalar(lex: var Lexer) =
           elif indent < maxLeadingSpaces:
             raise lex.generateError("Leading all-spaces line contains too many spaces")
         elif lex.currentIndentation() < indent: break body
+        if lex.cur == Token.Folded and lex.c in space:
+          moreIndented = true
         break
     for i in countup(0, separationLines - 1):
       lex.evaluated.add('\l')
+    separationLines = if moreIndented: 1 else: 0
 
     block content:
       while true:
         contentStart = lex.source.bufpos - 1
         while lex.c notin lineEnd: lex.advance()
         lex.evaluated.add(lex.source.buf[contentStart .. lex.source.bufpos - 2])
-        separationLines = 0
         if lex.c == EndOfFile:
           lex.state = streamEnd
           lex.streamEndAfterBlock()
@@ -524,7 +528,9 @@ proc readBlockScalar(lex: var Lexer) =
         lex.endToken()
         lex.endLine()
 
+        let oldMoreIndented = moreIndented
         # empty lines and indentation of next line
+        moreIndented = false
         while true:
           while lex.c == ' ' and lex.currentIndentation() < indent:
             lex.advance()
@@ -541,7 +547,11 @@ proc readBlockScalar(lex: var Lexer) =
             if lex.currentIndentation() < indent or
                 (indent == 0 and lex.dirEndFollows() or lex.docEndFollows()):
               break content
-            else: break
+            if lex.cur == Token.Folded and lex.c in space:
+              moreIndented = true
+              if not oldMoreIndented:
+                separationLines += 1
+            break
 
         # line folding
         if lex.cur == Token.Literal:
@@ -552,6 +562,7 @@ proc readBlockScalar(lex: var Lexer) =
         else:
           for i in countup(0, separationLines - 2):
             lex.evaluated.add('\l')
+        separationLines = if moreIndented: 1 else: 0
 
     let markerFollows = lex.currentIndentation() == 0 and
         (lex.dirEndFollows() or lex.docEndFollows())
@@ -718,16 +729,16 @@ proc basicInit(lex: var Lexer) =
 
 # interface
 
-proc lastScalarWasMultiline*(lex: Lexer): bool =
+proc lastScalarWasMultiline*(lex: Lexer): bool {.locks: 0.} =
   result = lex.seenMultiline
 
-proc shortLexeme*(lex: Lexer): string =
+proc shortLexeme*(lex: Lexer): string {.locks: 0.} =
   return lex.source.buf[lex.tokenStart..lex.source.bufpos-2]
 
-proc fullLexeme*(lex: Lexer): string =
+proc fullLexeme*(lex: Lexer): string {.locks: 0.} =
   return lex.source.buf[lex.tokenStart - 1..lex.source.bufpos-2]
 
-proc currentLine*(lex: Lexer): string =
+proc currentLine*(lex: Lexer): string {.locks: 0.} =
   return lex.source.getCurrentLine(false)
 
 proc next*(lex: var Lexer) =
@@ -900,6 +911,7 @@ proc flowLineStart(lex: var Lexer): bool =
     let lineStart = lex.source.bufpos
     while lex.c == ' ': lex.advance()
     indent = lex.source.bufpos - lineStart
+    while lex.c in space: lex.advance()
   if indent <= lex.indentation:
     raise lex.generateError("Too few indentation spaces (must surpass surrounding block level)")
   lex.state = insideLine
@@ -980,10 +992,8 @@ proc readAnchorName(lex: var Lexer) =
   lex.startToken()
   while true:
     lex.advance()
-    if lex.c notin tagShorthandChars + {'_'}: break
+    if lex.c in spaceOrLineEnd + flowIndicators: break
-  if lex.c notin spaceOrLineEnd + flowIndicators:
+  if lex.source.bufpos == lex.tokenStart + 1:
-    raise lex.generateError("Illegal character in anchor: " & escape("" & lex.c))
-  elif lex.source.bufpos == lex.tokenStart + 1:
     raise lex.generateError("Anchor name must not be empty")
   lex.state = afterToken
 
@@ -1052,7 +1062,7 @@ proc indentationSettingToken(lex: var Lexer): bool =
       lex.indentation = cachedIntentation
 
 proc afterToken(lex: var Lexer): bool =
-  while lex.c == ' ': lex.advance()
+  while lex.c in space: lex.advance()
   if lex.c in commentOrLineEnd:
     lex.endLine()
   else:
@@ -1115,8 +1125,20 @@ proc lineDocEnd(lex: var Lexer): bool =
 
 proc atSuffix(lex: var Lexer): bool =
   lex.startToken()
-  while lex.c in suffixChars: lex.advance()
+  lex.evaluated.setLen(0)
-  lex.evaluated = lex.fullLexeme()
+  var curStart = lex.tokenStart - 1
+  while true:
+    case lex.c
+    of suffixChars: lex.advance()
+    of '%':
+      if curStart <= lex.source.bufpos - 2:
+        lex.evaluated.add(lex.source.buf[curStart..lex.source.bufpos - 2])
+      lex.readHexSequence(2)
+      curStart = lex.source.bufpos
+      lex.advance()
+    else: break
+  if curStart <= lex.source.bufpos - 2:
+    lex.evaluated.add(lex.source.buf[curStart..lex.source.bufpos - 2])
   lex.endToken()
   lex.cur = Token.Suffix
   lex.state = afterToken

yaml/serialization.nim

@@ -141,11 +141,11 @@ template constructScalarItem*(s: var YamlStream, i: untyped,
   bind constructionError
   let i = s.next()
   if i.kind != yamlScalar:
-    raise s.constructionError(i.startPos, "Expected scalar")
+    raise constructionError(s, i.startPos, "Expected scalar")
   try: content
   except YamlConstructionError as e: raise e
   except Exception:
-    var e = s.constructionError(i.startPos,
+    var e = constructionError(s, i.startPos,
         "Cannot construct to " & name(t) & ": " & item.scalarContent &
         "; error: " & getCurrentExceptionMsg())
     e.parent = getCurrentException()
@@ -447,7 +447,7 @@ proc representObject*[T](value: seq[T]|set[T], ts: TagStyle,
     c: SerializationContext, tag: TagId) =
   ## represents a Nim seq as YAML sequence
   let childTagStyle = if ts == tsRootOnly: tsNone else: ts
-  c.put(startSeqEvent(csBlock, tag))
+  c.put(startSeqEvent(tag = tag))
   for item in value:
     representChild(item, childTagStyle, c)
   c.put(endSeqEvent())
@@ -478,7 +478,7 @@ proc representObject*[I, T](value: array[I, T], ts: TagStyle,
     c: SerializationContext, tag: TagId) =
   ## represents a Nim array as YAML sequence
   let childTagStyle = if ts == tsRootOnly: tsNone else: ts
-  c.put(startSeqEvent(tag))
+  c.put(startSeqEvent(tag = tag))
   for item in value:
     representChild(item, childTagStyle, c)
   c.put(endSeqEvent())
@@ -515,7 +515,7 @@ proc representObject*[K, V](value: Table[K, V], ts: TagStyle,
     c: SerializationContext, tag: TagId) =
   ## represents a Nim Table as YAML mapping
   let childTagStyle = if ts == tsRootOnly: tsNone else: ts
-  c.put(startMapEvent(tag))
+  c.put(startMapEvent(tag = tag))
   for key, value in value.pairs:
     representChild(key, childTagStyle, c)
     representChild(value, childTagStyle, c)
@@ -559,7 +559,7 @@ proc constructObject*[K, V](s: var YamlStream, c: ConstructionContext,
 proc representObject*[K, V](value: OrderedTable[K, V], ts: TagStyle,
     c: SerializationContext, tag: TagId) =
   let childTagStyle = if ts == tsRootOnly: tsNone else: ts
-  c.put(startSeqEvent(tag))
+  c.put(startSeqEvent(tag = tag))
   for key, value in value.pairs:
     c.put(startMapEvent())
     representChild(key, childTagStyle, c)
@@ -692,13 +692,13 @@ proc markAsFound(i: int, matched: NimNode): NimNode {.compileTime.} =
 
 proc ifNotTransient(o, field: NimNode,
     content: openarray[NimNode],
-    elseError: bool, s: NimNode, tName, fName: string = ""):
+    elseError: bool, s: NimNode, m: NimNode, tName, fName: string = ""):
     NimNode {.compileTime.} =
   var stmts = newStmtList(content)
   if elseError:
     result = quote do:
       when `o`.`field`.hasCustomPragma(transient):
-        raise constructionError(`s`, "While constructing " & `tName` &
+        raise constructionError(`s`, `m`, "While constructing " & `tName` &
             ": Field \"" & `fName` & "\" is transient and may not occur in input")
       else:
         `stmts`
@@ -804,12 +804,12 @@ macro constructFieldValue(t: typedesc, stream: untyped,
           var ifStmt = newIfStmt((cond: discTest, body: newStmtList(
               newCall("constructChild", stream, context, field))))
           ifStmt.add(newNimNode(nnkElse).add(newNimNode(nnkRaiseStmt).add(
-              newCall(bindSym("constructionError"), stream,
+              newCall(bindSym("constructionError"), stream, m,
               infix(newStrLitNode("Field " & $item & " not allowed for " &
               $child[0] & " == "), "&", prefix(discriminant, "$"))))))
           ob.add(ifNotTransient(o, item,
               [checkDuplicate(stream, tName, $item, fieldIndex, matched, m),
-              ifStmt, markAsFound(fieldIndex, matched)], true, stream, tName,
+              ifStmt, markAsFound(fieldIndex, matched)], true, stream, m, tName,
               $item))
           caseStmt.add(ob)
     else:
@@ -819,7 +819,7 @@ macro constructFieldValue(t: typedesc, stream: untyped,
       ob.add(ifNotTransient(o, child,
           [checkDuplicate(stream, tName, $child, fieldIndex, matched, m),
           newCall("constructChild", stream, context, field),
-          markAsFound(fieldIndex, matched)], true, stream, tName, $child))
+          markAsFound(fieldIndex, matched)], true, stream, m, tName, $child))
       caseStmt.add(ob)
     inc(fieldIndex)
   caseStmt.add(newNimNode(nnkElse).add(newNimNode(nnkWhenStmt).add(
@@ -942,9 +942,9 @@ macro genRepresentObject(t: typedesc, value, childTagStyle: typed) =
         fieldName = $child[0]
         fieldAccessor = newDotExpr(value, newIdentNode(fieldName))
       result.add(quote do:
-        c.put(startMapEvent(yTagQuestionMark, yAnchorNone))
+        c.put(startMapEvent())
-        c.put(scalarEvent(`fieldName`, if `childTagStyle` == tsNone:
+        c.put(scalarEvent(`fieldName`, tag = if `childTagStyle` == tsNone:
-            yTagQuestionMark else: yTagNimField, yAnchorNone))
+            yTagQuestionMark else: yTagNimField))
         representChild(`fieldAccessor`, `childTagStyle`, c)
         c.put(endMapEvent())
       )
@@ -973,9 +973,9 @@ macro genRepresentObject(t: typedesc, value, childTagStyle: typed) =
               itemAccessor = newDotExpr(value, newIdentNode(name))
             curStmtList.add(quote do:
               when not `itemAccessor`.hasCustomPragma(transient):
-                c.put(startMapEvent(yTagQuestionMark, yAnchorNone))
+                c.put(startMapEvent())
-                c.put(scalarEvent(`name`, if `childTagStyle` == tsNone:
+                c.put(scalarEvent(`name`, tag = if `childTagStyle` == tsNone:
-                    yTagQuestionMark else: yTagNimField, yAnchorNone))
+                    yTagQuestionMark else: yTagNimField))
                 representChild(`itemAccessor`, `childTagStyle`, c)
                 c.put(endMapEvent())
             )
@@ -990,7 +990,7 @@ macro genRepresentObject(t: typedesc, value, childTagStyle: typed) =
         childAccessor = newDotExpr(value, newIdentNode(name))
       result.add(quote do:
         when not `childAccessor`.hasCustomPragma(transient):
-          when bool(`isVO`): c.put(startMapEvent(yTagQuestionMark, yAnchorNone))
+          when bool(`isVO`): c.put(startMapEvent())
           c.put(scalarEvent(`name`, if `childTagStyle` == tsNone:
               yTagQuestionMark else: yTagNimField, yAnchorNone))
           representChild(`childAccessor`, `childTagStyle`, c)
@@ -1002,8 +1002,8 @@ proc representObject*[O: object](value: O, ts: TagStyle,
     c: SerializationContext, tag: TagId) =
   ## represents a Nim object or tuple as YAML mapping
   let childTagStyle = if ts == tsRootOnly: tsNone else: ts
-  when isVariantObject(getType(O)): c.put(startSeqEvent(csBlock, (yAnchorNone, tag)))
+  when isVariantObject(getType(O)): c.put(startSeqEvent(tag = tag))
-  else: c.put(startMapEvent(csBlock, (yAnchorNone, tag)))
+  else: c.put(startMapEvent(tag = tag))
   genRepresentObject(O, value, childTagStyle)
   when isVariantObject(getType(O)): c.put(endSeqEvent())
   else: c.put(endMapEvent())
@@ -1012,10 +1012,10 @@ proc representObject*[O: tuple](value: O, ts: TagStyle,
     c: SerializationContext, tag: TagId) =
   let childTagStyle = if ts == tsRootOnly: tsNone else: ts
   var fieldIndex = 0'i16
-  c.put(startMapEvent(tag, yAnchorNone))
+  c.put(startMapEvent(tag = tag))
   for name, fvalue in fieldPairs(value):
-    c.put(scalarEvent(name, if childTagStyle == tsNone:
+    c.put(scalarEvent(name, tag = if childTagStyle == tsNone:
-          yTagQuestionMark else: yTagNimField, yAnchorNone))
+          yTagQuestionMark else: yTagNimField))
     representChild(fvalue, childTagStyle, c)
     inc(fieldIndex)
   c.put(endMapEvent())
@@ -1041,7 +1041,7 @@ proc representObject*[O: enum](value: O, ts: TagStyle,
 
 proc yamlTag*[O](T: typedesc[ref O]): TagId {.inline, raises: [].} = yamlTag(O)
 
-macro constructImplicitVariantObject(s, c, r, possibleTagIds: untyped,
+macro constructImplicitVariantObject(s, m, c, r, possibleTagIds: untyped,
                                      t: typedesc) =
   let tDesc = getType(getType(t)[1])
   yAssert tDesc.kind == nnkObjectTy
@@ -1071,7 +1071,7 @@ macro constructImplicitVariantObject(s, c, r, possibleTagIds: untyped,
     branch.add(branchContent)
     result.add(branch)
   let raiseStmt = newNimNode(nnkRaiseStmt).add(
-      newCall(bindSym("constructionError"), s,
+      newCall(bindSym("constructionError"), s, m,
       infix(newStrLitNode("This value type does not map to any field in " &
                           getTypeImpl(t)[1].repr & ": "), "&",
             newCall("uri", newIdentNode("serializationTagLibrary"),
@@ -1114,7 +1114,7 @@ proc constructChild*[T](s: var YamlStream, c: ConstructionContext,
     var possibleTagIds = newSeq[TagId]()
     case item.kind
     of yamlScalar:
-      case item.scalarTag
+      case item.scalarProperties.tag
       of yTagQuestionMark:
         case guessType(item.scalarContent)
         of yTypeInteger:
@@ -1137,19 +1137,19 @@ proc constructChild*[T](s: var YamlStream, c: ConstructionContext,
       of yTagExclamationMark:
         possibleTagIds.add(yamlTag(string))
       else:
-        possibleTagIds.add(item.scalarTag)
+        possibleTagIds.add(item.scalarProperties.tag)
     of yamlStartMap:
-      if item.mapTag in [yTagQuestionMark, yTagExclamationMark]:
+      if item.mapProperties.tag in [yTagQuestionMark, yTagExclamationMark]:
         raise s.constructionError(item.startPos,
             "Complex value of implicit variant object type must have a tag.")
-      possibleTagIds.add(item.mapTag)
+      possibleTagIds.add(item.mapProperties.tag)
     of yamlStartSeq:
-      if item.seqTag in [yTagQuestionMark, yTagExclamationMark]:
+      if item.seqProperties.tag in [yTagQuestionMark, yTagExclamationMark]:
         raise s.constructionError(item.startPos,
             "Complex value of implicit variant object type must have a tag.")
-      possibleTagIds.add(item.seqTag)
+      possibleTagIds.add(item.seqProperties.tag)
     else: internalError("Unexpected item kind: " & $item.kind)
-    constructImplicitVariantObject(s, c, result, possibleTagIds, T)
+    constructImplicitVariantObject(s, item.startPos, c, result, possibleTagIds, T)
   else:
     case item.kind
     of yamlScalar:
@@ -1197,7 +1197,7 @@ proc constructChild*[T](s: var YamlStream, c: ConstructionContext,
   ## constructs an optional value. A value with a !!null tag will be loaded
   ## an empty value.
   let event = s.peek()
-  if event.kind == yamlScalar and event.scalarTag == yTagNull:
+  if event.kind == yamlScalar and event.scalarProperties.tag == yTagNull:
     result = none(T)
     discard s.next()
   else:
@@ -1250,9 +1250,9 @@ proc constructChild*[O](s: var YamlStream, c: ConstructionContext,
       anchor = yAnchorNone
 
   case e.kind
-  of yamlScalar: removeAnchor(e.scalarAnchor)
+  of yamlScalar: removeAnchor(e.scalarProperties.anchor)
-  of yamlStartMap: removeAnchor(e.mapAnchor)
+  of yamlStartMap: removeAnchor(e.mapProperties.anchor)
-  of yamlStartSeq: removeAnchor(e.seqAnchor)
+  of yamlStartSeq: removeAnchor(e.seqProperties.anchor)
   else: internalError("Unexpected event kind: " & $e.kind)
   s.peek = e
   try: constructChild(s, c, result[])
@@ -1297,17 +1297,17 @@ proc representChild*[O](value: ref O, ts: TagStyle, c: SerializationContext) =
       if c.refs.hasKey(p):
         val = c.refs.getOrDefault(p)
         if val == yAnchorNone:
-          val = c.nextAnchorId
+          val = c.nextAnchorId.Anchor
           c.refs[p] = val
-          nextAnchor(c, len(c.nextAnchorId) - 1)
+          nextAnchor(c.nextAnchorId, len(c.nextAnchorId) - 1)
         c.put(aliasEvent(val))
         return
     if c.style == asAlways:
-      val = c.nextAnchorId
+      val = c.nextAnchorId.Anchor
       when defined(JS):
         {.emit: [c, ".refs.set(", p, ", ", val, ");"].}
       else: c.refs[p] = val
-      nextAnchor(c, len(c.nextAnchorId) - 1)
+      nextAnchor(c.nextAnchorId, len(c.nextAnchorId) - 1)
     else: c.refs[p] = yAnchorNone
     let
       a = if c.style == asAlways: val else: cast[Anchor](p)
@@ -1317,15 +1317,15 @@ proc representChild*[O](value: ref O, ts: TagStyle, c: SerializationContext) =
       var ex = e
       case ex.kind
       of yamlStartMap:
-        ex.mapAnchor = a
+        ex.mapProperties.anchor = a
-        if ts == tsNone: ex.mapTag = yTagQuestionMark
+        if ts == tsNone: ex.mapProperties.tag = yTagQuestionMark
       of yamlStartSeq:
-        ex.seqAnchor = a
+        ex.seqProperties.anchor = a
-        if ts == tsNone: ex.seqTag = yTagQuestionMark
+        if ts == tsNone: ex.seqProperties.tag = yTagQuestionMark
       of yamlScalar:
-        ex.scalarAnchor = a
+        ex.scalarProperties.anchor = a
         if ts == tsNone and guessType(ex.scalarContent) != yTypeNull:
-          ex.scalarTag = yTagQuestionMark
+          ex.scalarProperties.tag = yTagQuestionMark
       else: discard
       c.put = origPut
       c.put(ex)
@@ -1400,14 +1400,16 @@ proc loadMultiDoc*[K](input: Stream | string, target: var seq[K]) =
   var parser: YamlParser
   parser.init(serializationTagLibrary)
   var events = parser.parse(input)
+  discard events.next() # stream start
   try:
-    while not events.finished():
+    while events.peek().kind == yamlStartDoc:
       var item: K
       construct(events, item)
       target.add(item)
+    discard events.next() # stream end
   except YamlConstructionError:
     var e = (ref YamlConstructionError)(getCurrentException())
-    discard events.getLastTokenContext(e.line, e.column, e.lineContent)
+    discard events.getLastTokenContext(e.lineContent)
     raise e
   except YamlStreamError:
     let e = (ref YamlStreamError)(getCurrentException())
@@ -1430,9 +1432,11 @@ proc represent*[T](value: T, ts: TagStyle = tsRootOnly,
   var context = newSerializationContext(a, proc(e: Event) =
         bys.put(e)
       )
+  bys.put(startStreamEvent())
   bys.put(startDocEvent())
   representChild(value, ts, context)
   bys.put(endDocEvent())
+  bys.put(endStreamEvent())
   if a == asTidy:
     for item in bys.mitems():
       case item.kind

yaml/stream.nim

@@ -31,7 +31,7 @@ type
     ## and is not required to check for it. The procs in this module will
     ## always yield a well-formed ``YamlStream`` and expect it to be
     ## well-formed if they take it as input parameter.
-    nextImpl*: proc(s: YamlStream, e: var Event): bool
+    nextImpl*: proc(s: YamlStream, e: var Event): bool {.gcSafe.}
     lastTokenContextImpl*:
         proc(s: YamlStream, lineContent: var string): bool {.raises: [].}
     peeked: bool
@@ -55,15 +55,15 @@ proc basicInit*(s: YamlStream, lastTokenContextImpl:
 
 when not defined(JS):
   type IteratorYamlStream = ref object of YamlStream
-    backend: iterator(): Event
+    backend: iterator(): Event {.gcSafe.}
 
-  proc initYamlStream*(backend: iterator(): Event): YamlStream
+  proc initYamlStream*(backend: iterator(): Event {.gcSafe.}): YamlStream
       {.raises: [].} =
     ## Creates a new ``YamlStream`` that uses the given iterator as backend.
     result = new(IteratorYamlStream)
     result.basicInit()
     IteratorYamlStream(result).backend = backend
-    result.nextImpl = proc(s: YamlStream, e: var Event): bool =
+    result.nextImpl = proc(s: YamlStream, e: var Event): bool {.gcSafe.} =
       e = IteratorYamlStream(s).backend()
       result = true
 
@@ -86,7 +86,7 @@ proc newBufferYamlStream*(): BufferYamlStream not nil =
 proc put*(bys: BufferYamlStream, e: Event) {.raises: [].} =
   bys.buf.add(e)
 
-proc next*(s: YamlStream): Event {.raises: [YamlStreamError].} =
+proc next*(s: YamlStream): Event {.raises: [YamlStreamError], gcSafe.} =
   ## Get the next item of the stream. Requires ``finished(s) == true``.
   ## If the backend yields an exception, that exception will be encapsulated
   ## into a ``YamlStreamError``, which will be raised.