Got tags working

src/yaml/private/lexer.nim

@@ -514,6 +514,11 @@ iterator tokens*(my: var YamlLexer): YamlLexerToken {.closure.} =
                         yieldToken(yamlDash)
                     of ',':
                         yieldToken(yamlComma)
+                    of '!':
+                        my.content = "!"
+                        yieldToken(yamlTagHandle)
+                        my.content = ""
+                        yieldToken(yamlTagSuffix)
                     else:
                         yieldError("Unexpected special char: \"" &
                                    lastSpecialChar & "\"")

src/yaml/sequential.nim

@@ -33,6 +33,7 @@ type
         ylBlockAfterAnchor, ylBlockAfterAnchorAndTag, ylBlockAfterScalar,
         ylBlockAfterColon, ylBlockMultilineScalar, ylBlockLineEnd,
         ylBlockScalarHeader, ylBlockScalar, ylFlow, ylFlowAfterObject,
+        ylFlowAfterTag, ylFlowAfterAnchor, ylFlowAfterAnchorAndTag,
         ylExpectingDocumentEnd
     
     DocumentLevelMode = enum
@@ -154,6 +155,12 @@ template yieldStart(k: YamlParserEventKind) {.dirty.} =
     anchor = nil
     tag = nil
 
+template yieldDocumentEnd() {.dirty.} =
+    yield YamlParserEvent(kind: yamlEndDocument)
+    tagShorthands = initTable[string, string]()
+    tagShorthands["!"] = "!"
+    tagShorthands["!!"] = "tag:yaml.org,2002:"
+
 template closeLevel(lvl: DocumentLevel) {.dirty.} =
     case lvl.mode
     of mExplicitBlockMapKey, mFlowMapKey:
@@ -239,7 +246,6 @@ template handleTagHandle() {.dirty.} =
             yieldError("Missing tag suffix")
             continue
         tag = tagShorthands[handle] & lex.content
-        level.indentationColumn = lex.column
     else:
         yieldError("Unknown tag shorthand: " & handle)
 
@@ -273,6 +279,8 @@ iterator events*(parser: var YamlSequentialParser,
         scalarCacheIsQuoted: bool = false
         
     lex.open(input)
+    tagShorthands["!"] = "!"
+    tagShorthands["!!"] = "tag:yaml.org,2002:"
     
     var nextToken = tokens
     var token = nextToken(lex)
@@ -326,7 +334,7 @@ iterator events*(parser: var YamlSequentialParser,
                 state = ylBlockLineStart
             of yamlDocumentEnd, yamlStreamEnd:
                 yield YamlParserEvent(kind: yamlStartDocument)
-                yield YamlParserEvent(kind: yamlEndDocument)
+                yieldDocumentEnd()
             else:
                 yield YamlParserEvent(kind: yamlStartDocument)
                 state = ylBlockLineStart
@@ -358,6 +366,7 @@ iterator events*(parser: var YamlSequentialParser,
                 level.mode = mScalar
             of yamlTagHandle:
                 handleTagHandle()
+                level.indentationColumn = lex.column
                 state = ylBlockAfterTag
             of yamlVerbatimTag:
                 tag = lex.content
@@ -404,7 +413,7 @@ iterator events*(parser: var YamlSequentialParser,
                 break
             of yamlDocumentEnd:
                 closeAllLevels()
-                yield YamlParserEvent(kind: yamlEndDocument)
+                yieldDocumentEnd()
                 state = ylInitial
             of yamlOpeningBrace:
                 state = ylFlow
@@ -559,6 +568,12 @@ iterator events*(parser: var YamlSequentialParser,
                 state = ylBlockScalarHeader
                 scalarCache = ""
                 level.mode = mScalar
+            of yamlTagHandle:
+                handleTagHandle()
+                state = ylBlockAfterTag
+            of yamlAnchor:
+                anchor = lex.content
+                state = ylBlockAfterAnchor
             else:
                 yieldError("Unexpected token (expected scalar or line end): " &
                            $token)
@@ -733,8 +748,43 @@ iterator events*(parser: var YamlSequentialParser,
                             state = ylBlockLineEnd
                     else:
                         state = ylExpectingDocumentEnd
+            of yamlTagHandle:
+                handleTagHandle()
+                state = ylFlowAfterTag
+            of yamlAnchor:
+                anchor = lex.content
+                state = ylFlowAfterAnchor
             else:
                 yieldError("Unexpected token: " & $token)
+        of ylFlowAfterTag:
+            case token
+            of yamlTagHandle:
+                yieldError("Multiple tags on same node!")
+            of yamlAnchor:
+                anchor = lex.content
+                state = ylFlowAfterAnchorAndTag
+            else:
+                state = ylFlow
+                continue
+        of ylFlowAfterAnchor:
+            case token
+            of yamlAnchor:
+                yieldError("Multiple anchors on same node!")
+            of yamlTagHandle:
+                handleTagHandle()
+                state = ylFlowAfterAnchorAndTag
+            else:
+                state = ylFlow
+                continue
+        of ylFlowAfterAnchorAndTag:
+            case token
+            of yamlAnchor:
+                yieldError("Multiple anchors on same node!")
+            of yamlTagHandle:
+                yieldError("Multiple tags on same node!")
+            else:
+                state = ylFlow
+                continue
         of ylFlowAfterObject:
             case token
             of yamlLineStart:
@@ -798,10 +848,10 @@ iterator events*(parser: var YamlSequentialParser,
             of yamlComment, yamlLineStart:
                 discard
             of yamlStreamEnd, yamlDocumentEnd:
-                yield YamlParserEvent(kind: yamlEndDocument)
+                yieldDocumentEnd()
                 state = ylInitial
             of yamlDirectivesEnd:
-                yield YamlParserEvent(kind: yamlEndDocument)
+                yieldDocumentEnd()
                 state = ylInitial
                 continue
             else:

test/parsing.nim

@@ -19,7 +19,7 @@ proc scalar(content: string, tag: TagId = tagNonSpecificQmark,
     result.scalarTag = tag
     result.scalarContent = content
 
-proc startSequence(anchor: string = nil, tag: TagId = tagNonSpecificQmark):
+proc startSequence(tag: TagId = tagNonSpecificQmark, anchor: string = nil):
         YamlParserEvent =
     new(result)
     result.kind = yamlStartSequence
@@ -30,7 +30,7 @@ proc endSequence(): YamlParserEvent =
     new(result)
     result.kind = yamlEndSequence
 
-proc startMap(anchor: string = nil, tag: TagId = tagNonSpecificQmark):
+proc startMap(tag: TagId = tagNonSpecificQmark, anchor: string = nil):
         YamlParserEvent =
     new(result)
     result.kind = yamlStartMap
@@ -83,9 +83,7 @@ proc printDifference(expected, actual: YamlParserEvent) =
             echo "Unknown difference in event kind " & $expected.kind
 
 template ensure(input: string, expected: varargs[YamlParserEvent]) {.dirty.} =
-    var
-        i = 0
-        parser = initParser()
+    var i = 0
     
     for token in parser.events(newStringStream(input)):
         if i >= expected.len:
@@ -101,6 +99,9 @@ template ensure(input: string, expected: varargs[YamlParserEvent]) {.dirty.} =
         i.inc()
 
 suite "Parsing":
+    setup:
+        var parser = initParser()
+    
     test "Parsing: Simple Scalar":
         ensure("Scalar", startDoc(), scalar("Scalar"), endDoc())
     test "Parsing: Simple Sequence":
@@ -167,4 +168,43 @@ suite "Parsing":
         ensure("a: |-\x0A ab\x0A \x0A \x0A", startDoc(), startMap(),
                scalar("a"), scalar("ab"), endMap(), endDoc())
     test "Parsing: non-specific tags of quoted strings":
-        ensure("\"a\"", startDoc(), scalar("a", tagNonSpecificEmark), endDoc())
+        ensure("\"a\"", startDoc(), scalar("a", tagNonSpecificEmark), endDoc())
+    test "Parsing: explicit non-specific tag":
+        ensure("! a", startDoc(), scalar("a", tagNonSpecificEmark), endDoc())
+    test "Parsing: secondary tag handle resolution":
+        let id = parser.registerUri("tag:yaml.org,2002:str")
+        ensure("!!str a", startDoc(), scalar("a", id), endDoc())
+    test "Parsing: resolving custom tag handles":
+        let id = parser.registerUri("tag:example.com,2015:foo")
+        ensure("%TAG !t! tag:example.com,2015:\n---\n!t!foo a", startDoc(),
+               scalar("a", id), endDoc())
+    test "Parsing: tags in sequence":
+        let
+            idStr = parser.registerUri("tag:yaml.org,2002:str")
+            idInt = parser.registerUri("tag:yaml.org,2002:int")
+        ensure(" - !!str a\n - b\n - !!int c\n - d", startDoc(),
+               startSequence(), scalar("a", idStr), scalar("b"),
+               scalar("c", idInt), scalar("d"), endSequence(), endDoc())
+    test "Parsing: tags in implicit map":
+        let
+            idStr = parser.registerUri("tag:yaml.org,2002:str")
+            idInt = parser.registerUri("tag:yaml.org,2002:int")
+        ensure("!!str a: b\nc: !!int d\ne: !!str f\ng: h", startDoc(), startMap(),
+               scalar("a", idStr), scalar("b"), scalar("c"), scalar("d", idInt),
+               scalar("e"), scalar("f", idStr), scalar("g"), scalar("h"),
+               endMap(), endDoc())
+    test "Parsing: tags in explicit map":
+        let
+            idStr = parser.registerUri("tag:yaml.org,2002:str")
+            idInt = parser.registerUri("tag:yaml.org,2002:int")
+        ensure("? !!str a\n: !!int b\n? c\n: !!str d", startDoc(), startMap(),
+               scalar("a", idStr), scalar("b", idInt), scalar("c"),
+               scalar("d", idStr), endMap(), endDoc())
+    test "Parsing: tags for flow objects":
+        let
+            idStr = parser.registerUri("tag:yaml.org,2002:str")
+            idMap = parser.registerUri("tag:yaml.org,2002:map")
+            idSeq = parser.registerUri("tag:yaml.org,2002:seq")
+        ensure("!!map { k: !!seq [ a, !!str b] }", startDoc(), startMap(idMap),
+               scalar("k"), startSequence(idSeq), scalar("a"),
+               scalar("b", idStr), endSequence(), endMap(), endDoc())