From f320449a2d24589efddb26de18eeaae9395b8ec8 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Mon, 21 Dec 2015 23:10:42 +0100 Subject: [PATCH] Got tags working --- src/yaml/private/lexer.nim | 5 ++++ src/yaml/sequential.nim | 60 ++++++++++++++++++++++++++++++++++---- test/parsing.nim | 52 +++++++++++++++++++++++++++++---- 3 files changed, 106 insertions(+), 11 deletions(-) diff --git a/src/yaml/private/lexer.nim b/src/yaml/private/lexer.nim index d478d4e..99fa1d2 100644 --- a/src/yaml/private/lexer.nim +++ b/src/yaml/private/lexer.nim @@ -514,6 +514,11 @@ iterator tokens*(my: var YamlLexer): YamlLexerToken {.closure.} = yieldToken(yamlDash) of ',': yieldToken(yamlComma) + of '!': + my.content = "!" + yieldToken(yamlTagHandle) + my.content = "" + yieldToken(yamlTagSuffix) else: yieldError("Unexpected special char: \"" & lastSpecialChar & "\"") diff --git a/src/yaml/sequential.nim b/src/yaml/sequential.nim index 0943b5c..d076651 100644 --- a/src/yaml/sequential.nim +++ b/src/yaml/sequential.nim @@ -33,6 +33,7 @@ type ylBlockAfterAnchor, ylBlockAfterAnchorAndTag, ylBlockAfterScalar, ylBlockAfterColon, ylBlockMultilineScalar, ylBlockLineEnd, ylBlockScalarHeader, ylBlockScalar, ylFlow, ylFlowAfterObject, + ylFlowAfterTag, ylFlowAfterAnchor, ylFlowAfterAnchorAndTag, ylExpectingDocumentEnd DocumentLevelMode = enum @@ -154,6 +155,12 @@ template yieldStart(k: YamlParserEventKind) {.dirty.} = anchor = nil tag = nil +template yieldDocumentEnd() {.dirty.} = + yield YamlParserEvent(kind: yamlEndDocument) + tagShorthands = initTable[string, string]() + tagShorthands["!"] = "!" + tagShorthands["!!"] = "tag:yaml.org,2002:" + template closeLevel(lvl: DocumentLevel) {.dirty.} = case lvl.mode of mExplicitBlockMapKey, mFlowMapKey: @@ -239,7 +246,6 @@ template handleTagHandle() {.dirty.} = yieldError("Missing tag suffix") continue tag = tagShorthands[handle] & lex.content - level.indentationColumn = lex.column else: yieldError("Unknown tag shorthand: " & handle) @@ -273,6 +279,8 @@ iterator events*(parser: var YamlSequentialParser, scalarCacheIsQuoted: bool = false lex.open(input) + tagShorthands["!"] = "!" + tagShorthands["!!"] = "tag:yaml.org,2002:" var nextToken = tokens var token = nextToken(lex) @@ -326,7 +334,7 @@ iterator events*(parser: var YamlSequentialParser, state = ylBlockLineStart of yamlDocumentEnd, yamlStreamEnd: yield YamlParserEvent(kind: yamlStartDocument) - yield YamlParserEvent(kind: yamlEndDocument) + yieldDocumentEnd() else: yield YamlParserEvent(kind: yamlStartDocument) state = ylBlockLineStart @@ -358,6 +366,7 @@ iterator events*(parser: var YamlSequentialParser, level.mode = mScalar of yamlTagHandle: handleTagHandle() + level.indentationColumn = lex.column state = ylBlockAfterTag of yamlVerbatimTag: tag = lex.content @@ -404,7 +413,7 @@ iterator events*(parser: var YamlSequentialParser, break of yamlDocumentEnd: closeAllLevels() - yield YamlParserEvent(kind: yamlEndDocument) + yieldDocumentEnd() state = ylInitial of yamlOpeningBrace: state = ylFlow @@ -559,6 +568,12 @@ iterator events*(parser: var YamlSequentialParser, state = ylBlockScalarHeader scalarCache = "" level.mode = mScalar + of yamlTagHandle: + handleTagHandle() + state = ylBlockAfterTag + of yamlAnchor: + anchor = lex.content + state = ylBlockAfterAnchor else: yieldError("Unexpected token (expected scalar or line end): " & $token) @@ -733,8 +748,43 @@ iterator events*(parser: var YamlSequentialParser, state = ylBlockLineEnd else: state = ylExpectingDocumentEnd + of yamlTagHandle: + handleTagHandle() + state = ylFlowAfterTag + of yamlAnchor: + anchor = lex.content + state = ylFlowAfterAnchor else: yieldError("Unexpected token: " & $token) + of ylFlowAfterTag: + case token + of yamlTagHandle: + yieldError("Multiple tags on same node!") + of yamlAnchor: + anchor = lex.content + state = ylFlowAfterAnchorAndTag + else: + state = ylFlow + continue + of ylFlowAfterAnchor: + case token + of yamlAnchor: + yieldError("Multiple anchors on same node!") + of yamlTagHandle: + handleTagHandle() + state = ylFlowAfterAnchorAndTag + else: + state = ylFlow + continue + of ylFlowAfterAnchorAndTag: + case token + of yamlAnchor: + yieldError("Multiple anchors on same node!") + of yamlTagHandle: + yieldError("Multiple tags on same node!") + else: + state = ylFlow + continue of ylFlowAfterObject: case token of yamlLineStart: @@ -798,10 +848,10 @@ iterator events*(parser: var YamlSequentialParser, of yamlComment, yamlLineStart: discard of yamlStreamEnd, yamlDocumentEnd: - yield YamlParserEvent(kind: yamlEndDocument) + yieldDocumentEnd() state = ylInitial of yamlDirectivesEnd: - yield YamlParserEvent(kind: yamlEndDocument) + yieldDocumentEnd() state = ylInitial continue else: diff --git a/test/parsing.nim b/test/parsing.nim index 3343c12..430f644 100644 --- a/test/parsing.nim +++ b/test/parsing.nim @@ -19,7 +19,7 @@ proc scalar(content: string, tag: TagId = tagNonSpecificQmark, result.scalarTag = tag result.scalarContent = content -proc startSequence(anchor: string = nil, tag: TagId = tagNonSpecificQmark): +proc startSequence(tag: TagId = tagNonSpecificQmark, anchor: string = nil): YamlParserEvent = new(result) result.kind = yamlStartSequence @@ -30,7 +30,7 @@ proc endSequence(): YamlParserEvent = new(result) result.kind = yamlEndSequence -proc startMap(anchor: string = nil, tag: TagId = tagNonSpecificQmark): +proc startMap(tag: TagId = tagNonSpecificQmark, anchor: string = nil): YamlParserEvent = new(result) result.kind = yamlStartMap @@ -83,9 +83,7 @@ proc printDifference(expected, actual: YamlParserEvent) = echo "Unknown difference in event kind " & $expected.kind template ensure(input: string, expected: varargs[YamlParserEvent]) {.dirty.} = - var - i = 0 - parser = initParser() + var i = 0 for token in parser.events(newStringStream(input)): if i >= expected.len: @@ -101,6 +99,9 @@ template ensure(input: string, expected: varargs[YamlParserEvent]) {.dirty.} = i.inc() suite "Parsing": + setup: + var parser = initParser() + test "Parsing: Simple Scalar": ensure("Scalar", startDoc(), scalar("Scalar"), endDoc()) test "Parsing: Simple Sequence": @@ -167,4 +168,43 @@ suite "Parsing": ensure("a: |-\x0A ab\x0A \x0A \x0A", startDoc(), startMap(), scalar("a"), scalar("ab"), endMap(), endDoc()) test "Parsing: non-specific tags of quoted strings": - ensure("\"a\"", startDoc(), scalar("a", tagNonSpecificEmark), endDoc()) \ No newline at end of file + ensure("\"a\"", startDoc(), scalar("a", tagNonSpecificEmark), endDoc()) + test "Parsing: explicit non-specific tag": + ensure("! a", startDoc(), scalar("a", tagNonSpecificEmark), endDoc()) + test "Parsing: secondary tag handle resolution": + let id = parser.registerUri("tag:yaml.org,2002:str") + ensure("!!str a", startDoc(), scalar("a", id), endDoc()) + test "Parsing: resolving custom tag handles": + let id = parser.registerUri("tag:example.com,2015:foo") + ensure("%TAG !t! tag:example.com,2015:\n---\n!t!foo a", startDoc(), + scalar("a", id), endDoc()) + test "Parsing: tags in sequence": + let + idStr = parser.registerUri("tag:yaml.org,2002:str") + idInt = parser.registerUri("tag:yaml.org,2002:int") + ensure(" - !!str a\n - b\n - !!int c\n - d", startDoc(), + startSequence(), scalar("a", idStr), scalar("b"), + scalar("c", idInt), scalar("d"), endSequence(), endDoc()) + test "Parsing: tags in implicit map": + let + idStr = parser.registerUri("tag:yaml.org,2002:str") + idInt = parser.registerUri("tag:yaml.org,2002:int") + ensure("!!str a: b\nc: !!int d\ne: !!str f\ng: h", startDoc(), startMap(), + scalar("a", idStr), scalar("b"), scalar("c"), scalar("d", idInt), + scalar("e"), scalar("f", idStr), scalar("g"), scalar("h"), + endMap(), endDoc()) + test "Parsing: tags in explicit map": + let + idStr = parser.registerUri("tag:yaml.org,2002:str") + idInt = parser.registerUri("tag:yaml.org,2002:int") + ensure("? !!str a\n: !!int b\n? c\n: !!str d", startDoc(), startMap(), + scalar("a", idStr), scalar("b", idInt), scalar("c"), + scalar("d", idStr), endMap(), endDoc()) + test "Parsing: tags for flow objects": + let + idStr = parser.registerUri("tag:yaml.org,2002:str") + idMap = parser.registerUri("tag:yaml.org,2002:map") + idSeq = parser.registerUri("tag:yaml.org,2002:seq") + ensure("!!map { k: !!seq [ a, !!str b] }", startDoc(), startMap(idMap), + scalar("k"), startSequence(idSeq), scalar("a"), + scalar("b", idStr), endSequence(), endMap(), endDoc()) \ No newline at end of file