From ae4c097a25b752a184f68f4fc839ab597cd3dc2f Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Wed, 4 Nov 2020 22:47:52 +0100 Subject: [PATCH] fixed parser test framework; started making parser tests green --- test/commonTestUtils.nim | 16 +++--- test/testEventParser.nim | 19 +++---- test/tlex.nim | 13 +++++ test/tparser.nim | 15 ++++-- yaml/data.nim | 6 +-- yaml/parser.nim | 106 +++++++++++++++++++++++---------------- yaml/private/lex.nim | 58 +++++++++++---------- yaml/serialization.nim | 2 +- 8 files changed, 139 insertions(+), 96 deletions(-) diff --git a/test/commonTestUtils.nim b/test/commonTestUtils.nim index ef8db4c..2fc6b46 100644 --- a/test/commonTestUtils.nim +++ b/test/commonTestUtils.nim @@ -15,22 +15,22 @@ proc escapeNewlines(s: string): string = of '\\': result.add("\\\\") else: result.add(c) -proc printDifference(expected, actual: Properties): bool = +proc printDifference(entity: string, expected, actual: Properties): bool = result = false if expected.tag != actual.tag: - echo "[scalar.tag] expected", $expected.tag, ", got", $actual.tag + echo "[", entity, ".tag] expected ", $expected.tag, ", got ", $actual.tag result = true if expected.anchor != actual.anchor: - echo "[scalar.anchor] expected", $expected.anchor, ", got", $actual.anchor + echo "[", entity, ".anchor] expected ", $expected.anchor, ", got ", $actual.anchor result = true proc printDifference*(expected, actual: Event) = if expected.kind != actual.kind: - echo "expected " & $expected.kind & ", got " & $actual.kind + echo "expected ", expected.kind, ", got ", $actual.kind else: case expected.kind of yamlScalar: - if not printDifference(expected.scalarProperties, actual.scalarProperties): + if not printDifference("scalar", expected.scalarProperties, actual.scalarProperties): if expected.scalarContent != actual.scalarContent: let msg = "[scalarEvent] content mismatch!\nexpected: " & escapeNewlines(expected.scalarContent) & @@ -46,10 +46,10 @@ proc printDifference*(expected, actual: Event) = break else: echo "[scalar] Unknown difference" of yamlStartMap: - if not printDifference(expected.mapProperties, actual.mapProperties): + if not printDifference("map", expected.mapProperties, actual.mapProperties): echo "[map] Unknown difference" of yamlStartSeq: - if not printDifference(expected.seqProperties, actual.seqProperties): + if not printDifference("seq", expected.seqProperties, actual.seqProperties): echo "[seq] Unknown difference" of yamlAlias: if expected.aliasTarget != actual.aliasTarget: @@ -67,7 +67,7 @@ template ensure*(input: var YamlStream, fail() break if token != expected[i]: - echo "at token #" & $i & ":" + echo "at event #" & $i & ":" printDifference(expected[i], token) fail() break diff --git a/test/testEventParser.nim b/test/testEventParser.nim index e74e0b5..240f0d3 100644 --- a/test/testEventParser.nim +++ b/test/testEventParser.nim @@ -101,10 +101,6 @@ proc nextToken(lex: var EventLexer): LexerToken = of "...": result = explDocEnd else: raise newException(EventStreamError, "Invalid token: " & lex.content) -template assertInStream() {.dirty.} = - if streamPos != inStream: - raise newException(EventStreamError, "Missing +STR") - template assertInEvent(name: string) {.dirty.} = if not inEvent: raise newException(EventStreamError, "Illegal token: " & name) @@ -168,8 +164,10 @@ template setCurAnchor(val: Anchor) = $curEvent.kind & " may not have an anchor") template eventStart(k: EventKind) {.dirty.} = - assertInStream() - yieldEvent() + if streamPos == beforeStream: + yield Event(kind: yamlStartStream) + streamPos = inStream + else: yieldEvent() curEvent = Event(kind: k) setTag(yTagQuestionMark) setAnchor(yAnchorNone) @@ -183,7 +181,6 @@ proc parseEventStream*(input: Stream, tagLib: TagLibrary): YamlStream = inEvent = false curEvent: Event streamPos: StreamPos = beforeStream - nextAnchorId = "a" while lex.buf[lex.bufpos] != EndOfFile: let token = lex.nextToken() case token @@ -195,8 +192,6 @@ proc parseEventStream*(input: Stream, tagLib: TagLibrary): YamlStream = of minusStr: if streamPos != inStream: raise newException(EventStreamError, "Illegal -STR") - if inEvent: yield curEvent - inEvent = false streamPos = afterStream eventStart(yamlEndStream) of plusDoc: eventStart(yamlStartDoc) @@ -220,8 +215,7 @@ proc parseEventStream*(input: Stream, tagLib: TagLibrary): YamlStream = if curAnchor() != yAnchorNone: raise newException(EventStreamError, "Duplicate anchor in " & $curEvent.kind) - setCurAnchor(nextAnchorId.Anchor) - nextAnchor(nextAnchorId, len(nextAnchorId)) + setCurAnchor(lex.content.Anchor) of starAnchor: assertInEvent("alias") if curEvent.kind != yamlAlias: @@ -255,4 +249,7 @@ proc parseEventStream*(input: Stream, tagLib: TagLibrary): YamlStream = raise newException(EventStreamError, "Unexpected explicit document end") of noToken: discard + yieldEvent() + if streamPos == inStream: + yield Event(kind: yamlEndStream) result = initYamlStream(backend) \ No newline at end of file diff --git a/test/tlex.nim b/test/tlex.nim index 0dda21b..6767293 100644 --- a/test/tlex.nim +++ b/test/tlex.nim @@ -186,6 +186,19 @@ suite "Lexer": ms(), an("b"), pl("b"), mv(), al("a"), sep(), al("b"), mv(), pl("c"), me(), e()) + test "Space at implicit key": + assertEquals("foo :\n bar", i(0), pl("foo"), mv(), i(2), pl("bar"), e()) + + test "inline anchor at implicit key": + assertEquals("top6: \l &anchor6 'key6' : scalar6", i(0), pl("top6"), mv(), + i(2), an("anchor6"), sq("key6"), mv(), pl("scalar6"), e()) + + test "Map in Sequence": + assertEquals("""- + a: b + c: d +""", i(0), si(), i(2), pl("a"), mv(), pl("b"), i(2), pl("c"), mv(), pl("d"), e()) + test "Empty lines": assertEquals("""block: foo diff --git a/test/tparser.nim b/test/tparser.nim index 501556a..1855f8b 100644 --- a/test/tparser.nim +++ b/test/tparser.nim @@ -36,9 +36,16 @@ proc parserTest(path: string, errorExpected : bool): bool = if expectedEvent != actualEvent: result = errorExpected if not result: + echoError("At event #" & $i & + ": Actual events do not match expected events") + echo ".. expected event:" + echo " ", expectedEvent + echo ".. actual event:" + echo " ", actualEvent + echo ".. difference:" + stdout.write(" ") printDifference(expectedEvent, actualEvent) - echoError("At token #" & $i & - ": Actual tokens do not match expected tokens") + return i.inc() if actualEvent.kind == yamlEndStream: @@ -49,14 +56,14 @@ proc parserTest(path: string, errorExpected : bool): bool = except: result = errorExpected if not result: + echoError("Caught an exception at event #" & $i & + " test was not successful") let e = getCurrentException() if e.parent of YamlParserError: let pe = (ref YamlParserError)(e.parent) echo "line ", pe.mark.line, ", column ", pe.mark.column, ": ", pe.msg echo pe.lineContent else: echo e.msg - echoError("Catched an exception at token #" & $i & - " test was not successful") macro genTests(): untyped = let diff --git a/yaml/data.nim b/yaml/data.nim index ab44ee8..ee5f7c6 100644 --- a/yaml/data.nim +++ b/yaml/data.nim @@ -156,11 +156,11 @@ proc collectionStyle*(event: Event): CollectionStyle = of yamlStartSeq: result = event.seqStyle else: raise (ref FieldDefect)(msg: "Event " & $event.kind & " has no collectionStyle") -proc startDocEvent*(explicit: bool = false, startPos, endPos: Mark = defaultMark): Event +proc startDocEvent*(explicit: bool = false, version: string = "", startPos, endPos: Mark = defaultMark): Event {.inline, raises: [].} = ## creates a new event that marks the start of a YAML document result = Event(startPos: startPos, endPos: endPos, - kind: yamlStartDoc, + kind: yamlStartDoc, version: version, explicitDirectivesEnd: explicit) proc endDocEvent*(explicit: bool = false, startPos, endPos: Mark = defaultMark): Event @@ -291,7 +291,7 @@ proc `$`*(event: Event): string {.raises: [].} = result = "-DOC" if event.explicitDocumentEnd: result &= " ..." of yamlStartMap: result = "+MAP" & renderAttrs(event.mapProperties) - of yamlStartSeq: result = "+SEQ" & renderAttrs(event.mapProperties) + of yamlStartSeq: result = "+SEQ" & renderAttrs(event.seqProperties) of yamlScalar: result = "=VAL" & renderAttrs(event.scalarProperties, event.scalarStyle == ssPlain or diff --git a/yaml/parser.nim b/yaml/parser.nim index b77e7b2..94fc194 100644 --- a/yaml/parser.nim +++ b/yaml/parser.nim @@ -34,7 +34,8 @@ type indentation: int Context = ref object of YamlStream - p: YamlParser + tagLib: TagLibrary + issueWarnings: bool lex: Lexer levels: seq[Level] @@ -119,32 +120,33 @@ proc afterFlowSeqItem(c: Context, e: var Event): bool proc afterPairValue(c: Context, e: var Event): bool {.pop.} -proc init[T](c: Context, source: T) {.inline.} = +proc init[T](c: Context, p: YamlParser, source: T) {.inline.} = c.levels.add(Level(state: atStreamStart, indentation: -2)) c.nextImpl = proc(s: YamlStream, e: var Event): bool = let c = Context(s) return c.levels[^1].state(c, e) c.headerProps = defaultProperties c.inlineProps = defaultProperties + c.tagLib = p.tagLib + c.issueWarnings = p.issueWarnings c.lex.init(source) # interface proc init*(p: var YamlParser, tagLib: TagLibrary = initExtendedTagLibrary(), issueWarnings: bool = false) = - ## Creates a YAML parser. if ``callback`` is not ``nil``, it will be called - ## whenever the parser yields a warning. + ## Creates a YAML parser. p.tagLib = tagLib p.issueWarnings = issueWarnings proc parse*(p: YamlParser, s: Stream): YamlStream = let c = new(Context) - c.init(s) + c.init(p, s) return c proc parse*(p: YamlParser, s: string): YamlStream = let c = new(Context) - c.init(s) + c.init(p, s) return c # implementation @@ -166,7 +168,7 @@ proc generateError(c: Context, message: string): proc parseTag(c: Context): TagId = let handle = c.lex.fullLexeme() - var uri = c.p.tagLib.resolve(handle) + var uri = c.tagLib.resolve(handle) if uri == "": raise c.generateError("unknown handle: " & escape(handle)) c.lex.next() @@ -174,9 +176,9 @@ proc parseTag(c: Context): TagId = raise c.generateError("unexpected token (expected tag suffix): " & $c.lex.cur) uri.add(c.lex.evaluated) try: - return c.p.tagLib.tags[uri] + return c.tagLib.tags[uri] except KeyError: - return c.p.tagLib.registerUri(uri) + return c.tagLib.registerUri(uri) proc toStyle(t: Token): ScalarStyle = return (case t @@ -187,10 +189,17 @@ proc toStyle(t: Token): ScalarStyle = of Folded: ssFolded else: ssAny) +proc autoScalarTag(props: Properties, t: Token): Properties = + result = props + if t in {Token.SingleQuoted, Token.DoubleQuoted} and + props.tag == yTagQuestionMark: + result.tag = yTagExclamationMark + proc atStreamStart(c: Context, e: var Event): bool = c.levels[0] = Level(state: atStreamEnd, indentation: -2) c.levels.add(Level(state: beforeDoc, indentation: -1)) e = Event(startPos: c.lex.curStartPos, endPos: c.lex.curStartPos, kind: yamlStartStream) + c.lex.next() return true proc atStreamEnd(c: Context, e : var Event): bool = @@ -208,6 +217,7 @@ proc beforeDoc(c: Context, e: var Event): bool = raise c.generateError("Missing `---` after directives") c.lex.next() of DirectivesEnd: + e = startDocEvent(true, version, c.lex.curStartPos, c.lex.curEndPos) c.lex.next() c.levels[1].state = beforeDocEnd c.levels.add(Level(state: afterDirectivesEnd, indentation: -1)) @@ -216,7 +226,7 @@ proc beforeDoc(c: Context, e: var Event): bool = discard c.levels.pop() return false of Indentation: - e = Event(kind: yamlStartDoc, explicitDirectivesEnd: false, version: version) + e = startDocEvent(false, version, c.lex.curStartPos, c.lex.curEndPos) c.levels[^1].state = beforeDocEnd c.levels.add(Level(state: beforeImplicitRoot, indentation: -1)) return true @@ -228,7 +238,7 @@ proc beforeDoc(c: Context, e: var Event): bool = elif version != "": raise c.generateError("Duplicate %YAML") version = c.lex.fullLexeme() - if version != "1.2" and c.p.issueWarnings: + if version != "1.2" and c.issueWarnings: discard # TODO c.lex.next() of TagDirective: @@ -240,7 +250,7 @@ proc beforeDoc(c: Context, e: var Event): bool = c.lex.next() if c.lex.cur != Token.Suffix: raise c.generateError("Invalid token (expected tag URI): " & $c.lex.cur) - c.p.tagLib.registerHandle(tagHandle, c.lex.fullLexeme()) + c.tagLib.registerHandle(tagHandle, c.lex.fullLexeme()) c.lex.next() of UnknownDirective: seenDirectives = true @@ -255,17 +265,21 @@ proc afterDirectivesEnd(c: Context, e: var Event): bool = case c.lex.cur of TagHandle, VerbatimTag, Token.Anchor: c.inlineStart = c.lex.curStartPos - c.levels.add(Level(state: beforeNodeProperties, indentation: 0)) + c.levels.add(Level(state: beforeNodeProperties)) + return false of Indentation: c.headerStart = c.inlineStart c.levels[^1].state = atBlockIndentation - c.levels.add(Level(state: beforeBlockIndentation, indentation: 0)) + c.levels.add(Level(state: beforeBlockIndentation)) + return false of DocumentEnd: e = scalarEvent("", c.inlineProps, ssPlain, c.lex.curStartPos, c.lex.curEndPos) + return true of Folded, Literal: e = scalarEvent(c.lex.evaluated, c.inlineProps, if c.lex.cur == Token.Folded: ssFolded else: ssLiteral, c.lex.curStartPos, c.lex.curEndPos) + return true else: raise c.generateError("Illegal content at `---`: " & $c.lex.cur) @@ -273,7 +287,7 @@ proc beforeImplicitRoot(c: Context, e: var Event): bool = if c.lex.cur != Token.Indentation: raise c.generateError("Unexpected token (expected line start): " & $c.lex.cur) c.inlineStart = c.lex.curEndPos - c.levels[^1].indentation = c.lex.currentIndentation() + c.levels[^1].indentation = c.lex.recentIndentation() c.lex.next() case c.lex.cur of SeqItemInd, MapKeyInd, MapValueInd: @@ -292,7 +306,7 @@ proc beforeImplicitRoot(c: Context, e: var Event): bool = raise c.generateError("Unexpected token (expected collection start): " & $c.lex.cur) proc requireImplicitMapStart(c: Context, e: var Event): bool = - c.levels[^1].indentation = c.lex.currentIndentation() + c.levels[^1].indentation = c.lex.recentIndentation() case c.lex.cur of Alias: e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) @@ -309,8 +323,8 @@ proc requireImplicitMapStart(c: Context, e: var Event): bool = discard c.levels.pop() return true of Plain, SingleQuoted, DoubleQuoted: - e = scalarEvent(c.lex.evaluated, c.inlineProps, toStyle(c.lex.cur), - c.inlineStart, c.lex.curEndPos) + e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur), + toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) c.inlineProps = defaultProperties let headerEnd = c.lex.curStartPos c.lex.next() @@ -346,34 +360,35 @@ proc atBlockIndentation(c: Context, e: var Event): bool = discard c.levels.pop() return true c.inlineStart = c.lex.curStartPos - c.levels[^1].indentation = c.lex.currentIndentation() + c.levels[^1].indentation = c.lex.recentIndentation() case c.lex.cur of nodePropertyKind: if isEmpty(c.headerProps): c.levels[^1].state = requireInlineBlockItem else: c.levels[^1].state = requireImplicitMapStart - c.levels.add(Level(state: beforeBlockIndentation, indentation: 0)) + c.levels.add(Level(state: beforeNodeProperties)) return false of SeqItemInd: e = startSeqEvent(csBlock, c.headerProps, c.headerStart, c.lex.curEndPos) c.headerProps = defaultProperties - c.levels[^1] = Level(state: inBlockSeq, indentation: c.lex.currentIndentation()) + c.levels[^1] = Level(state: inBlockSeq, indentation: c.lex.recentIndentation()) c.levels.add(Level(state: beforeBlockIndentation, indentation: 0)) - c.levels.add(Level(state: afterCompactParent, indentation: c.lex.currentIndentation())) + c.levels.add(Level(state: afterCompactParent, indentation: c.lex.recentIndentation())) c.lex.next() return true of MapKeyInd: e = startMapEvent(csBlock, c.headerProps, c.headerStart, c.lex.curEndPos) c.headerProps = defaultProperties - c.levels[^1] = Level(state: beforeBlockMapValue, indentation: 0) + c.levels[^1] = Level(state: beforeBlockMapValue, indentation: c.lex.recentIndentation()) c.levels.add(Level(state: beforeBlockIndentation)) - c.levels.add(Level(state: afterCompactParent, indentation: c.lex.currentIndentation())) + c.levels.add(Level(state: afterCompactParent, indentation: c.lex.recentIndentation())) c.lex.next() of Plain, SingleQuoted, DoubleQuoted: - c.levels[^1].indentation = c.lex.currentIndentation() + c.levels[^1].indentation = c.lex.recentIndentation() + let scalarToken = c.lex.cur e = scalarEvent(c.lex.evaluated, c.headerProps, toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) c.headerProps = defaultProperties @@ -383,11 +398,12 @@ proc atBlockIndentation(c: Context, e: var Event): bool = if c.lex.lastScalarWasMultiline(): raise c.generateError("Implicit mapping key may not be multiline") let props = e.scalarProperties - e.scalarProperties = defaultProperties + e.scalarProperties = autoScalarTag(defaultProperties, scalarToken) c.peek = move(e) e = startMapEvent(csBlock, props, c.headerStart, headerEnd) c.levels[^1].state = afterImplicitKey else: + e.scalarProperties = autoScalarTag(e.scalarProperties, scalarToken) discard c.levels.pop() return true of Alias: @@ -409,7 +425,7 @@ proc atBlockIndentation(c: Context, e: var Event): bool = c.levels[^1].state = atBlockIndentationProps proc atBlockIndentationProps(c: Context, e: var Event): bool = - c.levels[^1].indentation = c.lex.currentIndentation() + c.levels[^1].indentation = c.lex.recentIndentation() case c.lex.cur of MapValueInd: c.peek = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curEndPos) @@ -419,7 +435,8 @@ proc atBlockIndentationProps(c: Context, e: var Event): bool = c.levels[^1].state = afterImplicitKey return true of Plain, SingleQuoted, DoubleQuoted: - e = scalarEvent(c.lex.evaluated, c.inlineProps, toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) + e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur), + toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) c.inlineProps = defaultProperties let headerEnd = c.lex.curStartPos c.lex.next() @@ -458,9 +475,9 @@ proc beforeNodeProperties(c: Context, e: var Event): bool = if c.inlineProps.tag != yTagQuestionMark: raise c.generateError("Only one tag allowed per node") try: - c.inlineProps.tag = c.p.taglib.tags[c.lex.evaluated] + c.inlineProps.tag = c.tagLib.tags[c.lex.evaluated] except KeyError: - c.inlineProps.tag = c.p.taglib.registerUri(c.lex.evaluated) + c.inlineProps.tag = c.tagLib.registerUri(c.lex.evaluated) of Token.Anchor: if c.inlineProps.anchor != yAnchorNone: raise c.generateError("Only one anchor allowed per node") @@ -487,7 +504,8 @@ proc afterCompactParent(c: Context, e: var Event): bool = of SeqItemInd: e = startSeqEvent(csBlock, c.headerProps, c.headerStart, c.lex.curEndPos) c.headerProps = defaultProperties - c.levels[^1] = Level(state: inBlockSeq, indentation: c.lex.currentIndentation()) + c.levels[^1] = Level(state: inBlockSeq, indentation: c.lex.recentIndentation()) + echo "started seq at indentation ", c.lex.recentIndentation() c.levels.add(Level(state: beforeBlockIndentation)) c.levels.add(Level(state: afterCompactParent)) c.lex.next() @@ -495,7 +513,7 @@ proc afterCompactParent(c: Context, e: var Event): bool = of MapKeyInd: e = startMapEvent(csBlock, c.headerProps, c.headerStart, c.lex.curEndPos) c.headerProps = defaultProperties - c.levels[^1] = Level(state: beforeBlockMapValue, indentation: c.lex.currentIndentation()) + c.levels[^1] = Level(state: beforeBlockMapValue, indentation: c.lex.recentIndentation()) c.levels.add(Level(state: beforeBlockIndentation)) c.levels.add(Level(state: afterCompactParent)) return true @@ -504,7 +522,7 @@ proc afterCompactParent(c: Context, e: var Event): bool = return false proc afterCompactParentProps(c: Context, e: var Event): bool = - c.levels[^1].indentation = c.lex.currentIndentation() + c.levels[^1].indentation = c.lex.recentIndentation() case c.lex.cur of nodePropertyKind: c.levels.add(Level(state: beforeNodeProperties)) @@ -537,11 +555,11 @@ proc afterCompactParentProps(c: Context, e: var Event): bool = discard c.levels.pop() return true of scalarTokenKind: - e = scalarEvent(c.lex.evaluated, c.inlineProps, toStyle(c.lex.cur), - c.inlineStart, c.lex.curEndPos) + e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur), + toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) c.inlineProps = defaultProperties let headerEnd = c.lex.curStartPos - c.levels[^1].indentation = c.lex.currentIndentation() + c.levels[^1].indentation = c.lex.recentIndentation() c.lex.next() if c.lex.cur == Token.MapValueInd: if c.lex.lastScalarWasMultiline(): @@ -580,7 +598,7 @@ proc afterBlockParent(c: Context, e: var Event): bool = return false proc afterBlockParentProps(c: Context, e: var Event): bool = - c.levels[^1].indentation = c.lex.currentIndentation() + c.levels[^1].indentation = c.lex.recentIndentation() case c.lex.cur of nodePropertyKind: c.levels.add(Level(state: beforeNodeProperties)) @@ -588,7 +606,8 @@ proc afterBlockParentProps(c: Context, e: var Event): bool = of MapValueInd: raise c.generateError("Compact notation not allowed after implicit key") of scalarTokenKind: - e = scalarEvent(c.lex.evaluated, c.inlineProps, toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) + e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur), + toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) c.inlineProps = defaultProperties c.lex.next() if c.lex.cur == Token.MapValueInd: @@ -600,7 +619,7 @@ proc afterBlockParentProps(c: Context, e: var Event): bool = return false proc requireInlineBlockItem(c: Context, e: var Event): bool = - c.levels[^1].indentation = c.lex.currentIndentation() + c.levels[^1].indentation = c.lex.recentIndentation() case c.lex.cur of Indentation: raise c.generateError("Node properties may not stand alone on a line") @@ -677,7 +696,8 @@ proc atBlockMapKeyProps(c: Context, e: var Event): bool = of Alias: e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) of Plain, SingleQuoted, DoubleQuoted: - e = scalarEvent(c.lex.evaluated, c.inlineProps, toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) + e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur), + toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) c.inlineProps = defaultProperties if c.lex.lastScalarWasMultiline(): raise c.generateError("Implicit mapping key may not be multiline") @@ -698,7 +718,7 @@ proc afterImplicitKey(c: Context, e: var Event): bool = c.lex.next() c.levels[^1].state = beforeBlockMapKey c.levels.add(Level(state: beforeBlockIndentation)) - c.levels.add(Level(state: afterBlockParent, indentation: c.blockIndentation)) + c.levels.add(Level(state: afterBlockParent, indentation: c.levels[^2].indentation)) return false proc beforeBlockMapValue(c: Context, e: var Event): bool = @@ -781,7 +801,9 @@ proc beforeFlowItemProps(c: Context, e: var Event): bool = c.lex.next() discard c.levels.pop() of scalarTokenKind: - e = scalarEvent(c.lex.evaluated, c.inlineProps, toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) + e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur), + toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) + c.inlineProps = defaultProperties c.lex.next() discard c.levels.pop() of MapStart: diff --git a/yaml/private/lex.nim b/yaml/private/lex.nim index eb1178f..e5df90b 100644 --- a/yaml/private/lex.nim +++ b/yaml/private/lex.nim @@ -90,9 +90,12 @@ const UnknownIndentation* = int.low -proc currentIndentation*(lex: Lexer): Natural = +proc currentIndentation*(lex: Lexer): int = return lex.source.getColNumber(lex.source.bufpos) - 1 +proc recentIndentation*(lex: Lexer): int = + return lex.indentation + # lexer source handling proc advance(lex: var Lexer, step: int = 1) {.inline.} = @@ -332,35 +335,36 @@ proc readPlainScalar(lex: var Lexer) = case lex.c of ' ': lex.endToken() - let contentEnd = lex.source.bufpos - 2 + let spaceStart = lex.source.bufpos - 2 block spaceLoop: - lex.advance() - case lex.c - of '\l', '\c': - lex.evaluated.add(lex.source.buf[lineStartPos..contentEnd]) - break inlineLoop - of EndOfFile: - lex.evaluated.add(lex.source.buf[lineStartPos..contentEnd]) - lex.state = streamEnd - break multilineLoop - of '#': - lex.evaluated.add(lex.source.buf[lineStartPos..contentEnd]) - lex.state = expectLineEnd - break multilineLoop - of ':': - if not lex.isPlainSafe(): - lex.evaluated.add(lex.source.buf[lineStartPos..contentEnd]) - lex.state = insideLine + while true: + lex.advance() + case lex.c + of '\l', '\c': + lex.evaluated.add(lex.source.buf[lineStartPos..spaceStart]) + break inlineLoop + of EndOfFile: + lex.evaluated.add(lex.source.buf[lineStartPos..spaceStart]) + lex.state = streamEnd break multilineLoop - break spaceLoop - of flowIndicators: - if lex.flowDepth > 0: - lex.evaluated.add(lex.source.buf[lineStartPos..contentEnd]) - lex.state = insideLine + of '#': + lex.evaluated.add(lex.source.buf[lineStartPos..spaceStart]) + lex.state = expectLineEnd break multilineLoop - break spaceLoop - of ' ': discard - else: break spaceLoop + of ':': + if not lex.isPlainSafe(): + lex.evaluated.add(lex.source.buf[lineStartPos..spaceStart]) + lex.state = insideLine + break multilineLoop + break spaceLoop + of flowIndicators: + if lex.flowDepth > 0: + lex.evaluated.add(lex.source.buf[lineStartPos..spaceStart]) + lex.state = insideLine + break multilineLoop + break spaceLoop + of ' ': discard + else: break spaceLoop of ':': if not lex.isPlainSafe(): lex.evaluated.add(lex.source.buf[lineStartPos..lex.source.bufpos - 2]) diff --git a/yaml/serialization.nim b/yaml/serialization.nim index baf68ab..7d32bdd 100644 --- a/yaml/serialization.nim +++ b/yaml/serialization.nim @@ -136,7 +136,7 @@ template constructScalarItem*(s: var YamlStream, i: untyped, ## Helper template for implementing ``constructObject`` for types that ## are constructed from a scalar. ``i`` is the identifier that holds ## the scalar as ``Event`` in the content. Exceptions raised in - ## the content will be automatically catched and wrapped in + ## the content will be automatically caught and wrapped in ## ``YamlConstructionError``, which will then be raised. bind constructionError let i = s.next()