diff --git a/test/testEventParser.nim b/test/testEventParser.nim index 240f0d3..7595b27 100644 --- a/test/testEventParser.nim +++ b/test/testEventParser.nim @@ -10,7 +10,8 @@ import lexbase, streams, tables, strutils type LexerToken = enum plusStr, minusStr, plusDoc, minusDoc, plusMap, minusMap, plusSeq, minusSeq, - eqVal, eqAli, chevTag, andAnchor, starAnchor, quotContent, colonContent, + eqVal, eqAli, chevTag, andAnchor, starAnchor, colonContent, sqContent, + dqContent, litContent, foContent, explDirEnd, explDocEnd, noToken StreamPos = enum @@ -31,7 +32,14 @@ proc nextToken(lex: var EventLexer): LexerToken = if lex.buf[lex.bufpos] == EndOfFile: return noToken case lex.buf[lex.bufpos] of ':', '"', '\'', '|', '>': - let t = if lex.buf[lex.bufpos] == ':': colonContent else: quotContent + let t = case lex.buf[lex.bufpos] + of ':': colonContent + of '"': dqContent + of '\'': sqContent + of '|': litContent + of '>': foContent + else: colonContent + lex.content = "" lex.bufpos.inc() while true: @@ -226,19 +234,30 @@ proc parseEventStream*(input: Stream, tagLib: TagLibrary): YamlStream = escape(lex.content)) else: curEvent.aliasTarget = lex.content.Anchor - of quotContent: - assertInEvent("scalar content") - if curTag() == yTagQuestionMark: setCurTag(yTagExclamationMark) - if curEvent.kind != yamlScalar: - raise newException(EventStreamError, - "scalar content in non-scalar tag") - curEvent.scalarContent = lex.content of colonContent: assertInEvent("scalar content") curEvent.scalarContent = lex.content if curEvent.kind != yamlScalar: raise newException(EventStreamError, "scalar content in non-scalar tag") + of sqContent: + assertInEvent("scalar content") + curEvent.scalarContent = lex.content + if curTag() == yTagQuestionMark: setCurTag(yTagExclamationMark) + curEvent.scalarStyle = ssSingleQuoted + of dqContent: + assertInEvent("scalar content") + curEvent.scalarContent = lex.content + if curTag() == yTagQuestionMark: setCurTag(yTagExclamationMark) + curEvent.scalarStyle = ssDoubleQuoted + of litContent: + assertInEvent("scalar content") + curEvent.scalarContent = lex.content + curEvent.scalarStyle = ssLiteral + of foContent: + assertInEvent("scalar content") + curEvent.scalarContent = lex.content + curEvent.scalarStyle = ssFolded of explDirEnd: assertInEvent("explicit directives end") if curEvent.kind != yamlStartDoc: diff --git a/test/tlex.nim b/test/tlex.nim index 6767293..c3958b1 100644 --- a/test/tlex.nim +++ b/test/tlex.nim @@ -193,12 +193,23 @@ suite "Lexer": assertEquals("top6: \l &anchor6 'key6' : scalar6", i(0), pl("top6"), mv(), i(2), an("anchor6"), sq("key6"), mv(), pl("scalar6"), e()) + test "adjacent anchors": + assertEquals("foo: &a\n &b bar", i(0), pl("foo"), mv(), an("a"), i(2), + an("b"), pl("bar"), e()) + + test "comment at empty key/value pair": + assertEquals(": # foo\nbar:", i(0), mv(), i(0), pl("bar"), mv(), e()) + test "Map in Sequence": assertEquals("""- a: b c: d """, i(0), si(), i(2), pl("a"), mv(), pl("b"), i(2), pl("c"), mv(), pl("d"), e()) + test "dir end after multiline scalar": + assertEquals("foo:\n bar\n baz\n---\nderp", i(0), pl("foo"), mv(), i(2), + pl("bar baz"), dirE(), i(0), pl("derp"), e()) + test "Empty lines": assertEquals("""block: foo diff --git a/test/tparser.nim b/test/tparser.nim index 1855f8b..0d1aba9 100644 --- a/test/tparser.nim +++ b/test/tparser.nim @@ -74,8 +74,15 @@ macro genTests(): untyped = let errorTests = toHashSet(staticExec("cd " & (absolutePath / "tags" / "error") & " && ls -1d *").splitLines()) - let ignored = toHashSet(["3MYT", "JDH8", "2EBW", "9KAX", "AB8U", "B63P", "FBC9", - "Q5MG", "S98Z", ".git", "name", "tags", "meta"]) + var ignored = toHashSet([".git", "name", "tags", "meta"]) + #----------------------------------------------------------------------------- + # THE FOLLOWING TESTS WOULD FAIL FOR THE DOCUMENTED REASONS + ignored.incl("W5VH") + # YAML allows the colon as part of an anchor or alias name. + # For aliases, this leads to confusion becaues `*a:` looks like an implicit + # mapping key (but is not). + # Therefore, NimYAML disallows colons in anchor names. + #----------------------------------------------------------------------------- result = newStmtList() # walkDir for some crude reason does not work with travis build diff --git a/yaml/parser.nim b/yaml/parser.nim index 94fc194..bb2dacd 100644 --- a/yaml/parser.nim +++ b/yaml/parser.nim @@ -120,11 +120,43 @@ proc afterFlowSeqItem(c: Context, e: var Event): bool proc afterPairValue(c: Context, e: var Event): bool {.pop.} +template debug(message: string) {.dirty.} = + when defined(yamlDebug): + try: styledWriteLine(stdout, fgBlue, message) + except ValueError, IOError: discard + +template pushLevel(c: Context, newState: State, newIndent: int) = + debug("parser: push " & newState.astToStr & ", indent = " & $newIndent) + c.levels.add(Level(state: newState, indentation: newIndent)) + +template pushLevel(c: Context, newState: State) = + debug("parser: push " & newState.astToStr) + c.levels.add(Level(state: newState)) + +template transition(c: Context, newState: State) = + debug("parser: transition " & newState.astToStr) + c.levels[^1].state = newState + +template transition(c: Context, newState: State, newIndent) = + debug("parser: transtion " & newState.astToStr & ", indent = " & $newIndent) + c.levels[^1] = Level(state: newState, indentation: newIndent) + +template updateIndentation(c: Context, newIndent: int) = + debug("parser: update indent = " & $newIndent) + c.levels[^1].indentation = newIndent + +template popLevel(c: Context) = + debug("parser: pop") + discard c.levels.pop() + proc init[T](c: Context, p: YamlParser, source: T) {.inline.} = - c.levels.add(Level(state: atStreamStart, indentation: -2)) + c.pushLevel(atStreamStart, -2) c.nextImpl = proc(s: YamlStream, e: var Event): bool = let c = Context(s) return c.levels[^1].state(c, e) + c.lastTokenContextImpl = proc(s: YamlStream, lineContent: var string): bool = + lineContent = Context(s).lex.currentLine() + return true c.headerProps = defaultProperties c.inlineProps = defaultProperties c.tagLib = p.tagLib @@ -151,11 +183,6 @@ proc parse*(p: YamlParser, s: string): YamlStream = # implementation -template debug(message: string) {.dirty.} = - when defined(yamlDebug): - try: styledWriteLine(stdout, fgBlue, message) - except IOError: discard - proc isEmpty(props: Properties): bool = result = props.anchor == yAnchorNone and props.tag == yTagQuestionMark @@ -196,10 +223,11 @@ proc autoScalarTag(props: Properties, t: Token): Properties = result.tag = yTagExclamationMark proc atStreamStart(c: Context, e: var Event): bool = - c.levels[0] = Level(state: atStreamEnd, indentation: -2) - c.levels.add(Level(state: beforeDoc, indentation: -1)) + c.transition(atStreamEnd) + c.pushLevel(beforeDoc, -1) e = Event(startPos: c.lex.curStartPos, endPos: c.lex.curStartPos, kind: yamlStartStream) c.lex.next() + c.tagLib.resetPrefixes() return true proc atStreamEnd(c: Context, e : var Event): bool = @@ -219,16 +247,16 @@ proc beforeDoc(c: Context, e: var Event): bool = of DirectivesEnd: e = startDocEvent(true, version, c.lex.curStartPos, c.lex.curEndPos) c.lex.next() - c.levels[1].state = beforeDocEnd - c.levels.add(Level(state: afterDirectivesEnd, indentation: -1)) + c.transition(beforeDocEnd) + c.pushLevel(afterDirectivesEnd, -1) return true of StreamEnd: - discard c.levels.pop() + c.popLevel() return false of Indentation: e = startDocEvent(false, version, c.lex.curStartPos, c.lex.curEndPos) - c.levels[^1].state = beforeDocEnd - c.levels.add(Level(state: beforeImplicitRoot, indentation: -1)) + c.transition(beforeDocEnd) + c.pushLevel(beforeImplicitRoot, -1) return true of YamlDirective: seenDirectives = true @@ -250,7 +278,7 @@ proc beforeDoc(c: Context, e: var Event): bool = c.lex.next() if c.lex.cur != Token.Suffix: raise c.generateError("Invalid token (expected tag URI): " & $c.lex.cur) - c.tagLib.registerHandle(tagHandle, c.lex.fullLexeme()) + discard c.tagLib.registerHandle(c.lex.fullLexeme(), tagHandle) c.lex.next() of UnknownDirective: seenDirectives = true @@ -265,20 +293,23 @@ proc afterDirectivesEnd(c: Context, e: var Event): bool = case c.lex.cur of TagHandle, VerbatimTag, Token.Anchor: c.inlineStart = c.lex.curStartPos - c.levels.add(Level(state: beforeNodeProperties)) + c.pushLevel(beforeNodeProperties) return false of Indentation: c.headerStart = c.inlineStart - c.levels[^1].state = atBlockIndentation - c.levels.add(Level(state: beforeBlockIndentation)) + c.transition(atBlockIndentation) + c.pushLevel(beforeBlockIndentation) return false of DocumentEnd: e = scalarEvent("", c.inlineProps, ssPlain, c.lex.curStartPos, c.lex.curEndPos) + c.popLevel() return true of Folded, Literal: e = scalarEvent(c.lex.evaluated, c.inlineProps, if c.lex.cur == Token.Folded: ssFolded else: ssLiteral, c.lex.curStartPos, c.lex.curEndPos) + c.popLevel() + c.lex.next() return true else: raise c.generateError("Illegal content at `---`: " & $c.lex.cur) @@ -287,26 +318,26 @@ proc beforeImplicitRoot(c: Context, e: var Event): bool = if c.lex.cur != Token.Indentation: raise c.generateError("Unexpected token (expected line start): " & $c.lex.cur) c.inlineStart = c.lex.curEndPos - c.levels[^1].indentation = c.lex.recentIndentation() + c.updateIndentation(c.lex.recentIndentation()) c.lex.next() case c.lex.cur of SeqItemInd, MapKeyInd, MapValueInd: - c.levels[^1].state = afterCompactParent + c.transition(afterCompactParent) return false of scalarTokenKind: - c.levels[^1].state = requireImplicitMapStart + c.transition(requireImplicitMapStart) return false of nodePropertyKind: - c.levels[^1].state = requireImplicitMapStart - c.levels.add(Level(state: beforeNodeProperties, indentation: 0)) + c.transition(requireImplicitMapStart) + c.pushLevel(beforeNodeProperties) of MapStart, SeqStart: - c.levels[^1].state = afterCompactParentProps + c.transition(afterCompactParentProps) return false else: raise c.generateError("Unexpected token (expected collection start): " & $c.lex.cur) proc requireImplicitMapStart(c: Context, e: var Event): bool = - c.levels[^1].indentation = c.lex.recentIndentation() + c.updateIndentation(c.lex.recentIndentation()) case c.lex.cur of Alias: e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) @@ -316,11 +347,11 @@ proc requireImplicitMapStart(c: Context, e: var Event): bool = c.peek = e e = startMapEvent(csBlock, c.headerProps, c.headerStart, headerEnd) c.headerProps = defaultProperties - c.levels[^1].state = afterImplicitKey + c.transition(afterImplicitKey) else: if not isEmpty(c.headerProps): raise c.generateError("Alias may not have properties") - discard c.levels.pop() + c.popLevel() return true of Plain, SingleQuoted, DoubleQuoted: e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur), @@ -336,13 +367,18 @@ proc requireImplicitMapStart(c: Context, e: var Event): bool = e = startMapEvent(csBlock, c.headerProps, c.headerStart, headerEnd) c.headerProps = defaultProperties - c.levels[^1].state = afterImplicitKey - of Indentation, DocumentEnd, DirectivesEnd, StreamEnd: - raise c.generateError("Scalar at root level requires `---`") - else: discard + c.transition(afterImplicitKey) + else: c.popLevel() + return true + of Literal, Folded: + e = scalarEvent(c.lex.evaluated, c.inlineProps, toStyle(c.lex.cur), + c.inlineStart, c.lex.curEndPos) + c.inlineProps = defaultProperties + c.lex.next() + c.popLevel() return true of MapStart, SeqStart: - c.levels[^1].state = beforeFlowItemProps + c.transition(beforeFlowItemProps) return false of Indentation: raise c.generateError("Standalone node properties not allowed on non-header line") @@ -353,41 +389,42 @@ proc atBlockIndentation(c: Context, e: var Event): bool = if c.blockIndentation == c.levels[^1].indentation and (c.lex.cur != Token.SeqItemInd or c.levels[^3].state == inBlockSeq): - e = scalarEvent(c.lex.evaluated, c.headerProps, ssPlain, + e = scalarEvent("", c.headerProps, ssPlain, c.headerStart, c.headerStart) c.headerProps = defaultProperties - discard c.levels.pop() - discard c.levels.pop() + c.popLevel() + c.popLevel() return true c.inlineStart = c.lex.curStartPos - c.levels[^1].indentation = c.lex.recentIndentation() + c.updateIndentation(c.lex.recentIndentation()) case c.lex.cur of nodePropertyKind: if isEmpty(c.headerProps): - c.levels[^1].state = requireInlineBlockItem + c.transition(requireInlineBlockItem) else: - c.levels[^1].state = requireImplicitMapStart - c.levels.add(Level(state: beforeNodeProperties)) + c.transition(requireImplicitMapStart) + c.pushLevel(beforeNodeProperties) return false of SeqItemInd: e = startSeqEvent(csBlock, c.headerProps, c.headerStart, c.lex.curEndPos) c.headerProps = defaultProperties - c.levels[^1] = Level(state: inBlockSeq, indentation: c.lex.recentIndentation()) - c.levels.add(Level(state: beforeBlockIndentation, indentation: 0)) - c.levels.add(Level(state: afterCompactParent, indentation: c.lex.recentIndentation())) + c.transition(inBlockSeq, c.lex.recentIndentation()) + c.pushLevel(beforeBlockIndentation) + c.pushLevel(afterCompactParent, c.lex.recentIndentation()) c.lex.next() return true of MapKeyInd: e = startMapEvent(csBlock, c.headerProps, c.headerStart, c.lex.curEndPos) c.headerProps = defaultProperties - c.levels[^1] = Level(state: beforeBlockMapValue, indentation: c.lex.recentIndentation()) - c.levels.add(Level(state: beforeBlockIndentation)) - c.levels.add(Level(state: afterCompactParent, indentation: c.lex.recentIndentation())) + c.transition(beforeBlockMapValue, c.lex.recentIndentation()) + c.pushLevel(beforeBlockIndentation) + c.pushLevel(afterCompactParent, c.lex.recentIndentation()) c.lex.next() + return true of Plain, SingleQuoted, DoubleQuoted: - c.levels[^1].indentation = c.lex.recentIndentation() + c.updateIndentation(c.lex.recentIndentation()) let scalarToken = c.lex.cur e = scalarEvent(c.lex.evaluated, c.headerProps, toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) @@ -401,10 +438,10 @@ proc atBlockIndentation(c: Context, e: var Event): bool = e.scalarProperties = autoScalarTag(defaultProperties, scalarToken) c.peek = move(e) e = startMapEvent(csBlock, props, c.headerStart, headerEnd) - c.levels[^1].state = afterImplicitKey + c.transition(afterImplicitKey) else: e.scalarProperties = autoScalarTag(e.scalarProperties, scalarToken) - discard c.levels.pop() + c.popLevel() return true of Alias: e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) @@ -415,24 +452,25 @@ proc atBlockIndentation(c: Context, e: var Event): bool = c.peek = move(e) e = startMapEvent(csBlock, c.headerProps, c.headerStart, headerEnd) c.headerProps = defaultProperties - c.levels[^1].state = afterImplicitKey + c.transition(afterImplicitKey) elif not isEmpty(c.headerProps): raise c.generateError("Alias may not have properties") else: - discard c.levels.pop() + c.popLevel() return true else: - c.levels[^1].state = atBlockIndentationProps + c.transition(atBlockIndentationProps) + return false proc atBlockIndentationProps(c: Context, e: var Event): bool = - c.levels[^1].indentation = c.lex.recentIndentation() + c.updateIndentation(c.lex.recentIndentation()) case c.lex.cur of MapValueInd: c.peek = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curEndPos) c.inlineProps = defaultProperties e = startMapEvent(csBlock, c.headerProps, c.lex.curStartPos, c.lex.curEndPos) c.headerProps = defaultProperties - c.levels[^1].state = afterImplicitKey + c.transition(afterImplicitKey) return true of Plain, SingleQuoted, DoubleQuoted: e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur), @@ -446,20 +484,20 @@ proc atBlockIndentationProps(c: Context, e: var Event): bool = c.peek = move(e) e = startMapEvent(csBlock, c.headerProps, c.headerStart, headerEnd) c.headerProps = defaultProperties - c.levels[^1].state = afterImplicitKey + c.transition(afterImplicitKey) else: - discard c.levels.pop() + c.popLevel() return true of MapStart: e = startMapEvent(csFlow, c.headerProps, c.headerStart, c.lex.curEndPos) c.headerProps = defaultProperties - c.levels[^1].state = afterFlowMapSep + c.transition(afterFlowMapSep) c.lex.next() return true of SeqStart: e = startSeqEvent(csFlow, c.headerProps, c.headerStart, c.lex.curEndPos) c.headerProps = defaultProperties - c.levels[^1].state = afterFlowSeqSep + c.transition(afterFlowSeqSep) c.lex.next() return true else: @@ -485,12 +523,12 @@ proc beforeNodeProperties(c: Context, e: var Event): bool = of Indentation: c.headerProps = c.inlineProps c.inlineProps = defaultProperties - discard c.levels.pop() + c.popLevel() return false of Alias: raise c.generateError("Alias may not have node properties") else: - discard c.levels.pop() + c.popLevel() return false c.lex.next() return false @@ -499,49 +537,49 @@ proc afterCompactParent(c: Context, e: var Event): bool = c.inlineStart = c.lex.curStartPos case c.lex.cur of nodePropertyKind: - c.levels[^1].state = afterCompactParentProps - c.levels.add(Level(state: beforeNodeProperties)) + c.transition(afterCompactParentProps) + c.pushLevel(beforeNodeProperties) of SeqItemInd: e = startSeqEvent(csBlock, c.headerProps, c.headerStart, c.lex.curEndPos) c.headerProps = defaultProperties - c.levels[^1] = Level(state: inBlockSeq, indentation: c.lex.recentIndentation()) - echo "started seq at indentation ", c.lex.recentIndentation() - c.levels.add(Level(state: beforeBlockIndentation)) - c.levels.add(Level(state: afterCompactParent)) + c.transition(inBlockSeq, c.lex.recentIndentation()) + c.pushLevel(beforeBlockIndentation) + c.pushLevel(afterCompactParent, c.lex.recentIndentation()) c.lex.next() return true of MapKeyInd: e = startMapEvent(csBlock, c.headerProps, c.headerStart, c.lex.curEndPos) c.headerProps = defaultProperties - c.levels[^1] = Level(state: beforeBlockMapValue, indentation: c.lex.recentIndentation()) - c.levels.add(Level(state: beforeBlockIndentation)) - c.levels.add(Level(state: afterCompactParent)) + c.transition(beforeBlockMapValue, c.lex.recentIndentation()) + c.pushLevel(beforeBlockIndentation) + c.pushLevel(afterCompactParent, c.lex.recentIndentation) + c.lex.next() return true else: - c.levels[^1].state = afterCompactParentProps + c.transition(afterCompactParentProps) return false proc afterCompactParentProps(c: Context, e: var Event): bool = - c.levels[^1].indentation = c.lex.recentIndentation() + c.updateIndentation(c.lex.recentIndentation()) case c.lex.cur of nodePropertyKind: - c.levels.add(Level(state: beforeNodeProperties)) + c.pushLevel(beforeNodeProperties) return false of Indentation: c.headerStart = c.inlineStart - c.levels[^1] = Level(state: atBlockIndentation, indentation: c.levels[^3].indentation) - c.levels.add(Level(state: beforeBlockIndentation)) + c.transition(atBlockIndentation, c.levels[^3].indentation) + c.pushLevel(beforeBlockIndentation) return false of StreamEnd, DocumentEnd, DirectivesEnd: e = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos) c.inlineProps = defaultProperties - discard c.levels.pop() + c.popLevel() return true of MapValueInd: c.peek = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos) c.inlineProps = defaultProperties e = startMapEvent(csBlock, defaultProperties, c.lex.curStartPos, c.lex.curStartPos) - c.levels[^1].state = afterImplicitKey + c.transition(afterImplicitKey) return true of Alias: e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) @@ -550,36 +588,36 @@ proc afterCompactParentProps(c: Context, e: var Event): bool = if c.lex.cur == Token.MapValueInd: c.peek = move(e) e = startMapEvent(csBlock, defaultProperties, headerEnd, headerEnd) - c.levels[^1].state = afterImplicitKey + c.transition(afterImplicitKey) else: - discard c.levels.pop() + c.popLevel() return true of scalarTokenKind: e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur), toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) c.inlineProps = defaultProperties let headerEnd = c.lex.curStartPos - c.levels[^1].indentation = c.lex.recentIndentation() + c.updateIndentation(c.lex.recentIndentation()) c.lex.next() if c.lex.cur == Token.MapValueInd: if c.lex.lastScalarWasMultiline(): raise c.generateError("Implicit mapping key may not be multiline") c.peek = move(e) e = startMapEvent(csBlock, defaultProperties, headerEnd, headerEnd) - c.levels[^1].state = afterImplicitKey + c.transition(afterImplicitKey) else: - discard c.levels.pop() + c.popLevel() return true of MapStart: e = startMapEvent(csFlow, c.inlineProps, c.inlineStart, c.lex.curEndPos) c.inlineProps = defaultProperties - c.levels[^1].state = afterFlowMapSep + c.transition(afterFlowMapSep) c.lex.next() return true of SeqStart: e = startSeqEvent(csFlow, c.inlineProps, c.inlineStart, c.lex.curEndPos) c.inlineProps = defaultProperties - c.levels[^1].state = afterFlowSeqSep + c.transition(afterFlowSeqSep) c.lex.next() return true else: @@ -589,19 +627,19 @@ proc afterBlockParent(c: Context, e: var Event): bool = c.inlineStart = c.lex.curStartPos case c.lex.cur of nodePropertyKind: - c.levels[^1].state = afterBlockParentProps - c.levels.add(Level(state: beforeNodeProperties)) + c.transition(afterBlockParentProps) + c.pushLevel(beforeNodeProperties) of SeqItemInd, MapKeyInd: raise c.generateError("Compact notation not allowed after implicit key") else: - c.levels[^1].state = afterBlockParentProps + c.transition(afterBlockParentProps) return false proc afterBlockParentProps(c: Context, e: var Event): bool = - c.levels[^1].indentation = c.lex.recentIndentation() + c.updateIndentation(c.lex.recentIndentation()) case c.lex.cur of nodePropertyKind: - c.levels.add(Level(state: beforeNodeProperties)) + c.pushLevel(beforeNodeProperties) return false of MapValueInd: raise c.generateError("Compact notation not allowed after implicit key") @@ -612,33 +650,42 @@ proc afterBlockParentProps(c: Context, e: var Event): bool = c.lex.next() if c.lex.cur == Token.MapValueInd: raise c.generateError("Compact notation not allowed after implicit key") - discard c.levels.pop() + c.popLevel() return true else: - c.levels[^1].state = afterCompactParentProps + c.transition(afterCompactParentProps) return false proc requireInlineBlockItem(c: Context, e: var Event): bool = - c.levels[^1].indentation = c.lex.recentIndentation() - case c.lex.cur - of Indentation: - raise c.generateError("Node properties may not stand alone on a line") - else: - c.levels[^1].state = afterCompactParentProps - return false + c.updateIndentation(c.lex.recentIndentation()) + if c.lex.cur == Token.Indentation: + if c.inlineProps.tag != yTagQuestionMark: + if c.headerProps.tag != yTagQuestionMark: + raise c.generateError("Only one tag allowed per node") + c.headerProps.tag = c.inlineProps.tag + c.inlineProps.tag = yTagQuestionMark + if c.inlineProps.anchor != yAnchorNone: + if c.headerProps.anchor != yAnchorNone: + raise c.generateError("Only one anchor allowed per node") + c.headerProps.anchor = c.inlineProps.anchor + c.inlineProps.anchor = yAnchorNone + c.transition(afterCompactParentProps) + return false proc beforeDocEnd(c: Context, e: var Event): bool = case c.lex.cur of DocumentEnd: - e = endDocEvent(false, c.lex.curStartPos, c.lex.curEndPos) - c.levels[^1].state = beforeDoc - c.lex.next() - of StreamEnd: e = endDocEvent(true, c.lex.curStartPos, c.lex.curEndPos) - discard c.levels.pop() + c.transition(beforeDoc) + c.lex.next() + c.tagLib.resetPrefixes() + of StreamEnd: + e = endDocEvent(false, c.lex.curStartPos, c.lex.curEndPos) + c.popLevel() of DirectivesEnd: - e = endDocEvent(true, c.lex.curStartPos, c.lex.curStartPos) - c.levels[^1].state = beforeDoc + e = endDocEvent(false, c.lex.curStartPos, c.lex.curStartPos) + c.transition(beforeDoc) + c.tagLib.resetPrefixes() else: raise c.generateError("Unexpected token (expected document end): " & $c.lex.cur) return true @@ -649,14 +696,15 @@ proc inBlockSeq(c: Context, e: var Event): bool = case c.lex.cur of SeqItemInd: c.lex.next() - c.levels.add(Level(state: beforeBlockIndentation)) - c.levels.add(Level(state: afterCompactParent, indentation: c.blockIndentation)) + c.pushLevel(beforeBlockIndentation) + c.pushLevel(afterCompactParent, c.blockIndentation) return false else: if c.levels[^3].indentation == c.levels[^1].indentation: e = endSeqEvent(c.lex.curStartPos, c.lex.curEndPos) - discard c.levels.pop() - discard c.levels.pop() + c.popLevel() + c.popLevel() + return true else: raise c.generateError("Illegal token (expected block sequence indicator): " & $c.lex.cur) @@ -665,26 +713,26 @@ proc beforeBlockMapKey(c: Context, e: var Event): bool = raise c.generateError("Invalid indentation: got " & $c.blockIndentation & ", expected " & $c.levels[^1].indentation) case c.lex.cur of MapKeyInd: - c.levels[^1].state = beforeBlockMapValue - c.levels.add(Level(state: beforeBlockIndentation)) - c.levels.add(Level(state: afterCompactParent, indentation: c.blockIndentation)) + c.transition(beforeBlockMapValue) + c.pushLevel(beforeBlockIndentation) + c.pushLevel(afterCompactParent, c.blockIndentation) c.lex.next() return false of nodePropertyKind: - c.levels[^1].state = atBlockMapKeyProps - c.levels.add(Level(state: beforeNodeProperties)) + c.transition(atBlockMapKeyProps) + c.pushLevel(beforeNodeProperties) return false of Plain, SingleQuoted, DoubleQuoted: - c.levels[^1].state = atBlockMapKeyProps + c.transition(atBlockMapKeyProps) return false of Alias: e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) c.lex.next() - c.levels[^1].state = afterImplicitKey + c.transition(afterImplicitKey) return true of MapValueInd: e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curEndPos) - c.levels[^1].state = beforeBlockMapValue + c.transition(beforeBlockMapValue) return true else: raise c.generateError("Unexpected token (expected mapping key): " & $c.lex.cur) @@ -692,7 +740,7 @@ proc beforeBlockMapKey(c: Context, e: var Event): bool = proc atBlockMapKeyProps(c: Context, e: var Event): bool = case c.lex.cur of nodePropertyKind: - c.levels.add(Level(state: beforeNodeProperties)) + c.pushLevel(beforeNodeProperties) of Alias: e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) of Plain, SingleQuoted, DoubleQuoted: @@ -704,21 +752,21 @@ proc atBlockMapKeyProps(c: Context, e: var Event): bool = of MapValueInd: e = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos) c.inlineProps = defaultProperties - c.levels[^1].state = afterImplicitKey + c.transition(afterImplicitKey) return true else: raise c.generateError("Unexpected token (expected implicit mapping key): " & $c.lex.cur) c.lex.next() - c.levels[^1].state = afterImplicitKey + c.transition(afterImplicitKey) return true proc afterImplicitKey(c: Context, e: var Event): bool = if c.lex.cur != Token.MapValueInd: raise c.generateError("Unexpected token (expected ':'): " & $c.lex.cur) c.lex.next() - c.levels[^1].state = beforeBlockMapKey - c.levels.add(Level(state: beforeBlockIndentation)) - c.levels.add(Level(state: afterBlockParent, indentation: c.levels[^2].indentation)) + c.transition(beforeBlockMapKey) + c.pushLevel(beforeBlockIndentation) + c.pushLevel(afterBlockParent, max(0, c.levels[^2].indentation)) return false proc beforeBlockMapValue(c: Context, e: var Event): bool = @@ -726,14 +774,14 @@ proc beforeBlockMapValue(c: Context, e: var Event): bool = raise c.generateError("Invalid indentation") case c.lex.cur of MapValueInd: - c.levels[^1].state = beforeBlockMapKey - c.levels.add(Level(state: beforeBlockIndentation)) - c.levels.add(Level(state: afterCompactParent, indentation: c.blockIndentation)) + c.transition(beforeBlockMapKey) + c.pushLevel(beforeBlockIndentation) + c.pushLevel(afterCompactParent, c.blockIndentation) c.lex.next() of MapKeyInd, Plain, SingleQuoted, DoubleQuoted, nodePropertyKind: # the value is allowed to be missing after an explicit key e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curEndPos) - c.levels[^1].state = beforeBlockMapKey + c.transition(beforeBlockMapKey) return true else: raise c.generateError("Unexpected token (expected mapping value): " & $c.lex.cur) @@ -744,8 +792,8 @@ proc beforeBlockIndentation(c: Context, e: var Event): bool = e = endMapEvent(c.lex.curStartPos, c.lex.curEndPos) elif c.levels[^1].state == beforeBlockMapValue: e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curEndPos) - c.levels[^1].state = beforeBlockMapKey - c.levels.add(Level(state: beforeBlockIndentation)) + c.transition(beforeBlockMapKey) + c.pushLevel(beforeBlockIndentation) return elif c.levels[^1].state == inBlockSeq: e = endSeqEvent(c.lex.curStartPos, c.lex.curEndPos) @@ -756,8 +804,8 @@ proc beforeBlockIndentation(c: Context, e: var Event): bool = raise c.generateError("Unexpected double beforeBlockIndentation") else: raise c.generateError("Internal error (please report this bug)") - discard c.levels.pop() - discard c.levels.pop() + c.popLevel() + c.popLevel() case c.lex.cur of Indentation: c.blockIndentation = c.lex.currentIndentation() @@ -778,45 +826,46 @@ proc beforeBlockIndentation(c: Context, e: var Event): bool = raise c.generateError("Unexpected content after node in block context (expected newline): " & $c.lex.cur) proc beforeFlowItem(c: Context, e: var Event): bool = + debug("parse: beforeFlowItem") c.inlineStart = c.lex.curStartPos case c.lex.cur of nodePropertyKind: - c.levels[^1].state = beforeFlowItemProps - c.levels.add(Level(state: beforeNodeProperties)) + c.transition(beforeFlowItemProps) + c.pushLevel(beforeNodeProperties) of Alias: e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) c.lex.next() - discard c.levels.pop() + c.popLevel() return true else: - c.levels[^1].state = beforeFlowItemProps + c.transition(beforeFlowItemProps) return false proc beforeFlowItemProps(c: Context, e: var Event): bool = case c.lex.cur of nodePropertyKind: - c.levels.add(Level(state: beforeNodeProperties)) + c.pushLevel(beforeNodeProperties) of Alias: e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) c.lex.next() - discard c.levels.pop() + c.popLevel() of scalarTokenKind: e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur), toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) c.inlineProps = defaultProperties c.lex.next() - discard c.levels.pop() + c.popLevel() of MapStart: e = startMapEvent(csFlow, c.inlineProps, c.inlineStart, c.lex.curEndPos) - c.levels[^1].state = afterFlowMapSep + c.transition(afterFlowMapSep) c.lex.next() of SeqStart: e = startSeqEvent(csFlow, c.inlineProps, c.inlineStart, c.lex.curEndPos) - c.levels[^1].state = afterFlowSeqSep + c.transition(afterFlowSeqSep) c.lex.next() of MapEnd, SeqEnd, SeqSep, MapValueInd: e = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curEndPos) - discard c.levels.pop() + c.popLevel() else: raise c.generateError("Unexpected token (expected flow node): " & $c.lex.cur) c.inlineProps = defaultProperties @@ -825,13 +874,13 @@ proc beforeFlowItemProps(c: Context, e: var Event): bool = proc afterFlowMapKey(c: Context, e: var Event): bool = case c.lex.cur of MapValueInd: - c.levels[^1].state = afterFlowMapValue - c.levels.add(Level(state: beforeFlowItem)) + c.transition(afterFlowMapValue) + c.pushLevel(beforeFlowItem) c.lex.next() return false of SeqSep, MapEnd: e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curEndPos) - c.levels[^1].state = afterFlowMapValue + c.transition(afterFlowMapValue) return true else: raise c.generateError("Unexpected token (expected ':'): " & $c.lex.cur) @@ -839,13 +888,13 @@ proc afterFlowMapKey(c: Context, e: var Event): bool = proc afterFlowMapValue(c: Context, e: var Event): bool = case c.lex.cur of SeqSep: - c.levels[^1].state = afterFlowMapSep + c.transition(afterFlowMapSep) c.lex.next() return false of MapEnd: e = endMapEvent(c.lex.curStartPos, c.lex.curEndPos) c.lex.next() - discard c.levels.pop() + c.popLevel() return true of Plain, SingleQuoted, DoubleQuoted, MapKeyInd, Token.Anchor, Alias, MapStart, SeqStart: raise c.generateError("Missing ','") @@ -855,13 +904,13 @@ proc afterFlowMapValue(c: Context, e: var Event): bool = proc afterFlowSeqItem(c: Context, e: var Event): bool = case c.lex.cur of SeqSep: - c.levels[^1].state = afterFlowSeqSep + c.transition(afterFlowSeqSep) c.lex.next() return false of SeqEnd: e = endSeqEvent(c.lex.curStartPos, c.lex.curEndPos) c.lex.next() - discard c.levels.pop() + c.popLevel() return true of Plain, SingleQuoted, DoubleQuoted, MapKeyInd, Token.Anchor, Alias, MapStart, SeqStart: raise c.generateError("Missing ','") @@ -875,11 +924,11 @@ proc afterFlowMapSep(c: Context, e: var Event): bool = of MapEnd: e = endMapEvent(c.lex.curStartPos, c.lex.curEndPos) c.lex.next() - discard c.levels.pop() + c.popLevel() return true else: discard - c.levels[^1].state = afterFlowMapKey - c.levels.add(Level(state: beforeFlowItem)) + c.transition(afterFlowMapKey) + c.pushLevel(beforeFlowItem) return false proc possibleNextSequenceItem(c: Context, e: var Event, endToken: Token, afterProps, afterItem: State): bool = @@ -890,33 +939,33 @@ proc possibleNextSequenceItem(c: Context, e: var Event, endToken: Token, afterPr c.lex.next() return true of nodePropertyKind: - c.levels[^1].state = afterProps - c.levels.add(Level(state: beforeNodeProperties)) + c.transition(afterProps) + c.pushLevel(beforeNodeProperties) return false of Plain, SingleQuoted, DoubleQuoted: - c.levels[^1].state = afterProps + c.transition(afterProps) return false of MapKeyInd: - c.levels[^1].state = afterItem + c.transition(afterItem) e = startMapEvent(csFlow, defaultProperties, c.lex.curStartPos, c.lex.curEndPos) c.lex.next() - c.levels.add(Level(state: beforePairValue)) - c.levels.add(Level(state: beforeFlowItem)) + c.pushLevel(beforePairValue) + c.pushLevel(beforeFlowItem) return true of MapValueInd: - c.levels[^1].state = afterItem + c.transition(afterItem) e = startMapEvent(csFlow, defaultProperties, c.lex.curStartPos, c.lex.curEndPos) - c.levels.add(Level(state: atEmptyPairKey)) + c.pushLevel(atEmptyPairKey) return true else: if c.lex.cur == endToken: e = endSeqEvent(c.lex.curStartPos, c.lex.curEndPos) c.lex.next() - discard c.levels.pop() + c.popLevel() return true else: - c.levels[^1].state = afterItem - c.levels.add(Level(state: beforeFlowItem)) + c.transition(afterItem) + c.pushLevel(beforeFlowItem) return false proc afterFlowSeqSep(c: Context, e: var Event): bool = @@ -930,47 +979,44 @@ proc forcedNextSequenceItem(c: Context, e: var Event): bool = if c.lex.cur == Token.MapValueInd: c.peek = move(e) e = startMapEvent(csFlow, defaultProperties, c.lex.curStartPos, c.lex.curStartPos) - c.levels.add(Level(state: afterImplicitPairStart)) + c.pushLevel(afterImplicitPairStart) return true else: - c.levels.add(Level(state: beforeFlowItem)) + c.pushLevel(beforeFlowItem) return false proc afterFlowSeqSepProps(c: Context, e: var Event): bool = - c.levels[^1].state = afterFlowSeqItem + c.transition(afterFlowSeqItem) return forcedNextSequenceItem(c, e) proc atEmptyPairKey(c: Context, e: var Event): bool = - c.levels[^1].state = beforePairValue + c.transition(beforePairValue) e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curStartPos) return true proc beforePairValue(c: Context, e: var Event): bool = if c.lex.cur == Token.MapValueInd: - c.levels[^1].state = afterPairValue - c.levels.add(Level(state: beforeFlowItem)) + c.transition(afterPairValue) + c.pushLevel(beforeFlowItem) c.lex.next() return false else: # pair ends here without value e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curEndPos) - discard c.levels.pop() + c.popLevel() return true proc afterImplicitPairStart(c: Context, e: var Event): bool = c.lex.next() - c.levels[^1].state = afterPairValue - c.levels.add(Level(state: beforeFLowItem)) + c.transition(afterPairValue) + c.pushLevel(beforeFlowItem) return false proc afterPairValue(c: Context, e: var Event): bool = e = endMapEvent(c.lex.curStartPos, c.lex.curEndPos) - discard c.levels.pop() + c.popLevel() return true -# TODO -------------- - - proc display*(p: YamlParser, event: Event): string = ## Generate a representation of the given event with proper visualization of ## anchor and tag (if any). The generated representation is conformant to the @@ -988,30 +1034,22 @@ proc display*(p: YamlParser, event: Event): string = of yamlEndSeq: result = "-SEQ" of yamlStartDoc: result = "+DOC" - when defined(yamlScalarRepInd): - if event.explicitDirectivesEnd: result &= " ---" + if event.explicitDirectivesEnd: result &= " ---" of yamlEndDoc: result = "-DOC" - when defined(yamlScalarRepInd): - if event.explicitDocumentEnd: result &= " ..." + if event.explicitDocumentEnd: result &= " ..." of yamlStartMap: result = "+MAP" & renderAttrs(event.mapProperties, true) of yamlStartSeq: result = "+SEQ" & renderAttrs(event.seqProperties, true) of yamlScalar: - when defined(yamlScalarRepInd): - result = "=VAL" & renderAttrs(event.scalarProperties, - event.scalarRep == srPlain) - case event.scalarRep - of srPlain: result &= " :" - of srSingleQuoted: result &= " \'" - of srDoubleQuoted: result &= " \"" - of srLiteral: result &= " |" - of srFolded: result &= " >" - else: - let isPlain = event.scalarProperties.tag == yTagExclamationmark - result = "=VAL" & renderAttrs(event.scalarProperties, isPlain) - if isPlain: result &= " :" - else: result &= " \"" + result = "=VAL" & renderAttrs(event.scalarProperties, + event.scalarStyle in {ssPlain, ssFolded, ssLiteral}) + case event.scalarStyle + of ssPlain, ssAny: result &= " :" + of ssSingleQuoted: result &= " \'" + of ssDoubleQuoted: result &= " \"" + of ssLiteral: result &= " |" + of ssFolded: result &= " >" result &= yamlTestSuiteEscape(event.scalarContent) of yamlAlias: result = "=ALI *" & $event.aliasTarget \ No newline at end of file diff --git a/yaml/private/lex.nim b/yaml/private/lex.nim index e5df90b..2066b1f 100644 --- a/yaml/private/lex.nim +++ b/yaml/private/lex.nim @@ -172,7 +172,7 @@ proc streamEnd(lex: var Lexer): bool {.raises: [].} template debug(message: string) {.dirty.} = when defined(yamlDebug): try: styledWriteLine(stdout, fgBlue, message) - except IOError: discard + except ValueError, IOError: discard proc generateError(lex: Lexer, message: string): ref LexerError {.raises: [].} = @@ -428,6 +428,13 @@ proc streamEndAfterBlock(lex: var Lexer) = lex.endToken() lex.curEndPos.column -= 1 +proc dirEndFollows(lex: Lexer): bool = + return lex.c == '-' and lex.source.buf[lex.source.bufpos] == '-' and + lex.source.buf[lex.source.bufpos+1] == '-' + +proc docEndFollows(lex: Lexer): bool = + return lex.c == '.' and lex.source.buf[lex.source.bufpos] == '.' and + lex.source.buf[lex.source.bufpos+1] == '.' proc readBlockScalar(lex: var Lexer) = var @@ -492,7 +499,8 @@ proc readBlockScalar(lex: var Lexer) = else: if indent == 0: indent = lex.currentIndentation() - if indent <= max(0, lex.indentation): + if indent <= lex.indentation or + (indent == 0 and (lex.dirEndFollows() or lex.docEndFollows())): lex.state = lineIndentation break body elif indent < maxLeadingSpaces: @@ -530,7 +538,8 @@ proc readBlockScalar(lex: var Lexer) = lex.streamEndAfterBlock() break body else: - if lex.currentIndentation() < indent: + if lex.currentIndentation() < indent or + (indent == 0 and lex.dirEndFollows() or lex.docEndFollows()): break content else: break @@ -544,12 +553,14 @@ proc readBlockScalar(lex: var Lexer) = for i in countup(0, separationLines - 2): lex.evaluated.add('\l') - if lex.currentIndentation() > max(0, lex.indentation): + let markerFollows = lex.currentIndentation() == 0 and + (lex.dirEndFollows() or lex.docEndFollows()) + if lex.currentIndentation() > lex.indentation and not markerFollows: if lex.c == '#': lex.state = expectLineEnd else: - raise lex.generateError("This line at " & escape("" & lex.c) & " is less indented than necessary") - elif lex.columnNumber() == 1: + raise lex.generateError("This line #" & $lex.curStartPos.line & " at " & escape("" & lex.c) & " is less indented than necessary") + elif lex.currentIndentation() == 0: lex.state = lineStart else: lex.state = lineIndentation @@ -570,7 +581,7 @@ proc processQuotedWhitespace(lex: var Lexer, initial: int) = let firstSpace = lex.source.bufpos - 1 while true: case lex.c - of ' ': discard + of ' ', '\t': discard of '\l': lex.lexLF() break @@ -584,7 +595,11 @@ proc processQuotedWhitespace(lex: var Lexer, initial: int) = lex.seenMultiline = true while true: case lex.startLine() - of lsContent, lsComment: break + of lsContent, lsComment: + while lex.c in space: lex.advance() + if lex.c in {'\l', '\c'}: + lex.endLine() + else: break of lsDirectivesEndMarker: raise lex.generateError("Illegal `---` within quoted scalar") of lsDocumentEndMarker: @@ -619,7 +634,7 @@ proc readSingleQuotedScalar(lex: var Lexer) = literalStart = lex.source.bufpos lex.advance() else: break - of ' ', '\l', '\c': + of ' ', '\t', '\l', '\c': lex.evaluated.add(lex.source.buf[literalStart..lex.source.bufpos - 2]) lex.processQuotedWhitespace(1) literalStart = lex.source.bufpos - 1 @@ -681,7 +696,7 @@ proc readDoubleQuotedScalar(lex: var Lexer) = of '"': lex.evaluated.add(lex.source.buf[literalStart..lex.source.bufpos - 2]) break - of ' ', '\l', '\c': + of ' ', '\t', '\l', '\c': lex.evaluated.add(lex.source.buf[literalStart..lex.source.bufpos - 2]) lex.processQuotedWhitespace(1) literalStart = lex.source.bufpos - 1 @@ -756,7 +771,7 @@ proc outsideDoc(lex: var Lexer): bool = of '-': lex.startToken() if lex.isDirectivesEnd(): - lex.state = expectLineEnd + lex.state = afterToken lex.cur = Token.DirectivesEnd else: lex.state = indentationSettingToken @@ -783,6 +798,7 @@ proc outsideDoc(lex: var Lexer): bool = return false lex.endToken() lex.cur = Token.Indentation + lex.indentation = -1 lex.state = indentationSettingToken lex.lineStartState = lineStart return true @@ -1083,6 +1099,7 @@ proc lineDirEnd(lex: var Lexer): bool = lex.curStartPos.column = 1 lex.endToken() lex.cur = Token.DirectivesEnd + lex.state = afterToken lex.indentation = -1 lex.propertyIndentation = -1 return true diff --git a/yaml/serialization.nim b/yaml/serialization.nim index 7d32bdd..c166158 100644 --- a/yaml/serialization.nim +++ b/yaml/serialization.nim @@ -18,7 +18,7 @@ import tables, typetraits, strutils, macros, streams, times, parseutils, options import data, parser, taglib, presenter, stream, private/internal, hints, annotations -export stream, macros, annotations, options +export data, stream, macros, annotations, options # *something* in here needs externally visible `==`(x,y: AnchorId), # but I cannot figure out what. binding it would be the better option. @@ -125,10 +125,10 @@ proc safeTagUri(id: TagId): string {.raises: [].} = return uri except KeyError: internalError("Unexpected KeyError for TagId " & $id) -proc constructionError(s: YamlStream, msg: string): ref YamlConstructionError = +proc constructionError(s: YamlStream, mark: Mark, msg: string): ref YamlConstructionError = result = newException(YamlConstructionError, msg) - if not s.getLastTokenContext(result.mark.line, result.mark.column, result.lineContent): - (result.mark.line, result.mark.column) = (-1, -1) + result.mark = mark + if not s.getLastTokenContext(result.lineContent): result.lineContent = "" template constructScalarItem*(s: var YamlStream, i: untyped, @@ -141,11 +141,11 @@ template constructScalarItem*(s: var YamlStream, i: untyped, bind constructionError let i = s.next() if i.kind != yamlScalar: - raise constructionError(s, "Expected scalar") + raise s.constructionError(i.startPos, "Expected scalar") try: content except YamlConstructionError as e: raise e except Exception: - var e = constructionError(s, + var e = s.constructionError(i.startPos, "Cannot construct to " & name(t) & ": " & item.scalarContent & "; error: " & getCurrentExceptionMsg()) e.parent = getCurrentException() @@ -167,7 +167,7 @@ proc representObject*(value: string, ts: TagStyle, c.put(scalarEvent(value, tag, yAnchorNone)) proc parseHex[T: int8|int16|int32|int64|uint8|uint16|uint32|uint64]( - s: YamlStream, val: string): T = + s: YamlStream, mark: Mark, val: string): T = result = 0 for i in 2.. 1 and item.scalarContent[1] in {'x', 'X' }: - result = parseHex[T](s, item.scalarContent) + result = parseHex[T](s, item.startPos, item.scalarContent) elif item.scalarContent[0] == '0' and item.scalarContent.len > 1 and item.scalarContent[1] in {'o', 'O'}: - result = parseOctal[T](s, item.scalarContent) + result = parseOctal[T](s, item.startPos, item.scalarContent) else: let nInt = parseBiggestInt(item.scalarContent) if nInt <= T.high: # make sure we don't produce a range error result = T(nInt) else: - raise s.constructionError("Cannot construct int; out of range: " & + raise s.constructionError(item.startPos, "Cannot construct int; out of range: " & $nInt & " for type " & T.name & " with max of: " & $T.high) proc constructObject*(s: var YamlStream, c: ConstructionContext, @@ -245,9 +245,9 @@ proc constructObject*[T: DefiniteUIntTypes]( ## construct an unsigned integer value from a YAML scalar constructScalarItem(s, item, T): if item.scalarContent[0] == '0' and item.scalarContent[1] in {'x', 'X'}: - result = parseHex[T](s, item.scalarContent) + result = parseHex[T](s, item.startPos, item.scalarContent) elif item.scalarContent[0] == '0' and item.scalarContent[1] in {'o', 'O'}: - result = parseOctal[T](s, item.scalarContent) + result = parseOctal[T](s, item.startPos, item.scalarContent) else: result = T(parseBiggestUInt(item.scalarContent)) proc constructObject*(s: var YamlStream, c: ConstructionContext, @@ -295,7 +295,7 @@ proc constructObject*[T: float|float32|float64]( else: result = Inf of yTypeFloatNaN: result = NaN else: - raise s.constructionError("Cannot construct to float: " & + raise s.constructionError(item.startPos, "Cannot construct to float: " & escape(item.scalarContent)) proc representObject*[T: float|float32|float64](value: T, ts: TagStyle, @@ -318,7 +318,7 @@ proc constructObject*(s: var YamlStream, c: ConstructionContext, of yTypeBoolTrue: result = true of yTypeBoolFalse: result = false else: - raise s.constructionError("Cannot construct to bool: " & + raise s.constructionError(item.startPos, "Cannot construct to bool: " & escape(item.scalarContent)) proc representObject*(value: bool, ts: TagStyle, c: SerializationContext, @@ -332,7 +332,7 @@ proc constructObject*(s: var YamlStream, c: ConstructionContext, ## constructs a char value from a YAML scalar constructScalarItem(s, item, char): if item.scalarContent.len != 1: - raise s.constructionError("Cannot construct to char (length != 1): " & + raise s.constructionError(item.startPos, "Cannot construct to char (length != 1): " & escape(item.scalarContent)) else: result = item.scalarContent[0] @@ -399,7 +399,7 @@ proc constructObject*(s: var YamlStream, c: ConstructionContext, let info = tmp.parse("yyyy-M-d'T'H:mm:sszzz") result = info.toTime() else: - raise s.constructionError("Not a parsable timestamp: " & + raise s.constructionError(item.startPos, "Not a parsable timestamp: " & escape(item.scalarContent)) proc representObject*(value: Time, ts: TagStyle, c: SerializationContext, @@ -421,7 +421,7 @@ proc constructObject*[T](s: var YamlStream, c: ConstructionContext, ## constructs a Nim seq from a YAML sequence let event = s.next() if event.kind != yamlStartSeq: - raise s.constructionError("Expected sequence start") + raise s.constructionError(event.startPos, "Expected sequence start") result = newSeq[T]() while s.peek().kind != yamlEndSeq: var item: T @@ -435,7 +435,7 @@ proc constructObject*[T](s: var YamlStream, c: ConstructionContext, ## constructs a Nim seq from a YAML sequence let event = s.next() if event.kind != yamlStartSeq: - raise s.constructionError("Expected sequence start") + raise s.constructionError(event.startPos, "Expected sequence start") result = {} while s.peek().kind != yamlEndSeq: var item: T @@ -447,7 +447,7 @@ proc representObject*[T](value: seq[T]|set[T], ts: TagStyle, c: SerializationContext, tag: TagId) = ## represents a Nim seq as YAML sequence let childTagStyle = if ts == tsRootOnly: tsNone else: ts - c.put(startSeqEvent(tag)) + c.put(startSeqEvent(csBlock, tag)) for item in value: representChild(item, childTagStyle, c) c.put(endSeqEvent()) @@ -464,15 +464,15 @@ proc constructObject*[I, T](s: var YamlStream, c: ConstructionContext, ## constructs a Nim array from a YAML sequence var event = s.next() if event.kind != yamlStartSeq: - raise s.constructionError("Expected sequence start") + raise s.constructionError(event.startPos, "Expected sequence start") for index in low(I)..high(I): event = s.peek() if event.kind == yamlEndSeq: - raise s.constructionError("Too few array values") + raise s.constructionError(event.startPos, "Too few array values") constructChild(s, c, result[index]) event = s.next() if event.kind != yamlEndSeq: - raise s.constructionError("Too many array values") + raise s.constructionError(event.startPos, "Too many array values") proc representObject*[I, T](value: array[I, T], ts: TagStyle, c: SerializationContext, tag: TagId) = @@ -498,7 +498,7 @@ proc constructObject*[K, V](s: var YamlStream, c: ConstructionContext, ## constructs a Nim Table from a YAML mapping let event = s.next() if event.kind != yamlStartMap: - raise s.constructionError("Expected map start, got " & $event.kind) + raise s.constructionError(event.startPos, "Expected map start, got " & $event.kind) result = initTable[K, V]() while s.peek.kind != yamlEndMap: var @@ -507,7 +507,7 @@ proc constructObject*[K, V](s: var YamlStream, c: ConstructionContext, constructChild(s, c, key) constructChild(s, c, value) if result.contains(key): - raise s.constructionError("Duplicate table key!") + raise s.constructionError(event.startPos, "Duplicate table key!") result[key] = value discard s.next() @@ -537,7 +537,7 @@ proc constructObject*[K, V](s: var YamlStream, c: ConstructionContext, ## constructs a Nim OrderedTable from a YAML mapping var event = s.next() if event.kind != yamlStartSeq: - raise s.constructionError("Expected seq start, got " & $event.kind) + raise s.constructionError(event.startPos, "Expected seq start, got " & $event.kind) result = initOrderedTable[K, V]() while s.peek.kind != yamlEndSeq: var @@ -545,14 +545,14 @@ proc constructObject*[K, V](s: var YamlStream, c: ConstructionContext, value: V event = s.next() if event.kind != yamlStartMap: - raise s.constructionError("Expected map start, got " & $event.kind) + raise s.constructionError(event.startPos, "Expected map start, got " & $event.kind) constructChild(s, c, key) constructChild(s, c, value) event = s.next() if event.kind != yamlEndMap: - raise s.constructionError("Expected map end, got " & $event.kind) + raise s.constructionError(event.startPos, "Expected map end, got " & $event.kind) if result.contains(key): - raise s.constructionError("Duplicate table key!") + raise s.constructionError(event.startPos, "Duplicate table key!") result.add(key, value) discard s.next() @@ -643,9 +643,9 @@ macro matchMatrix(t: typedesc): untyped = result.add(newLit(false)) proc checkDuplicate(s: NimNode, tName: string, name: string, i: int, - matched: NimNode): NimNode {.compileTime.} = + matched: NimNode, m: NimNode): NimNode {.compileTime.} = result = newIfStmt((newNimNode(nnkBracketExpr).add(matched, newLit(i)), - newNimNode(nnkRaiseStmt).add(newCall(bindSym("constructionError"), s, + newNimNode(nnkRaiseStmt).add(newCall(bindSym("constructionError"), s, m, newLit("While constructing " & tName & ": Duplicate field: " & escape(name)))))) @@ -669,7 +669,7 @@ proc getOptionInner(fType: NimNode): NimNode {.compileTime.} = else: return nil proc checkMissing(s: NimNode, t: NimNode, tName: string, field: NimNode, - i: int, matched, o: NimNode): + i: int, matched, o: NimNode, m: NimNode): NimNode {.compileTime.} = let fType = getTypeInst(field) @@ -683,7 +683,7 @@ proc checkMissing(s: NimNode, t: NimNode, tName: string, field: NimNode, elif hasSparse(`t`) and `o`.`field` is Option: `o`.`field` = none(`optionInner`) else: - raise constructionError(`s`, "While constructing " & `tName` & + raise constructionError(`s`, `m`, "While constructing " & `tName` & ": Missing field: " & `fName`) proc markAsFound(i: int, matched: NimNode): NimNode {.compileTime.} = @@ -708,7 +708,7 @@ proc ifNotTransient(o, field: NimNode, `stmts` macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, o: typed, - matched: typed) = + matched: typed, m: Mark) = result = newStmtList() let tDecl = getType(t) @@ -718,7 +718,7 @@ macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, o: typed, for child in tDesc[2].children: if child.kind == nnkRecCase: result.add(checkMissing( - s, t, tName, child[0], field, matched, o)) + s, t, tName, child[0], field, matched, o, m)) for bIndex in 1 .. len(child) - 1: let discChecks = newStmtList() var @@ -735,16 +735,16 @@ macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, o: typed, for item in child[bIndex][recListIndex].recListItems: inc(field) discChecks.add(checkMissing( - s, t, tName, item, field, matched, o)) + s, t, tName, item, field, matched, o, m)) result.add(newIfStmt((infix(newDotExpr(o, newIdentNode($child[0])), "in", curValues), discChecks))) else: - result.add(checkMissing(s, t, tName, child, field, matched, o)) + result.add(checkMissing(s, t, tName, child, field, matched, o, m)) inc(field) macro constructFieldValue(t: typedesc, stream: untyped, context: untyped, name: untyped, o: untyped, - matched: untyped, failOnUnknown: bool) = + matched: untyped, failOnUnknown: bool, m: untyped) = let tDecl = getType(t) tName = $tDecl[1] @@ -770,7 +770,7 @@ macro constructFieldValue(t: typedesc, stream: untyped, objConstr.add(newColonExpr(newIdentNode($otherChild), newDotExpr(o, newIdentNode($otherChild)))) disOb.add(newStmtList( - checkDuplicate(stream, tName, $child[0], fieldIndex, matched), + checkDuplicate(stream, tName, $child[0], fieldIndex, matched, m), newNimNode(nnkVarSection).add( newNimNode(nnkIdentDefs).add( newIdentNode("value"), discType, newEmptyNode())), @@ -808,7 +808,7 @@ macro constructFieldValue(t: typedesc, stream: untyped, infix(newStrLitNode("Field " & $item & " not allowed for " & $child[0] & " == "), "&", prefix(discriminant, "$")))))) ob.add(ifNotTransient(o, item, - [checkDuplicate(stream, tName, $item, fieldIndex, matched), + [checkDuplicate(stream, tName, $item, fieldIndex, matched, m), ifStmt, markAsFound(fieldIndex, matched)], true, stream, tName, $item)) caseStmt.add(ob) @@ -817,7 +817,7 @@ macro constructFieldValue(t: typedesc, stream: untyped, var ob = newNimNode(nnkOfBranch).add(newStrLitNode($child)) let field = newDotExpr(o, newIdentNode($child)) ob.add(ifNotTransient(o, child, - [checkDuplicate(stream, tName, $child, fieldIndex, matched), + [checkDuplicate(stream, tName, $child, fieldIndex, matched, m), newCall("constructChild", stream, context, field), markAsFound(fieldIndex, matched)], true, stream, tName, $child)) caseStmt.add(ob) @@ -825,7 +825,7 @@ macro constructFieldValue(t: typedesc, stream: untyped, caseStmt.add(newNimNode(nnkElse).add(newNimNode(nnkWhenStmt).add( newNimNode(nnkElifBranch).add(failOnUnknown, newNimNode(nnkRaiseStmt).add( - newCall(bindSym("constructionError"), stream, + newCall(bindSym("constructionError"), stream, m, infix(newLit("While constructing " & tName & ": Unknown field: "), "&", newCall(bindSym("escape"), name)))))))) result.add(caseStmt) @@ -859,8 +859,9 @@ proc constructObjectDefault*[O: object|tuple]( startKind = when isVariantObject(getType(O)): yamlStartSeq else: yamlStartMap endKind = when isVariantObject(getType(O)): yamlEndSeq else: yamlEndMap if e.kind != startKind: - raise s.constructionError("While constructing " & + raise s.constructionError(e.startPos, "While constructing " & typetraits.name(O) & ": Expected " & $startKind & ", got " & $e.kind) + let startPos = e.startPos when hasIgnore(O): const ignoredKeyList = O.getCustomPragmaVal(ignore) const failOnUnknown = len(ignoredKeyList) > 0 @@ -870,10 +871,10 @@ proc constructObjectDefault*[O: object|tuple]( e = s.next() when isVariantObject(getType(O)): if e.kind != yamlStartMap: - raise s.constructionError("Expected single-pair map, got " & $e.kind) + raise s.constructionError(e.startPos, "Expected single-pair map, got " & $e.kind) e = s.next() if e.kind != yamlScalar: - raise s.constructionError("Expected field name, got " & $e.kind) + raise s.constructionError(e.startPos, "Expected field name, got " & $e.kind) let name = e.scalarContent when result is tuple: var i = 0 @@ -881,7 +882,7 @@ proc constructObjectDefault*[O: object|tuple]( for fname, value in fieldPairs(result): if fname == name: if matched[i]: - raise s.constructionError("While constructing " & + raise s.constructionError(e.startPos, "While constructing " & typetraits.name(O) & ": Duplicate field: " & escape(name)) constructChild(s, c, value) matched[i] = true @@ -890,12 +891,12 @@ proc constructObjectDefault*[O: object|tuple]( inc(i) when failOnUnknown: if not found: - raise s.constructionError("While constructing " & + raise s.constructionError(e.startPos, "While constructing " & typetraits.name(O) & ": Unknown field: " & escape(name)) else: when hasIgnore(O) and failOnUnknown: if name notin ignoredKeyList: - constructFieldValue(O, s, c, name, result, matched, failOnUnknown) + constructFieldValue(O, s, c, name, result, matched, failOnUnknown, e.startPos) else: e = s.next() var depth = int(e.kind in {yamlStartMap, yamlStartSeq}) @@ -906,21 +907,21 @@ proc constructObjectDefault*[O: object|tuple]( of yamlScalar: discard else: internalError("Unexpected event kind.") else: - constructFieldValue(O, s, c, name, result, matched, failOnUnknown) + constructFieldValue(O, s, c, name, result, matched, failOnUnknown, e.startPos) when isVariantObject(getType(O)): e = s.next() if e.kind != yamlEndMap: - raise s.constructionError("Expected end of single-pair map, got " & + raise s.constructionError(e.startPos, "Expected end of single-pair map, got " & $e.kind) discard s.next() when result is tuple: var i = 0 for fname, value in fieldPairs(result): if not matched[i]: - raise s.constructionError("While constructing " & + raise s.constructionError(e.startPos, "While constructing " & typetraits.name(O) & ": Missing field: " & escape(fname)) inc(i) - else: ensureAllFieldsPresent(s, O, result, matched) + else: ensureAllFieldsPresent(s, O, result, matched, startPos) proc constructObject*[O: object|tuple]( s: var YamlStream, c: ConstructionContext, result: var O) @@ -1001,8 +1002,8 @@ proc representObject*[O: object](value: O, ts: TagStyle, c: SerializationContext, tag: TagId) = ## represents a Nim object or tuple as YAML mapping let childTagStyle = if ts == tsRootOnly: tsNone else: ts - when isVariantObject(getType(O)): c.put(startSeqEvent(tag, yAnchorNone)) - else: c.put(startMapEvent(tag, yAnchorNone)) + when isVariantObject(getType(O)): c.put(startSeqEvent(csBlock, (yAnchorNone, tag))) + else: c.put(startMapEvent(csBlock, (yAnchorNone, tag))) genRepresentObject(O, value, childTagStyle) when isVariantObject(getType(O)): c.put(endSeqEvent()) else: c.put(endMapEvent()) @@ -1025,10 +1026,10 @@ proc constructObject*[O: enum](s: var YamlStream, c: ConstructionContext, ## constructs a Nim enum from a YAML scalar let e = s.next() if e.kind != yamlScalar: - raise s.constructionError("Expected scalar, got " & $e.kind) + raise s.constructionError(e.startPos, "Expected scalar, got " & $e.kind) try: result = parseEnum[O](e.scalarContent) except ValueError: - var ex = s.constructionError("Cannot parse '" & + var ex = s.constructionError(e.startPos, "Cannot parse '" & escape(e.scalarContent) & "' as " & type(O).name) ex.parent = getCurrentException() raise ex @@ -1128,7 +1129,7 @@ proc constructChild*[T](s: var YamlStream, c: ConstructionContext, of yTypeBoolTrue, yTypeBoolFalse: possibleTagIds.add(yamlTag(bool)) of yTypeNull: - raise s.constructionError("not implemented!") + raise s.constructionError(item.startPos, "not implemented!") of yTypeUnknown: possibleTagIds.add(yamlTag(string)) of yTypeTimestamp: @@ -1139,12 +1140,12 @@ proc constructChild*[T](s: var YamlStream, c: ConstructionContext, possibleTagIds.add(item.scalarTag) of yamlStartMap: if item.mapTag in [yTagQuestionMark, yTagExclamationMark]: - raise s.constructionError( + raise s.constructionError(item.startPos, "Complex value of implicit variant object type must have a tag.") possibleTagIds.add(item.mapTag) of yamlStartSeq: if item.seqTag in [yTagQuestionMark, yTagExclamationMark]: - raise s.constructionError( + raise s.constructionError(item.startPos, "Complex value of implicit variant object type must have a tag.") possibleTagIds.add(item.seqTag) else: internalError("Unexpected item kind: " & $item.kind) @@ -1152,21 +1153,21 @@ proc constructChild*[T](s: var YamlStream, c: ConstructionContext, else: case item.kind of yamlScalar: - if item.scalarTag notin [yTagQuestionMark, yTagExclamationMark, + if item.scalarProperties.tag notin [yTagQuestionMark, yTagExclamationMark, yamlTag(T)]: - raise s.constructionError("Wrong tag for " & typetraits.name(T)) - elif item.scalarAnchor != yAnchorNone: - raise s.constructionError("Anchor on non-ref type") + raise s.constructionError(item.startPos, "Wrong tag for " & typetraits.name(T)) + elif item.scalarProperties.anchor != yAnchorNone: + raise s.constructionError(item.startPos, "Anchor on non-ref type") of yamlStartMap: - if item.mapTag notin [yTagQuestionMark, yamlTag(T)]: - raise s.constructionError("Wrong tag for " & typetraits.name(T)) - elif item.mapAnchor != yAnchorNone: - raise s.constructionError("Anchor on non-ref type") + if item.mapProperties.tag notin [yTagQuestionMark, yamlTag(T)]: + raise s.constructionError(item.startPos, "Wrong tag for " & typetraits.name(T)) + elif item.mapProperties.anchor != yAnchorNone: + raise s.constructionError(item.startPos, "Anchor on non-ref type") of yamlStartSeq: - if item.seqTag notin [yTagQuestionMark, yamlTag(T)]: - raise s.constructionError("Wrong tag for " & typetraits.name(T)) - elif item.seqAnchor != yAnchorNone: - raise s.constructionError("Anchor on non-ref type") + if item.seqProperties.tag notin [yTagQuestionMark, yamlTag(T)]: + raise s.constructionError(item.startPos, "Wrong tag for " & typetraits.name(T)) + elif item.seqProperties.anchor != yAnchorNone: + raise s.constructionError(item.startPos, "Anchor on non-ref type") else: internalError("Unexpected item kind: " & $item.kind) constructObject(s, c, result) @@ -1176,19 +1177,19 @@ proc constructChild*(s: var YamlStream, c: ConstructionContext, if item.kind == yamlScalar: if item.scalarProperties.tag notin [yTagQuestionMark, yTagExclamationMark, yamlTag(string)]: - raise s.constructionError("Wrong tag for string") + raise s.constructionError(item.startPos, "Wrong tag for string") elif item.scalarProperties.anchor != yAnchorNone: - raise s.constructionError("Anchor on non-ref type") + raise s.constructionError(item.startPos, "Anchor on non-ref type") constructObject(s, c, result) proc constructChild*[T](s: var YamlStream, c: ConstructionContext, result: var seq[T]) = let item = s.peek() if item.kind == yamlStartSeq: - if item.seqTag notin [yTagQuestionMark, yamlTag(seq[T])]: - raise s.constructionError("Wrong tag for " & typetraits.name(seq[T])) - elif item.seqAnchor != yAnchorNone: - raise s.constructionError("Anchor on non-ref type") + if item.seqProperties.tag notin [yTagQuestionMark, yamlTag(seq[T])]: + raise s.constructionError(item.startPos, "Wrong tag for " & typetraits.name(seq[T])) + elif item.seqProperties.anchor != yAnchorNone: + raise s.constructionError(item.startPos, "Anchor on non-ref type") constructObject(s, c, result) proc constructChild*[T](s: var YamlStream, c: ConstructionContext, @@ -1211,21 +1212,22 @@ when defined(JS): result: var Time) = let e = s.peek() if e.kind == yamlScalar: - if e.scalarTag notin [yTagQuestionMark, yTagTimestamp]: - raise s.constructionError("Wrong tag for Time") + if e.scalarProperties.tag notin [yTagQuestionMark, yTagTimestamp]: + raise s.constructionError(e.startPos, "Wrong tag for Time") elif guessType(e.scalarContent) != yTypeTimestamp: - raise s.constructionError("Invalid timestamp") - elif e.scalarAnchor != yAnchorNone: - raise s.constructionError("Anchor on non-ref type") + raise s.constructionError(e.startPos, "Invalid timestamp") + elif e.scalarProperties.anchor != yAnchorNone: + raise s.constructionError(e.startPos, "Anchor on non-ref type") constructObject(s, c, result) else: - raise s.constructionError("Unexpected structure, expected timestamp") + raise s.constructionError(e.startPos, "Unexpected structure, expected timestamp") proc constructChild*[O](s: var YamlStream, c: ConstructionContext, result: var ref O) = var e = s.peek() if e.kind == yamlScalar: - if e.scalarTag == yTagNull or (e.scalarTag == yTagQuestionMark and + let props = e.scalarProperties + if props.tag == yTagNull or (props.tag == yTagQuestionMark and guessType(e.scalarContent) == yTypeNull): result = nil discard s.next() @@ -1359,11 +1361,15 @@ proc construct*[T](s: var YamlStream, target: var T) var context = newConstructionContext() try: var e = s.next() + yAssert(e.kind == yamlStartStream) + e = s.next() yAssert(e.kind == yamlStartDoc) constructChild(s, context, target) e = s.next() yAssert(e.kind == yamlEndDoc) + e = s.next() + yAssert(e.kind == yamlEndStream) except YamlConstructionError: raise (ref YamlConstructionError)(getCurrentException()) except YamlStreamError: @@ -1430,9 +1436,9 @@ proc represent*[T](value: T, ts: TagStyle = tsRootOnly, if a == asTidy: for item in bys.mitems(): case item.kind - of yamlStartMap: setAnchor(item.mapAnchor, context) - of yamlStartSeq: setAnchor(item.seqAnchor, context) - of yamlScalar: setAnchor(item.scalarAnchor, context) + of yamlStartMap: setAnchor(item.mapProperties.anchor, context) + of yamlStartSeq: setAnchor(item.seqProperties.anchor, context) + of yamlScalar: setAnchor(item.scalarProperties.anchor, context) else: discard result = bys diff --git a/yaml/stream.nim b/yaml/stream.nim index 9ddf6e8..72e989e 100644 --- a/yaml/stream.nim +++ b/yaml/stream.nim @@ -33,8 +33,7 @@ type ## well-formed if they take it as input parameter. nextImpl*: proc(s: YamlStream, e: var Event): bool lastTokenContextImpl*: - proc(s: YamlStream, line, column: var int, - lineContent: var string): bool {.raises: [].} + proc(s: YamlStream, lineContent: var string): bool {.raises: [].} peeked: bool cached: Event @@ -43,13 +42,11 @@ type ## backend raises an exception. The error that has occurred is ## available from ``parent``. -proc noLastContext(s: YamlStream, line, column: var int, - lineContent: var string): bool {.raises: [].} = - (line, column, lineContent) = (-1, -1, "") +proc noLastContext(s: YamlStream, lineContent: var string): bool {.raises: [].} = result = false proc basicInit*(s: YamlStream, lastTokenContextImpl: - proc(s: YamlStream, line, column: var int, lineContent: var string): bool + proc(s: YamlStream, lineContent: var string): bool {.raises: [].} = noLastContext) {.raises: [].} = ## initialize basic values of the YamlStream. Call this in your constructor ## if you subclass YamlStream. @@ -121,12 +118,11 @@ proc `peek=`*(s: YamlStream, value: Event) {.raises: [].} = s.cached = value s.peeked = true -proc getLastTokenContext*(s: YamlStream, line, column: var int, - lineContent: var string): bool = +proc getLastTokenContext*(s: YamlStream, lineContent: var string): bool = ## ``true`` if source context information is available about the last returned ## token. If ``true``, line, column and lineContent are set to position and ## line content where the last token has been read from. - result = s.lastTokenContextImpl(s, line, column, lineContent) + result = s.lastTokenContextImpl(s, lineContent) iterator items*(s: YamlStream): Event {.raises: [YamlStreamError].} = diff --git a/yaml/taglib.nim b/yaml/taglib.nim index 5e11899..f90195f 100644 --- a/yaml/taglib.nim +++ b/yaml/taglib.nim @@ -30,16 +30,21 @@ type ## `initExtendedTagLibrary <#initExtendedTagLibrary>`_. tags*: Table[string, TagId] nextCustomTagId*: TagId - tagHandles: Table[string, string] + prefixes: seq[tuple[prefix, uri: string]] proc initTagLibrary*(): TagLibrary {.raises: [].} = ## initializes the ``tags`` table and sets ``nextCustomTagId`` to ## ``yFirstCustomTagId``. new(result) result.tags = initTable[string, TagId]() - result.tagHandles = {"!": "!", yamlTagRepositoryPrefix : "!!"}.toTable() + result.prefixes = @[("!", "!"), ("!!", yamlTagRepositoryPrefix)] result.nextCustomTagId = yFirstCustomTagId +proc resetPrefixes*(tagLib: TagLibrary) {.raises: [].} = + ## resets the registered prefixes in the given tag library, so that it + ## only contains the prefixes `!` and `!!`. + tagLib.prefixes.setLen(2) + proc registerUri*(tagLib: TagLibrary, uri: string): TagId {.raises: [].} = ## registers a custom tag URI with a ``TagLibrary``. The URI will get ## the ``TagId`` ``nextCustomTagId``, which will be incremented. @@ -103,7 +108,7 @@ proc initExtendedTagLibrary*(): TagLibrary {.raises: [].} = proc initSerializationTagLibrary*(): TagLibrary = result = initTagLibrary() - result.tagHandles[nimyamlTagRepositoryPrefix] = "!n!" + result.prefixes.add(("!n!", nimyamlTagRepositoryPrefix)) result.tags["!"] = yTagExclamationMark result.tags["?"] = yTagQuestionMark result.tags[y"str"] = yTagString @@ -190,11 +195,19 @@ setTagUri(uint64, n"system:uint64", yTagNimUInt64) setTagUri(float32, n"system:float32", yTagNimFloat32) setTagUri(float64, n"system:float64", yTagNimFloat64) -proc registerHandle*(tagLib: TagLibrary, handle, prefix: string) = +proc registerHandle*(tagLib: TagLibrary, uri, prefix: string): bool = ## Registers a handle for a prefix. When presenting any tag that starts with ## this prefix, the handle is used instead. Also causes the presenter to ## output a TAG directive for the handle. - taglib.tagHandles[prefix] = handle + ## + ## Returns true iff a new item has been created, false if an existing item + ## has been updated. + for i in countup(0, len(tagLib.prefixes)-1): + if tagLib.prefixes[i].prefix == prefix: + tagLib.prefixes[i].uri = uri + return false + taglib.prefixes.add((prefix, uri)) + return false proc searchHandle*(tagLib: TagLibrary, tag: string): tuple[handle: string, len: int] {.raises: [].} = @@ -203,22 +216,25 @@ proc searchHandle*(tagLib: TagLibrary, tag: string): ## longest prefix is returned. If no registered handle matches, (nil, 0) is ## returned. result.len = 0 - for key, value in tagLib.tagHandles: - if key.len > result.len: - if tag.startsWith(key): - result.len = key.len - result.handle = value + for item in tagLib.prefixes: + if item.uri.len > result.len: + if tag.startsWith(item.uri): + result.len = item.uri.len + result.handle = item.prefix proc resolve*(tagLib: TagLibrary, handle: string): string {.raises: [].} = ## try to resolve the given tag handle. ## return the registered URI if the tag handle is found. ## if the handle is unknown, return the empty string. - return tagLib.tagHandles.getOrDefault(handle, "") + for item in tagLib.prefixes: + if item.prefix == handle: + return item.uri + return "" -iterator handles*(tagLib: TagLibrary): tuple[prefix, handle: string] = +iterator handles*(tagLib: TagLibrary): tuple[prefix, uri: string] = ## iterate over registered tag handles that may be used as shortcuts ## (e.g. ``!n!`` for ``tag:nimyaml.org,2016:``) - for key, value in tagLib.tagHandles: yield (key, value) + for item in tagLib.prefixes.items(): yield item proc nimTag*(suffix: string): string = ## prepends NimYAML's tag repository prefix to the given suffix. For example,