status-im
/
NimYAML
mirror of https://github.com/status-im/NimYAML.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

lots of fixes for parser, started updating serialization

This commit is contained in:
Felix Krause 2020-11-05 20:23:42 +01:00
parent ae4c097a25
commit e2f8e6419e
8 changed files with 433 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
test/testEventParser.nim
View File

@ -10,7 +10,8 @@ import lexbase, streams, tables, strutils
type type
LexerToken = enum LexerToken = enum
plusStr, minusStr, plusDoc, minusDoc, plusMap, minusMap, plusSeq, minusSeq, plusStr, minusStr, plusDoc, minusDoc, plusMap, minusMap, plusSeq, minusSeq,
eqVal, eqAli, chevTag, andAnchor, starAnchor, quotContent, colonContent, eqVal, eqAli, chevTag, andAnchor, starAnchor, colonContent, sqContent,
dqContent, litContent, foContent,
explDirEnd, explDocEnd, noToken explDirEnd, explDocEnd, noToken
StreamPos = enum StreamPos = enum
@ -31,7 +32,14 @@ proc nextToken(lex: var EventLexer): LexerToken =
if lex.buf[lex.bufpos] == EndOfFile: return noToken if lex.buf[lex.bufpos] == EndOfFile: return noToken
case lex.buf[lex.bufpos] case lex.buf[lex.bufpos]
of ':', '"', '\'', '|', '>': of ':', '"', '\'', '|', '>':
let t = if lex.buf[lex.bufpos] == ':': colonContent else: quotContent let t = case lex.buf[lex.bufpos]
of ':': colonContent
of '"': dqContent
of '\'': sqContent
of '|': litContent
of '>': foContent
else: colonContent
lex.content = "" lex.content = ""
lex.bufpos.inc() lex.bufpos.inc()
while true: while true:
@ -226,19 +234,30 @@ proc parseEventStream*(input: Stream, tagLib: TagLibrary): YamlStream =
escape(lex.content)) escape(lex.content))
else: else:
curEvent.aliasTarget = lex.content.Anchor curEvent.aliasTarget = lex.content.Anchor
of quotContent:
assertInEvent("scalar content")
if curTag() == yTagQuestionMark: setCurTag(yTagExclamationMark)
if curEvent.kind != yamlScalar:
raise newException(EventStreamError,
"scalar content in non-scalar tag")
curEvent.scalarContent = lex.content
of colonContent: of colonContent:
assertInEvent("scalar content") assertInEvent("scalar content")
curEvent.scalarContent = lex.content curEvent.scalarContent = lex.content
if curEvent.kind != yamlScalar: if curEvent.kind != yamlScalar:
raise newException(EventStreamError, raise newException(EventStreamError,
"scalar content in non-scalar tag") "scalar content in non-scalar tag")
of sqContent:
assertInEvent("scalar content")
curEvent.scalarContent = lex.content
if curTag() == yTagQuestionMark: setCurTag(yTagExclamationMark)
curEvent.scalarStyle = ssSingleQuoted
of dqContent:
assertInEvent("scalar content")
curEvent.scalarContent = lex.content
if curTag() == yTagQuestionMark: setCurTag(yTagExclamationMark)
curEvent.scalarStyle = ssDoubleQuoted
of litContent:
assertInEvent("scalar content")
curEvent.scalarContent = lex.content
curEvent.scalarStyle = ssLiteral
of foContent:
assertInEvent("scalar content")
curEvent.scalarContent = lex.content
curEvent.scalarStyle = ssFolded
of explDirEnd: of explDirEnd:
assertInEvent("explicit directives end") assertInEvent("explicit directives end")
if curEvent.kind != yamlStartDoc: if curEvent.kind != yamlStartDoc:

11
test/tlex.nim
View File

@ -193,12 +193,23 @@ suite "Lexer":
assertEquals("top6: \l &anchor6 'key6' : scalar6", i(0), pl("top6"), mv(), assertEquals("top6: \l &anchor6 'key6' : scalar6", i(0), pl("top6"), mv(),
i(2), an("anchor6"), sq("key6"), mv(), pl("scalar6"), e()) i(2), an("anchor6"), sq("key6"), mv(), pl("scalar6"), e())
test "adjacent anchors":
assertEquals("foo: &a\n &b bar", i(0), pl("foo"), mv(), an("a"), i(2),
an("b"), pl("bar"), e())
test "comment at empty key/value pair":
assertEquals(": # foo\nbar:", i(0), mv(), i(0), pl("bar"), mv(), e())
test "Map in Sequence": test "Map in Sequence":
assertEquals("""- assertEquals("""-
a: b a: b
c: d c: d
""", i(0), si(), i(2), pl("a"), mv(), pl("b"), i(2), pl("c"), mv(), pl("d"), e()) """, i(0), si(), i(2), pl("a"), mv(), pl("b"), i(2), pl("c"), mv(), pl("d"), e())
test "dir end after multiline scalar":
assertEquals("foo:\n bar\n baz\n---\nderp", i(0), pl("foo"), mv(), i(2),
pl("bar baz"), dirE(), i(0), pl("derp"), e())
test "Empty lines": test "Empty lines":
assertEquals("""block: foo assertEquals("""block: foo

11
test/tparser.nim
View File

@ -74,8 +74,15 @@ macro genTests(): untyped =
let errorTests = toHashSet(staticExec("cd " & (absolutePath / "tags" / "error") & let errorTests = toHashSet(staticExec("cd " & (absolutePath / "tags" / "error") &
" && ls -1d *").splitLines()) " && ls -1d *").splitLines())
let ignored = toHashSet(["3MYT", "JDH8", "2EBW", "9KAX", "AB8U", "B63P", "FBC9", var ignored = toHashSet([".git", "name", "tags", "meta"])
"Q5MG", "S98Z", ".git", "name", "tags", "meta"]) #-----------------------------------------------------------------------------
# THE FOLLOWING TESTS WOULD FAIL FOR THE DOCUMENTED REASONS
ignored.incl("W5VH")
# YAML allows the colon as part of an anchor or alias name.
# For aliases, this leads to confusion becaues `*a:` looks like an implicit
# mapping key (but is not).
# Therefore, NimYAML disallows colons in anchor names.
#-----------------------------------------------------------------------------
result = newStmtList() result = newStmtList()
# walkDir for some crude reason does not work with travis build # walkDir for some crude reason does not work with travis build

410
yaml/parser.nim
View File

@ -120,11 +120,43 @@ proc afterFlowSeqItem(c: Context, e: var Event): bool
proc afterPairValue(c: Context, e: var Event): bool proc afterPairValue(c: Context, e: var Event): bool
{.pop.} {.pop.}
template debug(message: string) {.dirty.} =
when defined(yamlDebug):
try: styledWriteLine(stdout, fgBlue, message)
except ValueError, IOError: discard
template pushLevel(c: Context, newState: State, newIndent: int) =
debug("parser: push " & newState.astToStr & ", indent = " & $newIndent)
c.levels.add(Level(state: newState, indentation: newIndent))
template pushLevel(c: Context, newState: State) =
debug("parser: push " & newState.astToStr)
c.levels.add(Level(state: newState))
template transition(c: Context, newState: State) =
debug("parser: transition " & newState.astToStr)
c.levels[^1].state = newState
template transition(c: Context, newState: State, newIndent) =
debug("parser: transtion " & newState.astToStr & ", indent = " & $newIndent)
c.levels[^1] = Level(state: newState, indentation: newIndent)
template updateIndentation(c: Context, newIndent: int) =
debug("parser: update indent = " & $newIndent)
c.levels[^1].indentation = newIndent
template popLevel(c: Context) =
debug("parser: pop")
discard c.levels.pop()
proc init[T](c: Context, p: YamlParser, source: T) {.inline.} = proc init[T](c: Context, p: YamlParser, source: T) {.inline.} =
c.levels.add(Level(state: atStreamStart, indentation: -2)) c.pushLevel(atStreamStart, -2)
c.nextImpl = proc(s: YamlStream, e: var Event): bool = c.nextImpl = proc(s: YamlStream, e: var Event): bool =
let c = Context(s) let c = Context(s)
return c.levels[^1].state(c, e) return c.levels[^1].state(c, e)
c.lastTokenContextImpl = proc(s: YamlStream, lineContent: var string): bool =
lineContent = Context(s).lex.currentLine()
return true
c.headerProps = defaultProperties c.headerProps = defaultProperties
c.inlineProps = defaultProperties c.inlineProps = defaultProperties
c.tagLib = p.tagLib c.tagLib = p.tagLib
@ -151,11 +183,6 @@ proc parse*(p: YamlParser, s: string): YamlStream =
# implementation # implementation
template debug(message: string) {.dirty.} =
when defined(yamlDebug):
try: styledWriteLine(stdout, fgBlue, message)
except IOError: discard
proc isEmpty(props: Properties): bool = proc isEmpty(props: Properties): bool =
result = props.anchor == yAnchorNone and result = props.anchor == yAnchorNone and
props.tag == yTagQuestionMark props.tag == yTagQuestionMark
@ -196,10 +223,11 @@ proc autoScalarTag(props: Properties, t: Token): Properties =
result.tag = yTagExclamationMark result.tag = yTagExclamationMark
proc atStreamStart(c: Context, e: var Event): bool = proc atStreamStart(c: Context, e: var Event): bool =
c.levels[0] = Level(state: atStreamEnd, indentation: -2) c.transition(atStreamEnd)
c.levels.add(Level(state: beforeDoc, indentation: -1)) c.pushLevel(beforeDoc, -1)
e = Event(startPos: c.lex.curStartPos, endPos: c.lex.curStartPos, kind: yamlStartStream) e = Event(startPos: c.lex.curStartPos, endPos: c.lex.curStartPos, kind: yamlStartStream)
c.lex.next() c.lex.next()
c.tagLib.resetPrefixes()
return true return true
proc atStreamEnd(c: Context, e : var Event): bool = proc atStreamEnd(c: Context, e : var Event): bool =
@ -219,16 +247,16 @@ proc beforeDoc(c: Context, e: var Event): bool =
of DirectivesEnd: of DirectivesEnd:
e = startDocEvent(true, version, c.lex.curStartPos, c.lex.curEndPos) e = startDocEvent(true, version, c.lex.curStartPos, c.lex.curEndPos)
c.lex.next() c.lex.next()
c.levels[1].state = beforeDocEnd c.transition(beforeDocEnd)
c.levels.add(Level(state: afterDirectivesEnd, indentation: -1)) c.pushLevel(afterDirectivesEnd, -1)
return true return true
of StreamEnd: of StreamEnd:
discard c.levels.pop() c.popLevel()
return false return false
of Indentation: of Indentation:
e = startDocEvent(false, version, c.lex.curStartPos, c.lex.curEndPos) e = startDocEvent(false, version, c.lex.curStartPos, c.lex.curEndPos)
c.levels[^1].state = beforeDocEnd c.transition(beforeDocEnd)
c.levels.add(Level(state: beforeImplicitRoot, indentation: -1)) c.pushLevel(beforeImplicitRoot, -1)
return true return true
of YamlDirective: of YamlDirective:
seenDirectives = true seenDirectives = true
@ -250,7 +278,7 @@ proc beforeDoc(c: Context, e: var Event): bool =
c.lex.next() c.lex.next()
if c.lex.cur != Token.Suffix: if c.lex.cur != Token.Suffix:
raise c.generateError("Invalid token (expected tag URI): " & $c.lex.cur) raise c.generateError("Invalid token (expected tag URI): " & $c.lex.cur)
c.tagLib.registerHandle(tagHandle, c.lex.fullLexeme()) discard c.tagLib.registerHandle(c.lex.fullLexeme(), tagHandle)
c.lex.next() c.lex.next()
of UnknownDirective: of UnknownDirective:
seenDirectives = true seenDirectives = true
@ -265,20 +293,23 @@ proc afterDirectivesEnd(c: Context, e: var Event): bool =
case c.lex.cur case c.lex.cur
of TagHandle, VerbatimTag, Token.Anchor: of TagHandle, VerbatimTag, Token.Anchor:
c.inlineStart = c.lex.curStartPos c.inlineStart = c.lex.curStartPos
c.levels.add(Level(state: beforeNodeProperties)) c.pushLevel(beforeNodeProperties)
return false return false
of Indentation: of Indentation:
c.headerStart = c.inlineStart c.headerStart = c.inlineStart
c.levels[^1].state = atBlockIndentation c.transition(atBlockIndentation)
c.levels.add(Level(state: beforeBlockIndentation)) c.pushLevel(beforeBlockIndentation)
return false return false
of DocumentEnd: of DocumentEnd:
e = scalarEvent("", c.inlineProps, ssPlain, c.lex.curStartPos, c.lex.curEndPos) e = scalarEvent("", c.inlineProps, ssPlain, c.lex.curStartPos, c.lex.curEndPos)
c.popLevel()
return true return true
of Folded, Literal: of Folded, Literal:
e = scalarEvent(c.lex.evaluated, c.inlineProps, e = scalarEvent(c.lex.evaluated, c.inlineProps,
if c.lex.cur == Token.Folded: ssFolded else: ssLiteral, if c.lex.cur == Token.Folded: ssFolded else: ssLiteral,
c.lex.curStartPos, c.lex.curEndPos) c.lex.curStartPos, c.lex.curEndPos)
c.popLevel()
c.lex.next()
return true return true
else: else:
raise c.generateError("Illegal content at `---`: " & $c.lex.cur) raise c.generateError("Illegal content at `---`: " & $c.lex.cur)
@ -287,26 +318,26 @@ proc beforeImplicitRoot(c: Context, e: var Event): bool =
if c.lex.cur != Token.Indentation: if c.lex.cur != Token.Indentation:
raise c.generateError("Unexpected token (expected line start): " & $c.lex.cur) raise c.generateError("Unexpected token (expected line start): " & $c.lex.cur)
c.inlineStart = c.lex.curEndPos c.inlineStart = c.lex.curEndPos
c.levels[^1].indentation = c.lex.recentIndentation() c.updateIndentation(c.lex.recentIndentation())
c.lex.next() c.lex.next()
case c.lex.cur case c.lex.cur
of SeqItemInd, MapKeyInd, MapValueInd: of SeqItemInd, MapKeyInd, MapValueInd:
c.levels[^1].state = afterCompactParent c.transition(afterCompactParent)
return false return false
of scalarTokenKind: of scalarTokenKind:
c.levels[^1].state = requireImplicitMapStart c.transition(requireImplicitMapStart)
return false return false
of nodePropertyKind: of nodePropertyKind:
c.levels[^1].state = requireImplicitMapStart c.transition(requireImplicitMapStart)
c.levels.add(Level(state: beforeNodeProperties, indentation: 0)) c.pushLevel(beforeNodeProperties)
of MapStart, SeqStart: of MapStart, SeqStart:
c.levels[^1].state = afterCompactParentProps c.transition(afterCompactParentProps)
return false return false
else: else:
raise c.generateError("Unexpected token (expected collection start): " & $c.lex.cur) raise c.generateError("Unexpected token (expected collection start): " & $c.lex.cur)
proc requireImplicitMapStart(c: Context, e: var Event): bool = proc requireImplicitMapStart(c: Context, e: var Event): bool =
c.levels[^1].indentation = c.lex.recentIndentation() c.updateIndentation(c.lex.recentIndentation())
case c.lex.cur case c.lex.cur
of Alias: of Alias:
e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos)
@ -316,11 +347,11 @@ proc requireImplicitMapStart(c: Context, e: var Event): bool =
c.peek = e c.peek = e
e = startMapEvent(csBlock, c.headerProps, c.headerStart, headerEnd) e = startMapEvent(csBlock, c.headerProps, c.headerStart, headerEnd)
c.headerProps = defaultProperties c.headerProps = defaultProperties
c.levels[^1].state = afterImplicitKey c.transition(afterImplicitKey)
else: else:
if not isEmpty(c.headerProps): if not isEmpty(c.headerProps):
raise c.generateError("Alias may not have properties") raise c.generateError("Alias may not have properties")
discard c.levels.pop() c.popLevel()
return true return true
of Plain, SingleQuoted, DoubleQuoted: of Plain, SingleQuoted, DoubleQuoted:
e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur), e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur),
@ -336,13 +367,18 @@ proc requireImplicitMapStart(c: Context, e: var Event): bool =
e = startMapEvent(csBlock, c.headerProps, e = startMapEvent(csBlock, c.headerProps,
c.headerStart, headerEnd) c.headerStart, headerEnd)
c.headerProps = defaultProperties c.headerProps = defaultProperties
c.levels[^1].state = afterImplicitKey c.transition(afterImplicitKey)
of Indentation, DocumentEnd, DirectivesEnd, StreamEnd: else: c.popLevel()
raise c.generateError("Scalar at root level requires `---`") return true
else: discard of Literal, Folded:
e = scalarEvent(c.lex.evaluated, c.inlineProps, toStyle(c.lex.cur),
c.inlineStart, c.lex.curEndPos)
c.inlineProps = defaultProperties
c.lex.next()
c.popLevel()
return true return true
of MapStart, SeqStart: of MapStart, SeqStart:
c.levels[^1].state = beforeFlowItemProps c.transition(beforeFlowItemProps)
return false return false
of Indentation: of Indentation:
raise c.generateError("Standalone node properties not allowed on non-header line") raise c.generateError("Standalone node properties not allowed on non-header line")
@ -353,41 +389,42 @@ proc atBlockIndentation(c: Context, e: var Event): bool =
if c.blockIndentation == c.levels[^1].indentation and if c.blockIndentation == c.levels[^1].indentation and
(c.lex.cur != Token.SeqItemInd or (c.lex.cur != Token.SeqItemInd or
c.levels[^3].state == inBlockSeq): c.levels[^3].state == inBlockSeq):
e = scalarEvent(c.lex.evaluated, c.headerProps, ssPlain, e = scalarEvent("", c.headerProps, ssPlain,
c.headerStart, c.headerStart) c.headerStart, c.headerStart)
c.headerProps = defaultProperties c.headerProps = defaultProperties
discard c.levels.pop() c.popLevel()
discard c.levels.pop() c.popLevel()
return true return true
c.inlineStart = c.lex.curStartPos c.inlineStart = c.lex.curStartPos
c.levels[^1].indentation = c.lex.recentIndentation() c.updateIndentation(c.lex.recentIndentation())
case c.lex.cur case c.lex.cur
of nodePropertyKind: of nodePropertyKind:
if isEmpty(c.headerProps): if isEmpty(c.headerProps):
c.levels[^1].state = requireInlineBlockItem c.transition(requireInlineBlockItem)
else: else:
c.levels[^1].state = requireImplicitMapStart c.transition(requireImplicitMapStart)
c.levels.add(Level(state: beforeNodeProperties)) c.pushLevel(beforeNodeProperties)
return false return false
of SeqItemInd: of SeqItemInd:
e = startSeqEvent(csBlock, c.headerProps, e = startSeqEvent(csBlock, c.headerProps,
c.headerStart, c.lex.curEndPos) c.headerStart, c.lex.curEndPos)
c.headerProps = defaultProperties c.headerProps = defaultProperties
c.levels[^1] = Level(state: inBlockSeq, indentation: c.lex.recentIndentation()) c.transition(inBlockSeq, c.lex.recentIndentation())
c.levels.add(Level(state: beforeBlockIndentation, indentation: 0)) c.pushLevel(beforeBlockIndentation)
c.levels.add(Level(state: afterCompactParent, indentation: c.lex.recentIndentation())) c.pushLevel(afterCompactParent, c.lex.recentIndentation())
c.lex.next() c.lex.next()
return true return true
of MapKeyInd: of MapKeyInd:
e = startMapEvent(csBlock, c.headerProps, e = startMapEvent(csBlock, c.headerProps,
c.headerStart, c.lex.curEndPos) c.headerStart, c.lex.curEndPos)
c.headerProps = defaultProperties c.headerProps = defaultProperties
c.levels[^1] = Level(state: beforeBlockMapValue, indentation: c.lex.recentIndentation()) c.transition(beforeBlockMapValue, c.lex.recentIndentation())
c.levels.add(Level(state: beforeBlockIndentation)) c.pushLevel(beforeBlockIndentation)
c.levels.add(Level(state: afterCompactParent, indentation: c.lex.recentIndentation())) c.pushLevel(afterCompactParent, c.lex.recentIndentation())
c.lex.next() c.lex.next()
return true
of Plain, SingleQuoted, DoubleQuoted: of Plain, SingleQuoted, DoubleQuoted:
c.levels[^1].indentation = c.lex.recentIndentation() c.updateIndentation(c.lex.recentIndentation())
let scalarToken = c.lex.cur let scalarToken = c.lex.cur
e = scalarEvent(c.lex.evaluated, c.headerProps, e = scalarEvent(c.lex.evaluated, c.headerProps,
toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos)
@ -401,10 +438,10 @@ proc atBlockIndentation(c: Context, e: var Event): bool =
e.scalarProperties = autoScalarTag(defaultProperties, scalarToken) e.scalarProperties = autoScalarTag(defaultProperties, scalarToken)
c.peek = move(e) c.peek = move(e)
e = startMapEvent(csBlock, props, c.headerStart, headerEnd) e = startMapEvent(csBlock, props, c.headerStart, headerEnd)
c.levels[^1].state = afterImplicitKey c.transition(afterImplicitKey)
else: else:
e.scalarProperties = autoScalarTag(e.scalarProperties, scalarToken) e.scalarProperties = autoScalarTag(e.scalarProperties, scalarToken)
discard c.levels.pop() c.popLevel()
return true return true
of Alias: of Alias:
e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos)
@ -415,24 +452,25 @@ proc atBlockIndentation(c: Context, e: var Event): bool =
c.peek = move(e) c.peek = move(e)
e = startMapEvent(csBlock, c.headerProps, c.headerStart, headerEnd) e = startMapEvent(csBlock, c.headerProps, c.headerStart, headerEnd)
c.headerProps = defaultProperties c.headerProps = defaultProperties
c.levels[^1].state = afterImplicitKey c.transition(afterImplicitKey)
elif not isEmpty(c.headerProps): elif not isEmpty(c.headerProps):
raise c.generateError("Alias may not have properties") raise c.generateError("Alias may not have properties")
else: else:
discard c.levels.pop() c.popLevel()
return true return true
else: else:
c.levels[^1].state = atBlockIndentationProps c.transition(atBlockIndentationProps)
return false
proc atBlockIndentationProps(c: Context, e: var Event): bool = proc atBlockIndentationProps(c: Context, e: var Event): bool =
c.levels[^1].indentation = c.lex.recentIndentation() c.updateIndentation(c.lex.recentIndentation())
case c.lex.cur case c.lex.cur
of MapValueInd: of MapValueInd:
c.peek = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curEndPos) c.peek = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curEndPos)
c.inlineProps = defaultProperties c.inlineProps = defaultProperties
e = startMapEvent(csBlock, c.headerProps, c.lex.curStartPos, c.lex.curEndPos) e = startMapEvent(csBlock, c.headerProps, c.lex.curStartPos, c.lex.curEndPos)
c.headerProps = defaultProperties c.headerProps = defaultProperties
c.levels[^1].state = afterImplicitKey c.transition(afterImplicitKey)
return true return true
of Plain, SingleQuoted, DoubleQuoted: of Plain, SingleQuoted, DoubleQuoted:
e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur), e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur),
@ -446,20 +484,20 @@ proc atBlockIndentationProps(c: Context, e: var Event): bool =
c.peek = move(e) c.peek = move(e)
e = startMapEvent(csBlock, c.headerProps, c.headerStart, headerEnd) e = startMapEvent(csBlock, c.headerProps, c.headerStart, headerEnd)
c.headerProps = defaultProperties c.headerProps = defaultProperties
c.levels[^1].state = afterImplicitKey c.transition(afterImplicitKey)
else: else:
discard c.levels.pop() c.popLevel()
return true return true
of MapStart: of MapStart:
e = startMapEvent(csFlow, c.headerProps, c.headerStart, c.lex.curEndPos) e = startMapEvent(csFlow, c.headerProps, c.headerStart, c.lex.curEndPos)
c.headerProps = defaultProperties c.headerProps = defaultProperties
c.levels[^1].state = afterFlowMapSep c.transition(afterFlowMapSep)
c.lex.next() c.lex.next()
return true return true
of SeqStart: of SeqStart:
e = startSeqEvent(csFlow, c.headerProps, c.headerStart, c.lex.curEndPos) e = startSeqEvent(csFlow, c.headerProps, c.headerStart, c.lex.curEndPos)
c.headerProps = defaultProperties c.headerProps = defaultProperties
c.levels[^1].state = afterFlowSeqSep c.transition(afterFlowSeqSep)
c.lex.next() c.lex.next()
return true return true
else: else:
@ -485,12 +523,12 @@ proc beforeNodeProperties(c: Context, e: var Event): bool =
of Indentation: of Indentation:
c.headerProps = c.inlineProps c.headerProps = c.inlineProps
c.inlineProps = defaultProperties c.inlineProps = defaultProperties
discard c.levels.pop() c.popLevel()
return false return false
of Alias: of Alias:
raise c.generateError("Alias may not have node properties") raise c.generateError("Alias may not have node properties")
else: else:
discard c.levels.pop() c.popLevel()
return false return false
c.lex.next() c.lex.next()
return false return false
@ -499,49 +537,49 @@ proc afterCompactParent(c: Context, e: var Event): bool =
c.inlineStart = c.lex.curStartPos c.inlineStart = c.lex.curStartPos
case c.lex.cur case c.lex.cur
of nodePropertyKind: of nodePropertyKind:
c.levels[^1].state = afterCompactParentProps c.transition(afterCompactParentProps)
c.levels.add(Level(state: beforeNodeProperties)) c.pushLevel(beforeNodeProperties)
of SeqItemInd: of SeqItemInd:
e = startSeqEvent(csBlock, c.headerProps, c.headerStart, c.lex.curEndPos) e = startSeqEvent(csBlock, c.headerProps, c.headerStart, c.lex.curEndPos)
c.headerProps = defaultProperties c.headerProps = defaultProperties
c.levels[^1] = Level(state: inBlockSeq, indentation: c.lex.recentIndentation()) c.transition(inBlockSeq, c.lex.recentIndentation())
echo "started seq at indentation ", c.lex.recentIndentation() c.pushLevel(beforeBlockIndentation)
c.levels.add(Level(state: beforeBlockIndentation)) c.pushLevel(afterCompactParent, c.lex.recentIndentation())
c.levels.add(Level(state: afterCompactParent))
c.lex.next() c.lex.next()
return true return true
of MapKeyInd: of MapKeyInd:
e = startMapEvent(csBlock, c.headerProps, c.headerStart, c.lex.curEndPos) e = startMapEvent(csBlock, c.headerProps, c.headerStart, c.lex.curEndPos)
c.headerProps = defaultProperties c.headerProps = defaultProperties
c.levels[^1] = Level(state: beforeBlockMapValue, indentation: c.lex.recentIndentation()) c.transition(beforeBlockMapValue, c.lex.recentIndentation())
c.levels.add(Level(state: beforeBlockIndentation)) c.pushLevel(beforeBlockIndentation)
c.levels.add(Level(state: afterCompactParent)) c.pushLevel(afterCompactParent, c.lex.recentIndentation)
c.lex.next()
return true return true
else: else:
c.levels[^1].state = afterCompactParentProps c.transition(afterCompactParentProps)
return false return false
proc afterCompactParentProps(c: Context, e: var Event): bool = proc afterCompactParentProps(c: Context, e: var Event): bool =
c.levels[^1].indentation = c.lex.recentIndentation() c.updateIndentation(c.lex.recentIndentation())
case c.lex.cur case c.lex.cur
of nodePropertyKind: of nodePropertyKind:
c.levels.add(Level(state: beforeNodeProperties)) c.pushLevel(beforeNodeProperties)
return false return false
of Indentation: of Indentation:
c.headerStart = c.inlineStart c.headerStart = c.inlineStart
c.levels[^1] = Level(state: atBlockIndentation, indentation: c.levels[^3].indentation) c.transition(atBlockIndentation, c.levels[^3].indentation)
c.levels.add(Level(state: beforeBlockIndentation)) c.pushLevel(beforeBlockIndentation)
return false return false
of StreamEnd, DocumentEnd, DirectivesEnd: of StreamEnd, DocumentEnd, DirectivesEnd:
e = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos) e = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos)
c.inlineProps = defaultProperties c.inlineProps = defaultProperties
discard c.levels.pop() c.popLevel()
return true return true
of MapValueInd: of MapValueInd:
c.peek = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos) c.peek = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos)
c.inlineProps = defaultProperties c.inlineProps = defaultProperties
e = startMapEvent(csBlock, defaultProperties, c.lex.curStartPos, c.lex.curStartPos) e = startMapEvent(csBlock, defaultProperties, c.lex.curStartPos, c.lex.curStartPos)
c.levels[^1].state = afterImplicitKey c.transition(afterImplicitKey)
return true return true
of Alias: of Alias:
e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos)
@ -550,36 +588,36 @@ proc afterCompactParentProps(c: Context, e: var Event): bool =
if c.lex.cur == Token.MapValueInd: if c.lex.cur == Token.MapValueInd:
c.peek = move(e) c.peek = move(e)
e = startMapEvent(csBlock, defaultProperties, headerEnd, headerEnd) e = startMapEvent(csBlock, defaultProperties, headerEnd, headerEnd)
c.levels[^1].state = afterImplicitKey c.transition(afterImplicitKey)
else: else:
discard c.levels.pop() c.popLevel()
return true return true
of scalarTokenKind: of scalarTokenKind:
e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur), e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur),
toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos)
c.inlineProps = defaultProperties c.inlineProps = defaultProperties
let headerEnd = c.lex.curStartPos let headerEnd = c.lex.curStartPos
c.levels[^1].indentation = c.lex.recentIndentation() c.updateIndentation(c.lex.recentIndentation())
c.lex.next() c.lex.next()
if c.lex.cur == Token.MapValueInd: if c.lex.cur == Token.MapValueInd:
if c.lex.lastScalarWasMultiline(): if c.lex.lastScalarWasMultiline():
raise c.generateError("Implicit mapping key may not be multiline") raise c.generateError("Implicit mapping key may not be multiline")
c.peek = move(e) c.peek = move(e)
e = startMapEvent(csBlock, defaultProperties, headerEnd, headerEnd) e = startMapEvent(csBlock, defaultProperties, headerEnd, headerEnd)
c.levels[^1].state = afterImplicitKey c.transition(afterImplicitKey)
else: else:
discard c.levels.pop() c.popLevel()
return true return true
of MapStart: of MapStart:
e = startMapEvent(csFlow, c.inlineProps, c.inlineStart, c.lex.curEndPos) e = startMapEvent(csFlow, c.inlineProps, c.inlineStart, c.lex.curEndPos)
c.inlineProps = defaultProperties c.inlineProps = defaultProperties
c.levels[^1].state = afterFlowMapSep c.transition(afterFlowMapSep)
c.lex.next() c.lex.next()
return true return true
of SeqStart: of SeqStart:
e = startSeqEvent(csFlow, c.inlineProps, c.inlineStart, c.lex.curEndPos) e = startSeqEvent(csFlow, c.inlineProps, c.inlineStart, c.lex.curEndPos)
c.inlineProps = defaultProperties c.inlineProps = defaultProperties
c.levels[^1].state = afterFlowSeqSep c.transition(afterFlowSeqSep)
c.lex.next() c.lex.next()
return true return true
else: else:
@ -589,19 +627,19 @@ proc afterBlockParent(c: Context, e: var Event): bool =
c.inlineStart = c.lex.curStartPos c.inlineStart = c.lex.curStartPos
case c.lex.cur case c.lex.cur
of nodePropertyKind: of nodePropertyKind:
c.levels[^1].state = afterBlockParentProps c.transition(afterBlockParentProps)
c.levels.add(Level(state: beforeNodeProperties)) c.pushLevel(beforeNodeProperties)
of SeqItemInd, MapKeyInd: of SeqItemInd, MapKeyInd:
raise c.generateError("Compact notation not allowed after implicit key") raise c.generateError("Compact notation not allowed after implicit key")
else: else:
c.levels[^1].state = afterBlockParentProps c.transition(afterBlockParentProps)
return false return false
proc afterBlockParentProps(c: Context, e: var Event): bool = proc afterBlockParentProps(c: Context, e: var Event): bool =
c.levels[^1].indentation = c.lex.recentIndentation() c.updateIndentation(c.lex.recentIndentation())
case c.lex.cur case c.lex.cur
of nodePropertyKind: of nodePropertyKind:
c.levels.add(Level(state: beforeNodeProperties)) c.pushLevel(beforeNodeProperties)
return false return false
of MapValueInd: of MapValueInd:
raise c.generateError("Compact notation not allowed after implicit key") raise c.generateError("Compact notation not allowed after implicit key")
@ -612,33 +650,42 @@ proc afterBlockParentProps(c: Context, e: var Event): bool =
c.lex.next() c.lex.next()
if c.lex.cur == Token.MapValueInd: if c.lex.cur == Token.MapValueInd:
raise c.generateError("Compact notation not allowed after implicit key") raise c.generateError("Compact notation not allowed after implicit key")
discard c.levels.pop() c.popLevel()
return true return true
else: else:
c.levels[^1].state = afterCompactParentProps c.transition(afterCompactParentProps)
return false return false
proc requireInlineBlockItem(c: Context, e: var Event): bool = proc requireInlineBlockItem(c: Context, e: var Event): bool =
c.levels[^1].indentation = c.lex.recentIndentation() c.updateIndentation(c.lex.recentIndentation())
case c.lex.cur if c.lex.cur == Token.Indentation:
of Indentation: if c.inlineProps.tag != yTagQuestionMark:
raise c.generateError("Node properties may not stand alone on a line") if c.headerProps.tag != yTagQuestionMark:
else: raise c.generateError("Only one tag allowed per node")
c.levels[^1].state = afterCompactParentProps c.headerProps.tag = c.inlineProps.tag
c.inlineProps.tag = yTagQuestionMark
if c.inlineProps.anchor != yAnchorNone:
if c.headerProps.anchor != yAnchorNone:
raise c.generateError("Only one anchor allowed per node")
c.headerProps.anchor = c.inlineProps.anchor
c.inlineProps.anchor = yAnchorNone
c.transition(afterCompactParentProps)
return false return false
proc beforeDocEnd(c: Context, e: var Event): bool = proc beforeDocEnd(c: Context, e: var Event): bool =
case c.lex.cur case c.lex.cur
of DocumentEnd: of DocumentEnd:
e = endDocEvent(false, c.lex.curStartPos, c.lex.curEndPos)
c.levels[^1].state = beforeDoc
c.lex.next()
of StreamEnd:
e = endDocEvent(true, c.lex.curStartPos, c.lex.curEndPos) e = endDocEvent(true, c.lex.curStartPos, c.lex.curEndPos)
discard c.levels.pop() c.transition(beforeDoc)
c.lex.next()
c.tagLib.resetPrefixes()
of StreamEnd:
e = endDocEvent(false, c.lex.curStartPos, c.lex.curEndPos)
c.popLevel()
of DirectivesEnd: of DirectivesEnd:
e = endDocEvent(true, c.lex.curStartPos, c.lex.curStartPos) e = endDocEvent(false, c.lex.curStartPos, c.lex.curStartPos)
c.levels[^1].state = beforeDoc c.transition(beforeDoc)
c.tagLib.resetPrefixes()
else: else:
raise c.generateError("Unexpected token (expected document end): " & $c.lex.cur) raise c.generateError("Unexpected token (expected document end): " & $c.lex.cur)
return true return true
@ -649,14 +696,15 @@ proc inBlockSeq(c: Context, e: var Event): bool =
case c.lex.cur case c.lex.cur
of SeqItemInd: of SeqItemInd:
c.lex.next() c.lex.next()
c.levels.add(Level(state: beforeBlockIndentation)) c.pushLevel(beforeBlockIndentation)
c.levels.add(Level(state: afterCompactParent, indentation: c.blockIndentation)) c.pushLevel(afterCompactParent, c.blockIndentation)
return false return false
else: else:
if c.levels[^3].indentation == c.levels[^1].indentation: if c.levels[^3].indentation == c.levels[^1].indentation:
e = endSeqEvent(c.lex.curStartPos, c.lex.curEndPos) e = endSeqEvent(c.lex.curStartPos, c.lex.curEndPos)
discard c.levels.pop() c.popLevel()
discard c.levels.pop() c.popLevel()
return true
else: else:
raise c.generateError("Illegal token (expected block sequence indicator): " & $c.lex.cur) raise c.generateError("Illegal token (expected block sequence indicator): " & $c.lex.cur)
@ -665,26 +713,26 @@ proc beforeBlockMapKey(c: Context, e: var Event): bool =
raise c.generateError("Invalid indentation: got " & $c.blockIndentation & ", expected " & $c.levels[^1].indentation) raise c.generateError("Invalid indentation: got " & $c.blockIndentation & ", expected " & $c.levels[^1].indentation)
case c.lex.cur case c.lex.cur
of MapKeyInd: of MapKeyInd:
c.levels[^1].state = beforeBlockMapValue c.transition(beforeBlockMapValue)
c.levels.add(Level(state: beforeBlockIndentation)) c.pushLevel(beforeBlockIndentation)
c.levels.add(Level(state: afterCompactParent, indentation: c.blockIndentation)) c.pushLevel(afterCompactParent, c.blockIndentation)
c.lex.next() c.lex.next()
return false return false
of nodePropertyKind: of nodePropertyKind:
c.levels[^1].state = atBlockMapKeyProps c.transition(atBlockMapKeyProps)
c.levels.add(Level(state: beforeNodeProperties)) c.pushLevel(beforeNodeProperties)
return false return false
of Plain, SingleQuoted, DoubleQuoted: of Plain, SingleQuoted, DoubleQuoted:
c.levels[^1].state = atBlockMapKeyProps c.transition(atBlockMapKeyProps)
return false return false
of Alias: of Alias:
e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos)
c.lex.next() c.lex.next()
c.levels[^1].state = afterImplicitKey c.transition(afterImplicitKey)
return true return true
of MapValueInd: of MapValueInd:
e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curEndPos) e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curEndPos)
c.levels[^1].state = beforeBlockMapValue c.transition(beforeBlockMapValue)
return true return true
else: else:
raise c.generateError("Unexpected token (expected mapping key): " & $c.lex.cur) raise c.generateError("Unexpected token (expected mapping key): " & $c.lex.cur)
@ -692,7 +740,7 @@ proc beforeBlockMapKey(c: Context, e: var Event): bool =
proc atBlockMapKeyProps(c: Context, e: var Event): bool = proc atBlockMapKeyProps(c: Context, e: var Event): bool =
case c.lex.cur case c.lex.cur
of nodePropertyKind: of nodePropertyKind:
c.levels.add(Level(state: beforeNodeProperties)) c.pushLevel(beforeNodeProperties)
of Alias: of Alias:
e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos)
of Plain, SingleQuoted, DoubleQuoted: of Plain, SingleQuoted, DoubleQuoted:
@ -704,21 +752,21 @@ proc atBlockMapKeyProps(c: Context, e: var Event): bool =
of MapValueInd: of MapValueInd:
e = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos) e = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos)
c.inlineProps = defaultProperties c.inlineProps = defaultProperties
c.levels[^1].state = afterImplicitKey c.transition(afterImplicitKey)
return true return true
else: else:
raise c.generateError("Unexpected token (expected implicit mapping key): " & $c.lex.cur) raise c.generateError("Unexpected token (expected implicit mapping key): " & $c.lex.cur)
c.lex.next() c.lex.next()
c.levels[^1].state = afterImplicitKey c.transition(afterImplicitKey)
return true return true
proc afterImplicitKey(c: Context, e: var Event): bool = proc afterImplicitKey(c: Context, e: var Event): bool =
if c.lex.cur != Token.MapValueInd: if c.lex.cur != Token.MapValueInd:
raise c.generateError("Unexpected token (expected ':'): " & $c.lex.cur) raise c.generateError("Unexpected token (expected ':'): " & $c.lex.cur)
c.lex.next() c.lex.next()
c.levels[^1].state = beforeBlockMapKey c.transition(beforeBlockMapKey)
c.levels.add(Level(state: beforeBlockIndentation)) c.pushLevel(beforeBlockIndentation)
c.levels.add(Level(state: afterBlockParent, indentation: c.levels[^2].indentation)) c.pushLevel(afterBlockParent, max(0, c.levels[^2].indentation))
return false return false
proc beforeBlockMapValue(c: Context, e: var Event): bool = proc beforeBlockMapValue(c: Context, e: var Event): bool =
@ -726,14 +774,14 @@ proc beforeBlockMapValue(c: Context, e: var Event): bool =
raise c.generateError("Invalid indentation") raise c.generateError("Invalid indentation")
case c.lex.cur case c.lex.cur
of MapValueInd: of MapValueInd:
c.levels[^1].state = beforeBlockMapKey c.transition(beforeBlockMapKey)
c.levels.add(Level(state: beforeBlockIndentation)) c.pushLevel(beforeBlockIndentation)
c.levels.add(Level(state: afterCompactParent, indentation: c.blockIndentation)) c.pushLevel(afterCompactParent, c.blockIndentation)
c.lex.next() c.lex.next()
of MapKeyInd, Plain, SingleQuoted, DoubleQuoted, nodePropertyKind: of MapKeyInd, Plain, SingleQuoted, DoubleQuoted, nodePropertyKind:
# the value is allowed to be missing after an explicit key # the value is allowed to be missing after an explicit key
e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curEndPos) e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curEndPos)
c.levels[^1].state = beforeBlockMapKey c.transition(beforeBlockMapKey)
return true return true
else: else:
raise c.generateError("Unexpected token (expected mapping value): " & $c.lex.cur) raise c.generateError("Unexpected token (expected mapping value): " & $c.lex.cur)
@ -744,8 +792,8 @@ proc beforeBlockIndentation(c: Context, e: var Event): bool =
e = endMapEvent(c.lex.curStartPos, c.lex.curEndPos) e = endMapEvent(c.lex.curStartPos, c.lex.curEndPos)
elif c.levels[^1].state == beforeBlockMapValue: elif c.levels[^1].state == beforeBlockMapValue:
e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curEndPos) e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curEndPos)
c.levels[^1].state = beforeBlockMapKey c.transition(beforeBlockMapKey)
c.levels.add(Level(state: beforeBlockIndentation)) c.pushLevel(beforeBlockIndentation)
return return
elif c.levels[^1].state == inBlockSeq: elif c.levels[^1].state == inBlockSeq:
e = endSeqEvent(c.lex.curStartPos, c.lex.curEndPos) e = endSeqEvent(c.lex.curStartPos, c.lex.curEndPos)
@ -756,8 +804,8 @@ proc beforeBlockIndentation(c: Context, e: var Event): bool =
raise c.generateError("Unexpected double beforeBlockIndentation") raise c.generateError("Unexpected double beforeBlockIndentation")
else: else:
raise c.generateError("Internal error (please report this bug)") raise c.generateError("Internal error (please report this bug)")
discard c.levels.pop() c.popLevel()
discard c.levels.pop() c.popLevel()
case c.lex.cur case c.lex.cur
of Indentation: of Indentation:
c.blockIndentation = c.lex.currentIndentation() c.blockIndentation = c.lex.currentIndentation()
@ -778,45 +826,46 @@ proc beforeBlockIndentation(c: Context, e: var Event): bool =
raise c.generateError("Unexpected content after node in block context (expected newline): " & $c.lex.cur) raise c.generateError("Unexpected content after node in block context (expected newline): " & $c.lex.cur)
proc beforeFlowItem(c: Context, e: var Event): bool = proc beforeFlowItem(c: Context, e: var Event): bool =
debug("parse: beforeFlowItem")
c.inlineStart = c.lex.curStartPos c.inlineStart = c.lex.curStartPos
case c.lex.cur case c.lex.cur
of nodePropertyKind: of nodePropertyKind:
c.levels[^1].state = beforeFlowItemProps c.transition(beforeFlowItemProps)
c.levels.add(Level(state: beforeNodeProperties)) c.pushLevel(beforeNodeProperties)
of Alias: of Alias:
e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos)
c.lex.next() c.lex.next()
discard c.levels.pop() c.popLevel()
return true return true
else: else:
c.levels[^1].state = beforeFlowItemProps c.transition(beforeFlowItemProps)
return false return false
proc beforeFlowItemProps(c: Context, e: var Event): bool = proc beforeFlowItemProps(c: Context, e: var Event): bool =
case c.lex.cur case c.lex.cur
of nodePropertyKind: of nodePropertyKind:
c.levels.add(Level(state: beforeNodeProperties)) c.pushLevel(beforeNodeProperties)
of Alias: of Alias:
e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos)
c.lex.next() c.lex.next()
discard c.levels.pop() c.popLevel()
of scalarTokenKind: of scalarTokenKind:
e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur), e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur),
toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos) toStyle(c.lex.cur), c.inlineStart, c.lex.curEndPos)
c.inlineProps = defaultProperties c.inlineProps = defaultProperties
c.lex.next() c.lex.next()
discard c.levels.pop() c.popLevel()
of MapStart: of MapStart:
e = startMapEvent(csFlow, c.inlineProps, c.inlineStart, c.lex.curEndPos) e = startMapEvent(csFlow, c.inlineProps, c.inlineStart, c.lex.curEndPos)
c.levels[^1].state = afterFlowMapSep c.transition(afterFlowMapSep)
c.lex.next() c.lex.next()
of SeqStart: of SeqStart:
e = startSeqEvent(csFlow, c.inlineProps, c.inlineStart, c.lex.curEndPos) e = startSeqEvent(csFlow, c.inlineProps, c.inlineStart, c.lex.curEndPos)
c.levels[^1].state = afterFlowSeqSep c.transition(afterFlowSeqSep)
c.lex.next() c.lex.next()
of MapEnd, SeqEnd, SeqSep, MapValueInd: of MapEnd, SeqEnd, SeqSep, MapValueInd:
e = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curEndPos) e = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curEndPos)
discard c.levels.pop() c.popLevel()
else: else:
raise c.generateError("Unexpected token (expected flow node): " & $c.lex.cur) raise c.generateError("Unexpected token (expected flow node): " & $c.lex.cur)
c.inlineProps = defaultProperties c.inlineProps = defaultProperties
@ -825,13 +874,13 @@ proc beforeFlowItemProps(c: Context, e: var Event): bool =
proc afterFlowMapKey(c: Context, e: var Event): bool = proc afterFlowMapKey(c: Context, e: var Event): bool =
case c.lex.cur case c.lex.cur
of MapValueInd: of MapValueInd:
c.levels[^1].state = afterFlowMapValue c.transition(afterFlowMapValue)
c.levels.add(Level(state: beforeFlowItem)) c.pushLevel(beforeFlowItem)
c.lex.next() c.lex.next()
return false return false
of SeqSep, MapEnd: of SeqSep, MapEnd:
e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curEndPos) e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curEndPos)
c.levels[^1].state = afterFlowMapValue c.transition(afterFlowMapValue)
return true return true
else: else:
raise c.generateError("Unexpected token (expected ':'): " & $c.lex.cur) raise c.generateError("Unexpected token (expected ':'): " & $c.lex.cur)
@ -839,13 +888,13 @@ proc afterFlowMapKey(c: Context, e: var Event): bool =
proc afterFlowMapValue(c: Context, e: var Event): bool = proc afterFlowMapValue(c: Context, e: var Event): bool =
case c.lex.cur case c.lex.cur
of SeqSep: of SeqSep:
c.levels[^1].state = afterFlowMapSep c.transition(afterFlowMapSep)
c.lex.next() c.lex.next()
return false return false
of MapEnd: of MapEnd:
e = endMapEvent(c.lex.curStartPos, c.lex.curEndPos) e = endMapEvent(c.lex.curStartPos, c.lex.curEndPos)
c.lex.next() c.lex.next()
discard c.levels.pop() c.popLevel()
return true return true
of Plain, SingleQuoted, DoubleQuoted, MapKeyInd, Token.Anchor, Alias, MapStart, SeqStart: of Plain, SingleQuoted, DoubleQuoted, MapKeyInd, Token.Anchor, Alias, MapStart, SeqStart:
raise c.generateError("Missing ','") raise c.generateError("Missing ','")
@ -855,13 +904,13 @@ proc afterFlowMapValue(c: Context, e: var Event): bool =
proc afterFlowSeqItem(c: Context, e: var Event): bool = proc afterFlowSeqItem(c: Context, e: var Event): bool =
case c.lex.cur case c.lex.cur
of SeqSep: of SeqSep:
c.levels[^1].state = afterFlowSeqSep c.transition(afterFlowSeqSep)
c.lex.next() c.lex.next()
return false return false
of SeqEnd: of SeqEnd:
e = endSeqEvent(c.lex.curStartPos, c.lex.curEndPos) e = endSeqEvent(c.lex.curStartPos, c.lex.curEndPos)
c.lex.next() c.lex.next()
discard c.levels.pop() c.popLevel()
return true return true
of Plain, SingleQuoted, DoubleQuoted, MapKeyInd, Token.Anchor, Alias, MapStart, SeqStart: of Plain, SingleQuoted, DoubleQuoted, MapKeyInd, Token.Anchor, Alias, MapStart, SeqStart:
raise c.generateError("Missing ','") raise c.generateError("Missing ','")
@ -875,11 +924,11 @@ proc afterFlowMapSep(c: Context, e: var Event): bool =
of MapEnd: of MapEnd:
e = endMapEvent(c.lex.curStartPos, c.lex.curEndPos) e = endMapEvent(c.lex.curStartPos, c.lex.curEndPos)
c.lex.next() c.lex.next()
discard c.levels.pop() c.popLevel()
return true return true
else: discard else: discard
c.levels[^1].state = afterFlowMapKey c.transition(afterFlowMapKey)
c.levels.add(Level(state: beforeFlowItem)) c.pushLevel(beforeFlowItem)
return false return false
proc possibleNextSequenceItem(c: Context, e: var Event, endToken: Token, afterProps, afterItem: State): bool = proc possibleNextSequenceItem(c: Context, e: var Event, endToken: Token, afterProps, afterItem: State): bool =
@ -890,33 +939,33 @@ proc possibleNextSequenceItem(c: Context, e: var Event, endToken: Token, afterPr
c.lex.next() c.lex.next()
return true return true
of nodePropertyKind: of nodePropertyKind:
c.levels[^1].state = afterProps c.transition(afterProps)
c.levels.add(Level(state: beforeNodeProperties)) c.pushLevel(beforeNodeProperties)
return false return false
of Plain, SingleQuoted, DoubleQuoted: of Plain, SingleQuoted, DoubleQuoted:
c.levels[^1].state = afterProps c.transition(afterProps)
return false return false
of MapKeyInd: of MapKeyInd:
c.levels[^1].state = afterItem c.transition(afterItem)
e = startMapEvent(csFlow, defaultProperties, c.lex.curStartPos, c.lex.curEndPos) e = startMapEvent(csFlow, defaultProperties, c.lex.curStartPos, c.lex.curEndPos)
c.lex.next() c.lex.next()
c.levels.add(Level(state: beforePairValue)) c.pushLevel(beforePairValue)
c.levels.add(Level(state: beforeFlowItem)) c.pushLevel(beforeFlowItem)
return true return true
of MapValueInd: of MapValueInd:
c.levels[^1].state = afterItem c.transition(afterItem)
e = startMapEvent(csFlow, defaultProperties, c.lex.curStartPos, c.lex.curEndPos) e = startMapEvent(csFlow, defaultProperties, c.lex.curStartPos, c.lex.curEndPos)
c.levels.add(Level(state: atEmptyPairKey)) c.pushLevel(atEmptyPairKey)
return true return true
else: else:
if c.lex.cur == endToken: if c.lex.cur == endToken:
e = endSeqEvent(c.lex.curStartPos, c.lex.curEndPos) e = endSeqEvent(c.lex.curStartPos, c.lex.curEndPos)
c.lex.next() c.lex.next()
discard c.levels.pop() c.popLevel()
return true return true
else: else:
c.levels[^1].state = afterItem c.transition(afterItem)
c.levels.add(Level(state: beforeFlowItem)) c.pushLevel(beforeFlowItem)
return false return false
proc afterFlowSeqSep(c: Context, e: var Event): bool = proc afterFlowSeqSep(c: Context, e: var Event): bool =
@ -930,47 +979,44 @@ proc forcedNextSequenceItem(c: Context, e: var Event): bool =
if c.lex.cur == Token.MapValueInd: if c.lex.cur == Token.MapValueInd:
c.peek = move(e) c.peek = move(e)
e = startMapEvent(csFlow, defaultProperties, c.lex.curStartPos, c.lex.curStartPos) e = startMapEvent(csFlow, defaultProperties, c.lex.curStartPos, c.lex.curStartPos)
c.levels.add(Level(state: afterImplicitPairStart)) c.pushLevel(afterImplicitPairStart)
return true return true
else: else:
c.levels.add(Level(state: beforeFlowItem)) c.pushLevel(beforeFlowItem)
return false return false
proc afterFlowSeqSepProps(c: Context, e: var Event): bool = proc afterFlowSeqSepProps(c: Context, e: var Event): bool =
c.levels[^1].state = afterFlowSeqItem c.transition(afterFlowSeqItem)
return forcedNextSequenceItem(c, e) return forcedNextSequenceItem(c, e)
proc atEmptyPairKey(c: Context, e: var Event): bool = proc atEmptyPairKey(c: Context, e: var Event): bool =
c.levels[^1].state = beforePairValue c.transition(beforePairValue)
e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curStartPos) e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curStartPos)
return true return true
proc beforePairValue(c: Context, e: var Event): bool = proc beforePairValue(c: Context, e: var Event): bool =
if c.lex.cur == Token.MapValueInd: if c.lex.cur == Token.MapValueInd:
c.levels[^1].state = afterPairValue c.transition(afterPairValue)
c.levels.add(Level(state: beforeFlowItem)) c.pushLevel(beforeFlowItem)
c.lex.next() c.lex.next()
return false return false
else: else:
# pair ends here without value # pair ends here without value
e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curEndPos) e = scalarEvent("", defaultProperties, ssPlain, c.lex.curStartPos, c.lex.curEndPos)
discard c.levels.pop() c.popLevel()
return true return true
proc afterImplicitPairStart(c: Context, e: var Event): bool = proc afterImplicitPairStart(c: Context, e: var Event): bool =
c.lex.next() c.lex.next()
c.levels[^1].state = afterPairValue c.transition(afterPairValue)
c.levels.add(Level(state: beforeFLowItem)) c.pushLevel(beforeFlowItem)
return false return false
proc afterPairValue(c: Context, e: var Event): bool = proc afterPairValue(c: Context, e: var Event): bool =
e = endMapEvent(c.lex.curStartPos, c.lex.curEndPos) e = endMapEvent(c.lex.curStartPos, c.lex.curEndPos)
discard c.levels.pop() c.popLevel()
return true return true
# TODO --------------
proc display*(p: YamlParser, event: Event): string = proc display*(p: YamlParser, event: Event): string =
## Generate a representation of the given event with proper visualization of ## Generate a representation of the given event with proper visualization of
## anchor and tag (if any). The generated representation is conformant to the ## anchor and tag (if any). The generated representation is conformant to the
@ -988,30 +1034,22 @@ proc display*(p: YamlParser, event: Event): string =
of yamlEndSeq: result = "-SEQ" of yamlEndSeq: result = "-SEQ"
of yamlStartDoc: of yamlStartDoc:
result = "+DOC" result = "+DOC"
when defined(yamlScalarRepInd):
if event.explicitDirectivesEnd: result &= " ---" if event.explicitDirectivesEnd: result &= " ---"
of yamlEndDoc: of yamlEndDoc:
result = "-DOC" result = "-DOC"
when defined(yamlScalarRepInd):
if event.explicitDocumentEnd: result &= " ..." if event.explicitDocumentEnd: result &= " ..."
of yamlStartMap: of yamlStartMap:
result = "+MAP" & renderAttrs(event.mapProperties, true) result = "+MAP" & renderAttrs(event.mapProperties, true)
of yamlStartSeq: of yamlStartSeq:
result = "+SEQ" & renderAttrs(event.seqProperties, true) result = "+SEQ" & renderAttrs(event.seqProperties, true)
of yamlScalar: of yamlScalar:
when defined(yamlScalarRepInd):
result = "=VAL" & renderAttrs(event.scalarProperties, result = "=VAL" & renderAttrs(event.scalarProperties,
event.scalarRep == srPlain) event.scalarStyle in {ssPlain, ssFolded, ssLiteral})
case event.scalarRep case event.scalarStyle
of srPlain: result &= " :" of ssPlain, ssAny: result &= " :"
of srSingleQuoted: result &= " \'" of ssSingleQuoted: result &= " \'"
of srDoubleQuoted: result &= " \"" of ssDoubleQuoted: result &= " \""
of srLiteral: result &= " |" of ssLiteral: result &= " |"
of srFolded: result &= " >" of ssFolded: result &= " >"
else:
let isPlain = event.scalarProperties.tag == yTagExclamationmark
result = "=VAL" & renderAttrs(event.scalarProperties, isPlain)
if isPlain: result &= " :"
else: result &= " \""
result &= yamlTestSuiteEscape(event.scalarContent) result &= yamlTestSuiteEscape(event.scalarContent)
of yamlAlias: result = "=ALI *" & $event.aliasTarget of yamlAlias: result = "=ALI *" & $event.aliasTarget

39
yaml/private/lex.nim
View File

@ -172,7 +172,7 @@ proc streamEnd(lex: var Lexer): bool {.raises: [].}
template debug(message: string) {.dirty.} = template debug(message: string) {.dirty.} =
when defined(yamlDebug): when defined(yamlDebug):
try: styledWriteLine(stdout, fgBlue, message) try: styledWriteLine(stdout, fgBlue, message)
except IOError: discard except ValueError, IOError: discard
proc generateError(lex: Lexer, message: string): proc generateError(lex: Lexer, message: string):
ref LexerError {.raises: [].} = ref LexerError {.raises: [].} =
@ -428,6 +428,13 @@ proc streamEndAfterBlock(lex: var Lexer) =
lex.endToken() lex.endToken()
lex.curEndPos.column -= 1 lex.curEndPos.column -= 1
proc dirEndFollows(lex: Lexer): bool =
return lex.c == '-' and lex.source.buf[lex.source.bufpos] == '-' and
lex.source.buf[lex.source.bufpos+1] == '-'
proc docEndFollows(lex: Lexer): bool =
return lex.c == '.' and lex.source.buf[lex.source.bufpos] == '.' and
lex.source.buf[lex.source.bufpos+1] == '.'
proc readBlockScalar(lex: var Lexer) = proc readBlockScalar(lex: var Lexer) =
var var
@ -492,7 +499,8 @@ proc readBlockScalar(lex: var Lexer) =
else: else:
if indent == 0: if indent == 0:
indent = lex.currentIndentation() indent = lex.currentIndentation()
if indent <= max(0, lex.indentation): if indent <= lex.indentation or
(indent == 0 and (lex.dirEndFollows() or lex.docEndFollows())):
lex.state = lineIndentation lex.state = lineIndentation
break body break body
elif indent < maxLeadingSpaces: elif indent < maxLeadingSpaces:
@ -530,7 +538,8 @@ proc readBlockScalar(lex: var Lexer) =
lex.streamEndAfterBlock() lex.streamEndAfterBlock()
break body break body
else: else:
if lex.currentIndentation() < indent: if lex.currentIndentation() < indent or
(indent == 0 and lex.dirEndFollows() or lex.docEndFollows()):
break content break content
else: break else: break
@ -544,12 +553,14 @@ proc readBlockScalar(lex: var Lexer) =
for i in countup(0, separationLines - 2): for i in countup(0, separationLines - 2):
lex.evaluated.add('\l') lex.evaluated.add('\l')
if lex.currentIndentation() > max(0, lex.indentation): let markerFollows = lex.currentIndentation() == 0 and
(lex.dirEndFollows() or lex.docEndFollows())
if lex.currentIndentation() > lex.indentation and not markerFollows:
if lex.c == '#': if lex.c == '#':
lex.state = expectLineEnd lex.state = expectLineEnd
else: else:
raise lex.generateError("This line at " & escape("" & lex.c) & " is less indented than necessary") raise lex.generateError("This line #" & $lex.curStartPos.line & " at " & escape("" & lex.c) & " is less indented than necessary")
elif lex.columnNumber() == 1: elif lex.currentIndentation() == 0:
lex.state = lineStart lex.state = lineStart
else: else:
lex.state = lineIndentation lex.state = lineIndentation
@ -570,7 +581,7 @@ proc processQuotedWhitespace(lex: var Lexer, initial: int) =
let firstSpace = lex.source.bufpos - 1 let firstSpace = lex.source.bufpos - 1
while true: while true:
case lex.c case lex.c
of ' ': discard of ' ', '\t': discard
of '\l': of '\l':
lex.lexLF() lex.lexLF()
break break
@ -584,7 +595,11 @@ proc processQuotedWhitespace(lex: var Lexer, initial: int) =
lex.seenMultiline = true lex.seenMultiline = true
while true: while true:
case lex.startLine() case lex.startLine()
of lsContent, lsComment: break of lsContent, lsComment:
while lex.c in space: lex.advance()
if lex.c in {'\l', '\c'}:
lex.endLine()
else: break
of lsDirectivesEndMarker: of lsDirectivesEndMarker:
raise lex.generateError("Illegal `---` within quoted scalar") raise lex.generateError("Illegal `---` within quoted scalar")
of lsDocumentEndMarker: of lsDocumentEndMarker:
@ -619,7 +634,7 @@ proc readSingleQuotedScalar(lex: var Lexer) =
literalStart = lex.source.bufpos literalStart = lex.source.bufpos
lex.advance() lex.advance()
else: break else: break
of ' ', '\l', '\c': of ' ', '\t', '\l', '\c':
lex.evaluated.add(lex.source.buf[literalStart..lex.source.bufpos - 2]) lex.evaluated.add(lex.source.buf[literalStart..lex.source.bufpos - 2])
lex.processQuotedWhitespace(1) lex.processQuotedWhitespace(1)
literalStart = lex.source.bufpos - 1 literalStart = lex.source.bufpos - 1
@ -681,7 +696,7 @@ proc readDoubleQuotedScalar(lex: var Lexer) =
of '"': of '"':
lex.evaluated.add(lex.source.buf[literalStart..lex.source.bufpos - 2]) lex.evaluated.add(lex.source.buf[literalStart..lex.source.bufpos - 2])
break break
of ' ', '\l', '\c': of ' ', '\t', '\l', '\c':
lex.evaluated.add(lex.source.buf[literalStart..lex.source.bufpos - 2]) lex.evaluated.add(lex.source.buf[literalStart..lex.source.bufpos - 2])
lex.processQuotedWhitespace(1) lex.processQuotedWhitespace(1)
literalStart = lex.source.bufpos - 1 literalStart = lex.source.bufpos - 1
@ -756,7 +771,7 @@ proc outsideDoc(lex: var Lexer): bool =
of '-': of '-':
lex.startToken() lex.startToken()
if lex.isDirectivesEnd(): if lex.isDirectivesEnd():
lex.state = expectLineEnd lex.state = afterToken
lex.cur = Token.DirectivesEnd lex.cur = Token.DirectivesEnd
else: else:
lex.state = indentationSettingToken lex.state = indentationSettingToken
@ -783,6 +798,7 @@ proc outsideDoc(lex: var Lexer): bool =
return false return false
lex.endToken() lex.endToken()
lex.cur = Token.Indentation lex.cur = Token.Indentation
lex.indentation = -1
lex.state = indentationSettingToken lex.state = indentationSettingToken
lex.lineStartState = lineStart lex.lineStartState = lineStart
return true return true
@ -1083,6 +1099,7 @@ proc lineDirEnd(lex: var Lexer): bool =
lex.curStartPos.column = 1 lex.curStartPos.column = 1
lex.endToken() lex.endToken()
lex.cur = Token.DirectivesEnd lex.cur = Token.DirectivesEnd
lex.state = afterToken
lex.indentation = -1 lex.indentation = -1
lex.propertyIndentation = -1 lex.propertyIndentation = -1
return true return true

184
yaml/serialization.nim
View File

@ -18,7 +18,7 @@
import tables, typetraits, strutils, macros, streams, times, parseutils, options import tables, typetraits, strutils, macros, streams, times, parseutils, options
import data, parser, taglib, presenter, stream, private/internal, hints, annotations import data, parser, taglib, presenter, stream, private/internal, hints, annotations
export stream, macros, annotations, options export data, stream, macros, annotations, options
# *something* in here needs externally visible `==`(x,y: AnchorId), # *something* in here needs externally visible `==`(x,y: AnchorId),
# but I cannot figure out what. binding it would be the better option. # but I cannot figure out what. binding it would be the better option.
@ -125,10 +125,10 @@ proc safeTagUri(id: TagId): string {.raises: [].} =
return uri return uri
except KeyError: internalError("Unexpected KeyError for TagId " & $id) except KeyError: internalError("Unexpected KeyError for TagId " & $id)
proc constructionError(s: YamlStream, msg: string): ref YamlConstructionError = proc constructionError(s: YamlStream, mark: Mark, msg: string): ref YamlConstructionError =
result = newException(YamlConstructionError, msg) result = newException(YamlConstructionError, msg)
if not s.getLastTokenContext(result.mark.line, result.mark.column, result.lineContent): result.mark = mark
(result.mark.line, result.mark.column) = (-1, -1) if not s.getLastTokenContext(result.lineContent):
result.lineContent = "" result.lineContent = ""
template constructScalarItem*(s: var YamlStream, i: untyped, template constructScalarItem*(s: var YamlStream, i: untyped,
@ -141,11 +141,11 @@ template constructScalarItem*(s: var YamlStream, i: untyped,
bind constructionError bind constructionError
let i = s.next() let i = s.next()
if i.kind != yamlScalar: if i.kind != yamlScalar:
raise constructionError(s, "Expected scalar") raise s.constructionError(i.startPos, "Expected scalar")
try: content try: content
except YamlConstructionError as e: raise e except YamlConstructionError as e: raise e
except Exception: except Exception:
var e = constructionError(s, var e = s.constructionError(i.startPos,
"Cannot construct to " & name(t) & ": " & item.scalarContent & "Cannot construct to " & name(t) & ": " & item.scalarContent &
"; error: " & getCurrentExceptionMsg()) "; error: " & getCurrentExceptionMsg())
e.parent = getCurrentException() e.parent = getCurrentException()
@ -167,7 +167,7 @@ proc representObject*(value: string, ts: TagStyle,
c.put(scalarEvent(value, tag, yAnchorNone)) c.put(scalarEvent(value, tag, yAnchorNone))
proc parseHex[T: int8|int16|int32|int64|uint8|uint16|uint32|uint64]( proc parseHex[T: int8|int16|int32|int64|uint8|uint16|uint32|uint64](
s: YamlStream, val: string): T = s: YamlStream, mark: Mark, val: string): T =
result = 0 result = 0
for i in 2..<val.len: for i in 2..<val.len:
case val[i] case val[i]
@ -176,17 +176,17 @@ proc parseHex[T: int8|int16|int32|int64|uint8|uint16|uint32|uint64](
of 'a'..'f': result = result shl 4 or T(ord(val[i]) - ord('a') + 10) of 'a'..'f': result = result shl 4 or T(ord(val[i]) - ord('a') + 10)
of 'A'..'F': result = result shl 4 or T(ord(val[i]) - ord('A') + 10) of 'A'..'F': result = result shl 4 or T(ord(val[i]) - ord('A') + 10)
else: else:
raise s.constructionError("Invalid character in hex: " & raise s.constructionError(mark, "Invalid character in hex: " &
escape("" & val[i])) escape("" & val[i]))
proc parseOctal[T: int8|int16|int32|int64|uint8|uint16|uint32|uint64]( proc parseOctal[T: int8|int16|int32|int64|uint8|uint16|uint32|uint64](
s: YamlStream, val: string): T = s: YamlStream, mark: Mark, val: string): T =
for i in 2..<val.len: for i in 2..<val.len:
case val[i] case val[i]
of '_': discard of '_': discard
of '0'..'7': result = result shl 3 + T((ord(val[i]) - ord('0'))) of '0'..'7': result = result shl 3 + T((ord(val[i]) - ord('0')))
else: else:
raise s.constructionError("Invalid character in hex: " & raise s.constructionError(mark, "Invalid character in hex: " &
escape("" & val[i])) escape("" & val[i]))
proc constructObject*[T: int8|int16|int32|int64]( proc constructObject*[T: int8|int16|int32|int64](
@ -195,16 +195,16 @@ proc constructObject*[T: int8|int16|int32|int64](
## constructs an integer value from a YAML scalar ## constructs an integer value from a YAML scalar
constructScalarItem(s, item, T): constructScalarItem(s, item, T):
if item.scalarContent[0] == '0' and item.scalarContent.len > 1 and item.scalarContent[1] in {'x', 'X' }: if item.scalarContent[0] == '0' and item.scalarContent.len > 1 and item.scalarContent[1] in {'x', 'X' }:
result = parseHex[T](s, item.scalarContent) result = parseHex[T](s, item.startPos, item.scalarContent)
elif item.scalarContent[0] == '0' and item.scalarContent.len > 1 and item.scalarContent[1] in {'o', 'O'}: elif item.scalarContent[0] == '0' and item.scalarContent.len > 1 and item.scalarContent[1] in {'o', 'O'}:
result = parseOctal[T](s, item.scalarContent) result = parseOctal[T](s, item.startPos, item.scalarContent)
else: else:
let nInt = parseBiggestInt(item.scalarContent) let nInt = parseBiggestInt(item.scalarContent)
if nInt <= T.high: if nInt <= T.high:
# make sure we don't produce a range error # make sure we don't produce a range error
result = T(nInt) result = T(nInt)
else: else:
raise s.constructionError("Cannot construct int; out of range: " & raise s.constructionError(item.startPos, "Cannot construct int; out of range: " &
$nInt & " for type " & T.name & " with max of: " & $T.high) $nInt & " for type " & T.name & " with max of: " & $T.high)
proc constructObject*(s: var YamlStream, c: ConstructionContext, proc constructObject*(s: var YamlStream, c: ConstructionContext,
@ -245,9 +245,9 @@ proc constructObject*[T: DefiniteUIntTypes](
## construct an unsigned integer value from a YAML scalar ## construct an unsigned integer value from a YAML scalar
constructScalarItem(s, item, T): constructScalarItem(s, item, T):
if item.scalarContent[0] == '0' and item.scalarContent[1] in {'x', 'X'}: if item.scalarContent[0] == '0' and item.scalarContent[1] in {'x', 'X'}:
result = parseHex[T](s, item.scalarContent) result = parseHex[T](s, item.startPos, item.scalarContent)
elif item.scalarContent[0] == '0' and item.scalarContent[1] in {'o', 'O'}: elif item.scalarContent[0] == '0' and item.scalarContent[1] in {'o', 'O'}:
result = parseOctal[T](s, item.scalarContent) result = parseOctal[T](s, item.startPos, item.scalarContent)
else: result = T(parseBiggestUInt(item.scalarContent)) else: result = T(parseBiggestUInt(item.scalarContent))
proc constructObject*(s: var YamlStream, c: ConstructionContext, proc constructObject*(s: var YamlStream, c: ConstructionContext,
@ -295,7 +295,7 @@ proc constructObject*[T: float|float32|float64](
else: result = Inf else: result = Inf
of yTypeFloatNaN: result = NaN of yTypeFloatNaN: result = NaN
else: else:
raise s.constructionError("Cannot construct to float: " & raise s.constructionError(item.startPos, "Cannot construct to float: " &
escape(item.scalarContent)) escape(item.scalarContent))
proc representObject*[T: float|float32|float64](value: T, ts: TagStyle, proc representObject*[T: float|float32|float64](value: T, ts: TagStyle,
@ -318,7 +318,7 @@ proc constructObject*(s: var YamlStream, c: ConstructionContext,
of yTypeBoolTrue: result = true of yTypeBoolTrue: result = true
of yTypeBoolFalse: result = false of yTypeBoolFalse: result = false
else: else:
raise s.constructionError("Cannot construct to bool: " & raise s.constructionError(item.startPos, "Cannot construct to bool: " &
escape(item.scalarContent)) escape(item.scalarContent))
proc representObject*(value: bool, ts: TagStyle, c: SerializationContext, proc representObject*(value: bool, ts: TagStyle, c: SerializationContext,
@ -332,7 +332,7 @@ proc constructObject*(s: var YamlStream, c: ConstructionContext,
## constructs a char value from a YAML scalar ## constructs a char value from a YAML scalar
constructScalarItem(s, item, char): constructScalarItem(s, item, char):
if item.scalarContent.len != 1: if item.scalarContent.len != 1:
raise s.constructionError("Cannot construct to char (length != 1): " & raise s.constructionError(item.startPos, "Cannot construct to char (length != 1): " &
escape(item.scalarContent)) escape(item.scalarContent))
else: result = item.scalarContent[0] else: result = item.scalarContent[0]
@ -399,7 +399,7 @@ proc constructObject*(s: var YamlStream, c: ConstructionContext,
let info = tmp.parse("yyyy-M-d'T'H:mm:sszzz") let info = tmp.parse("yyyy-M-d'T'H:mm:sszzz")
result = info.toTime() result = info.toTime()
else: else:
raise s.constructionError("Not a parsable timestamp: " & raise s.constructionError(item.startPos, "Not a parsable timestamp: " &
escape(item.scalarContent)) escape(item.scalarContent))
proc representObject*(value: Time, ts: TagStyle, c: SerializationContext, proc representObject*(value: Time, ts: TagStyle, c: SerializationContext,
@ -421,7 +421,7 @@ proc constructObject*[T](s: var YamlStream, c: ConstructionContext,
## constructs a Nim seq from a YAML sequence ## constructs a Nim seq from a YAML sequence
let event = s.next() let event = s.next()
if event.kind != yamlStartSeq: if event.kind != yamlStartSeq:
raise s.constructionError("Expected sequence start") raise s.constructionError(event.startPos, "Expected sequence start")
result = newSeq[T]() result = newSeq[T]()
while s.peek().kind != yamlEndSeq: while s.peek().kind != yamlEndSeq:
var item: T var item: T
@ -435,7 +435,7 @@ proc constructObject*[T](s: var YamlStream, c: ConstructionContext,
## constructs a Nim seq from a YAML sequence ## constructs a Nim seq from a YAML sequence
let event = s.next() let event = s.next()
if event.kind != yamlStartSeq: if event.kind != yamlStartSeq:
raise s.constructionError("Expected sequence start") raise s.constructionError(event.startPos, "Expected sequence start")
result = {} result = {}
while s.peek().kind != yamlEndSeq: while s.peek().kind != yamlEndSeq:
var item: T var item: T
@ -447,7 +447,7 @@ proc representObject*[T](value: seq[T]|set[T], ts: TagStyle,
c: SerializationContext, tag: TagId) = c: SerializationContext, tag: TagId) =
## represents a Nim seq as YAML sequence ## represents a Nim seq as YAML sequence
let childTagStyle = if ts == tsRootOnly: tsNone else: ts let childTagStyle = if ts == tsRootOnly: tsNone else: ts
c.put(startSeqEvent(tag)) c.put(startSeqEvent(csBlock, tag))
for item in value: for item in value:
representChild(item, childTagStyle, c) representChild(item, childTagStyle, c)
c.put(endSeqEvent()) c.put(endSeqEvent())
@ -464,15 +464,15 @@ proc constructObject*[I, T](s: var YamlStream, c: ConstructionContext,
## constructs a Nim array from a YAML sequence ## constructs a Nim array from a YAML sequence
var event = s.next() var event = s.next()
if event.kind != yamlStartSeq: if event.kind != yamlStartSeq:
raise s.constructionError("Expected sequence start") raise s.constructionError(event.startPos, "Expected sequence start")
for index in low(I)..high(I): for index in low(I)..high(I):
event = s.peek() event = s.peek()
if event.kind == yamlEndSeq: if event.kind == yamlEndSeq:
raise s.constructionError("Too few array values") raise s.constructionError(event.startPos, "Too few array values")
constructChild(s, c, result[index]) constructChild(s, c, result[index])
event = s.next() event = s.next()
if event.kind != yamlEndSeq: if event.kind != yamlEndSeq:
raise s.constructionError("Too many array values") raise s.constructionError(event.startPos, "Too many array values")
proc representObject*[I, T](value: array[I, T], ts: TagStyle, proc representObject*[I, T](value: array[I, T], ts: TagStyle,
c: SerializationContext, tag: TagId) = c: SerializationContext, tag: TagId) =
@ -498,7 +498,7 @@ proc constructObject*[K, V](s: var YamlStream, c: ConstructionContext,
## constructs a Nim Table from a YAML mapping ## constructs a Nim Table from a YAML mapping
let event = s.next() let event = s.next()
if event.kind != yamlStartMap: if event.kind != yamlStartMap:
raise s.constructionError("Expected map start, got " & $event.kind) raise s.constructionError(event.startPos, "Expected map start, got " & $event.kind)
result = initTable[K, V]() result = initTable[K, V]()
while s.peek.kind != yamlEndMap: while s.peek.kind != yamlEndMap:
var var
@ -507,7 +507,7 @@ proc constructObject*[K, V](s: var YamlStream, c: ConstructionContext,
constructChild(s, c, key) constructChild(s, c, key)
constructChild(s, c, value) constructChild(s, c, value)
if result.contains(key): if result.contains(key):
raise s.constructionError("Duplicate table key!") raise s.constructionError(event.startPos, "Duplicate table key!")
result[key] = value result[key] = value
discard s.next() discard s.next()
@ -537,7 +537,7 @@ proc constructObject*[K, V](s: var YamlStream, c: ConstructionContext,
## constructs a Nim OrderedTable from a YAML mapping ## constructs a Nim OrderedTable from a YAML mapping
var event = s.next() var event = s.next()
if event.kind != yamlStartSeq: if event.kind != yamlStartSeq:
raise s.constructionError("Expected seq start, got " & $event.kind) raise s.constructionError(event.startPos, "Expected seq start, got " & $event.kind)
result = initOrderedTable[K, V]() result = initOrderedTable[K, V]()
while s.peek.kind != yamlEndSeq: while s.peek.kind != yamlEndSeq:
var var
@ -545,14 +545,14 @@ proc constructObject*[K, V](s: var YamlStream, c: ConstructionContext,
value: V value: V
event = s.next() event = s.next()
if event.kind != yamlStartMap: if event.kind != yamlStartMap:
raise s.constructionError("Expected map start, got " & $event.kind) raise s.constructionError(event.startPos, "Expected map start, got " & $event.kind)
constructChild(s, c, key) constructChild(s, c, key)
constructChild(s, c, value) constructChild(s, c, value)
event = s.next() event = s.next()
if event.kind != yamlEndMap: if event.kind != yamlEndMap:
raise s.constructionError("Expected map end, got " & $event.kind) raise s.constructionError(event.startPos, "Expected map end, got " & $event.kind)
if result.contains(key): if result.contains(key):
raise s.constructionError("Duplicate table key!") raise s.constructionError(event.startPos, "Duplicate table key!")
result.add(key, value) result.add(key, value)
discard s.next() discard s.next()
@ -643,9 +643,9 @@ macro matchMatrix(t: typedesc): untyped =
result.add(newLit(false)) result.add(newLit(false))
proc checkDuplicate(s: NimNode, tName: string, name: string, i: int, proc checkDuplicate(s: NimNode, tName: string, name: string, i: int,
matched: NimNode): NimNode {.compileTime.} = matched: NimNode, m: NimNode): NimNode {.compileTime.} =
result = newIfStmt((newNimNode(nnkBracketExpr).add(matched, newLit(i)), result = newIfStmt((newNimNode(nnkBracketExpr).add(matched, newLit(i)),
newNimNode(nnkRaiseStmt).add(newCall(bindSym("constructionError"), s, newNimNode(nnkRaiseStmt).add(newCall(bindSym("constructionError"), s, m,
newLit("While constructing " & tName & ": Duplicate field: " & newLit("While constructing " & tName & ": Duplicate field: " &
escape(name)))))) escape(name))))))
@ -669,7 +669,7 @@ proc getOptionInner(fType: NimNode): NimNode {.compileTime.} =
else: return nil else: return nil
proc checkMissing(s: NimNode, t: NimNode, tName: string, field: NimNode, proc checkMissing(s: NimNode, t: NimNode, tName: string, field: NimNode,
i: int, matched, o: NimNode): i: int, matched, o: NimNode, m: NimNode):
NimNode {.compileTime.} = NimNode {.compileTime.} =
let let
fType = getTypeInst(field) fType = getTypeInst(field)
@ -683,7 +683,7 @@ proc checkMissing(s: NimNode, t: NimNode, tName: string, field: NimNode,
elif hasSparse(`t`) and `o`.`field` is Option: elif hasSparse(`t`) and `o`.`field` is Option:
`o`.`field` = none(`optionInner`) `o`.`field` = none(`optionInner`)
else: else:
raise constructionError(`s`, "While constructing " & `tName` & raise constructionError(`s`, `m`, "While constructing " & `tName` &
": Missing field: " & `fName`) ": Missing field: " & `fName`)
proc markAsFound(i: int, matched: NimNode): NimNode {.compileTime.} = proc markAsFound(i: int, matched: NimNode): NimNode {.compileTime.} =
@ -708,7 +708,7 @@ proc ifNotTransient(o, field: NimNode,
`stmts` `stmts`
macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, o: typed, macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, o: typed,
matched: typed) = matched: typed, m: Mark) =
result = newStmtList() result = newStmtList()
let let
tDecl = getType(t) tDecl = getType(t)
@ -718,7 +718,7 @@ macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, o: typed,
for child in tDesc[2].children: for child in tDesc[2].children:
if child.kind == nnkRecCase: if child.kind == nnkRecCase:
result.add(checkMissing( result.add(checkMissing(
s, t, tName, child[0], field, matched, o)) s, t, tName, child[0], field, matched, o, m))
for bIndex in 1 .. len(child) - 1: for bIndex in 1 .. len(child) - 1:
let discChecks = newStmtList() let discChecks = newStmtList()
var var
@ -735,16 +735,16 @@ macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, o: typed,
for item in child[bIndex][recListIndex].recListItems: for item in child[bIndex][recListIndex].recListItems:
inc(field) inc(field)
discChecks.add(checkMissing( discChecks.add(checkMissing(
s, t, tName, item, field, matched, o)) s, t, tName, item, field, matched, o, m))
result.add(newIfStmt((infix(newDotExpr(o, newIdentNode($child[0])), result.add(newIfStmt((infix(newDotExpr(o, newIdentNode($child[0])),
"in", curValues), discChecks))) "in", curValues), discChecks)))
else: else:
result.add(checkMissing(s, t, tName, child, field, matched, o)) result.add(checkMissing(s, t, tName, child, field, matched, o, m))
inc(field) inc(field)
macro constructFieldValue(t: typedesc, stream: untyped, macro constructFieldValue(t: typedesc, stream: untyped,
context: untyped, name: untyped, o: untyped, context: untyped, name: untyped, o: untyped,
matched: untyped, failOnUnknown: bool) = matched: untyped, failOnUnknown: bool, m: untyped) =
let let
tDecl = getType(t) tDecl = getType(t)
tName = $tDecl[1] tName = $tDecl[1]
@ -770,7 +770,7 @@ macro constructFieldValue(t: typedesc, stream: untyped,
objConstr.add(newColonExpr(newIdentNode($otherChild), newDotExpr(o, objConstr.add(newColonExpr(newIdentNode($otherChild), newDotExpr(o,
newIdentNode($otherChild)))) newIdentNode($otherChild))))
disOb.add(newStmtList( disOb.add(newStmtList(
checkDuplicate(stream, tName, $child[0], fieldIndex, matched), checkDuplicate(stream, tName, $child[0], fieldIndex, matched, m),
newNimNode(nnkVarSection).add( newNimNode(nnkVarSection).add(
newNimNode(nnkIdentDefs).add( newNimNode(nnkIdentDefs).add(
newIdentNode("value"), discType, newEmptyNode())), newIdentNode("value"), discType, newEmptyNode())),
@ -808,7 +808,7 @@ macro constructFieldValue(t: typedesc, stream: untyped,
infix(newStrLitNode("Field " & $item & " not allowed for " & infix(newStrLitNode("Field " & $item & " not allowed for " &
$child[0] & " == "), "&", prefix(discriminant, "$")))))) $child[0] & " == "), "&", prefix(discriminant, "$"))))))
ob.add(ifNotTransient(o, item, ob.add(ifNotTransient(o, item,
[checkDuplicate(stream, tName, $item, fieldIndex, matched), [checkDuplicate(stream, tName, $item, fieldIndex, matched, m),
ifStmt, markAsFound(fieldIndex, matched)], true, stream, tName, ifStmt, markAsFound(fieldIndex, matched)], true, stream, tName,
$item)) $item))
caseStmt.add(ob) caseStmt.add(ob)
@ -817,7 +817,7 @@ macro constructFieldValue(t: typedesc, stream: untyped,
var ob = newNimNode(nnkOfBranch).add(newStrLitNode($child)) var ob = newNimNode(nnkOfBranch).add(newStrLitNode($child))
let field = newDotExpr(o, newIdentNode($child)) let field = newDotExpr(o, newIdentNode($child))
ob.add(ifNotTransient(o, child, ob.add(ifNotTransient(o, child,
[checkDuplicate(stream, tName, $child, fieldIndex, matched), [checkDuplicate(stream, tName, $child, fieldIndex, matched, m),
newCall("constructChild", stream, context, field), newCall("constructChild", stream, context, field),
markAsFound(fieldIndex, matched)], true, stream, tName, $child)) markAsFound(fieldIndex, matched)], true, stream, tName, $child))
caseStmt.add(ob) caseStmt.add(ob)
@ -825,7 +825,7 @@ macro constructFieldValue(t: typedesc, stream: untyped,
caseStmt.add(newNimNode(nnkElse).add(newNimNode(nnkWhenStmt).add( caseStmt.add(newNimNode(nnkElse).add(newNimNode(nnkWhenStmt).add(
newNimNode(nnkElifBranch).add(failOnUnknown, newNimNode(nnkElifBranch).add(failOnUnknown,
newNimNode(nnkRaiseStmt).add( newNimNode(nnkRaiseStmt).add(
newCall(bindSym("constructionError"), stream, newCall(bindSym("constructionError"), stream, m,
infix(newLit("While constructing " & tName & ": Unknown field: "), "&", infix(newLit("While constructing " & tName & ": Unknown field: "), "&",
newCall(bindSym("escape"), name)))))))) newCall(bindSym("escape"), name))))))))
result.add(caseStmt) result.add(caseStmt)
@ -859,8 +859,9 @@ proc constructObjectDefault*[O: object|tuple](
startKind = when isVariantObject(getType(O)): yamlStartSeq else: yamlStartMap startKind = when isVariantObject(getType(O)): yamlStartSeq else: yamlStartMap
endKind = when isVariantObject(getType(O)): yamlEndSeq else: yamlEndMap endKind = when isVariantObject(getType(O)): yamlEndSeq else: yamlEndMap
if e.kind != startKind: if e.kind != startKind:
raise s.constructionError("While constructing " & raise s.constructionError(e.startPos, "While constructing " &
typetraits.name(O) & ": Expected " & $startKind & ", got " & $e.kind) typetraits.name(O) & ": Expected " & $startKind & ", got " & $e.kind)
let startPos = e.startPos
when hasIgnore(O): when hasIgnore(O):
const ignoredKeyList = O.getCustomPragmaVal(ignore) const ignoredKeyList = O.getCustomPragmaVal(ignore)
const failOnUnknown = len(ignoredKeyList) > 0 const failOnUnknown = len(ignoredKeyList) > 0
@ -870,10 +871,10 @@ proc constructObjectDefault*[O: object|tuple](
e = s.next() e = s.next()
when isVariantObject(getType(O)): when isVariantObject(getType(O)):
if e.kind != yamlStartMap: if e.kind != yamlStartMap:
raise s.constructionError("Expected single-pair map, got " & $e.kind) raise s.constructionError(e.startPos, "Expected single-pair map, got " & $e.kind)
e = s.next() e = s.next()
if e.kind != yamlScalar: if e.kind != yamlScalar:
raise s.constructionError("Expected field name, got " & $e.kind) raise s.constructionError(e.startPos, "Expected field name, got " & $e.kind)
let name = e.scalarContent let name = e.scalarContent
when result is tuple: when result is tuple:
var i = 0 var i = 0
@ -881,7 +882,7 @@ proc constructObjectDefault*[O: object|tuple](
for fname, value in fieldPairs(result): for fname, value in fieldPairs(result):
if fname == name: if fname == name:
if matched[i]: if matched[i]:
raise s.constructionError("While constructing " & raise s.constructionError(e.startPos, "While constructing " &
typetraits.name(O) & ": Duplicate field: " & escape(name)) typetraits.name(O) & ": Duplicate field: " & escape(name))
constructChild(s, c, value) constructChild(s, c, value)
matched[i] = true matched[i] = true
@ -890,12 +891,12 @@ proc constructObjectDefault*[O: object|tuple](
inc(i) inc(i)
when failOnUnknown: when failOnUnknown:
if not found: if not found:
raise s.constructionError("While constructing " & raise s.constructionError(e.startPos, "While constructing " &
typetraits.name(O) & ": Unknown field: " & escape(name)) typetraits.name(O) & ": Unknown field: " & escape(name))
else: else:
when hasIgnore(O) and failOnUnknown: when hasIgnore(O) and failOnUnknown:
if name notin ignoredKeyList: if name notin ignoredKeyList:
constructFieldValue(O, s, c, name, result, matched, failOnUnknown) constructFieldValue(O, s, c, name, result, matched, failOnUnknown, e.startPos)
else: else:
e = s.next() e = s.next()
var depth = int(e.kind in {yamlStartMap, yamlStartSeq}) var depth = int(e.kind in {yamlStartMap, yamlStartSeq})
@ -906,21 +907,21 @@ proc constructObjectDefault*[O: object|tuple](
of yamlScalar: discard of yamlScalar: discard
else: internalError("Unexpected event kind.") else: internalError("Unexpected event kind.")
else: else:
constructFieldValue(O, s, c, name, result, matched, failOnUnknown) constructFieldValue(O, s, c, name, result, matched, failOnUnknown, e.startPos)
when isVariantObject(getType(O)): when isVariantObject(getType(O)):
e = s.next() e = s.next()
if e.kind != yamlEndMap: if e.kind != yamlEndMap:
raise s.constructionError("Expected end of single-pair map, got " & raise s.constructionError(e.startPos, "Expected end of single-pair map, got " &
$e.kind) $e.kind)
discard s.next() discard s.next()
when result is tuple: when result is tuple:
var i = 0 var i = 0
for fname, value in fieldPairs(result): for fname, value in fieldPairs(result):
if not matched[i]: if not matched[i]:
raise s.constructionError("While constructing " & raise s.constructionError(e.startPos, "While constructing " &
typetraits.name(O) & ": Missing field: " & escape(fname)) typetraits.name(O) & ": Missing field: " & escape(fname))
inc(i) inc(i)
else: ensureAllFieldsPresent(s, O, result, matched) else: ensureAllFieldsPresent(s, O, result, matched, startPos)
proc constructObject*[O: object|tuple]( proc constructObject*[O: object|tuple](
s: var YamlStream, c: ConstructionContext, result: var O) s: var YamlStream, c: ConstructionContext, result: var O)
@ -1001,8 +1002,8 @@ proc representObject*[O: object](value: O, ts: TagStyle,
c: SerializationContext, tag: TagId) = c: SerializationContext, tag: TagId) =
## represents a Nim object or tuple as YAML mapping ## represents a Nim object or tuple as YAML mapping
let childTagStyle = if ts == tsRootOnly: tsNone else: ts let childTagStyle = if ts == tsRootOnly: tsNone else: ts
when isVariantObject(getType(O)): c.put(startSeqEvent(tag, yAnchorNone)) when isVariantObject(getType(O)): c.put(startSeqEvent(csBlock, (yAnchorNone, tag)))
else: c.put(startMapEvent(tag, yAnchorNone)) else: c.put(startMapEvent(csBlock, (yAnchorNone, tag)))
genRepresentObject(O, value, childTagStyle) genRepresentObject(O, value, childTagStyle)
when isVariantObject(getType(O)): c.put(endSeqEvent()) when isVariantObject(getType(O)): c.put(endSeqEvent())
else: c.put(endMapEvent()) else: c.put(endMapEvent())
@ -1025,10 +1026,10 @@ proc constructObject*[O: enum](s: var YamlStream, c: ConstructionContext,
## constructs a Nim enum from a YAML scalar ## constructs a Nim enum from a YAML scalar
let e = s.next() let e = s.next()
if e.kind != yamlScalar: if e.kind != yamlScalar:
raise s.constructionError("Expected scalar, got " & $e.kind) raise s.constructionError(e.startPos, "Expected scalar, got " & $e.kind)
try: result = parseEnum[O](e.scalarContent) try: result = parseEnum[O](e.scalarContent)
except ValueError: except ValueError:
var ex = s.constructionError("Cannot parse '" & var ex = s.constructionError(e.startPos, "Cannot parse '" &
escape(e.scalarContent) & "' as " & type(O).name) escape(e.scalarContent) & "' as " & type(O).name)
ex.parent = getCurrentException() ex.parent = getCurrentException()
raise ex raise ex
@ -1128,7 +1129,7 @@ proc constructChild*[T](s: var YamlStream, c: ConstructionContext,
of yTypeBoolTrue, yTypeBoolFalse: of yTypeBoolTrue, yTypeBoolFalse:
possibleTagIds.add(yamlTag(bool)) possibleTagIds.add(yamlTag(bool))
of yTypeNull: of yTypeNull:
raise s.constructionError("not implemented!") raise s.constructionError(item.startPos, "not implemented!")
of yTypeUnknown: of yTypeUnknown:
possibleTagIds.add(yamlTag(string)) possibleTagIds.add(yamlTag(string))
of yTypeTimestamp: of yTypeTimestamp:
@ -1139,12 +1140,12 @@ proc constructChild*[T](s: var YamlStream, c: ConstructionContext,
possibleTagIds.add(item.scalarTag) possibleTagIds.add(item.scalarTag)
of yamlStartMap: of yamlStartMap:
if item.mapTag in [yTagQuestionMark, yTagExclamationMark]: if item.mapTag in [yTagQuestionMark, yTagExclamationMark]:
raise s.constructionError( raise s.constructionError(item.startPos,
"Complex value of implicit variant object type must have a tag.") "Complex value of implicit variant object type must have a tag.")
possibleTagIds.add(item.mapTag) possibleTagIds.add(item.mapTag)
of yamlStartSeq: of yamlStartSeq:
if item.seqTag in [yTagQuestionMark, yTagExclamationMark]: if item.seqTag in [yTagQuestionMark, yTagExclamationMark]:
raise s.constructionError( raise s.constructionError(item.startPos,
"Complex value of implicit variant object type must have a tag.") "Complex value of implicit variant object type must have a tag.")
possibleTagIds.add(item.seqTag) possibleTagIds.add(item.seqTag)
else: internalError("Unexpected item kind: " & $item.kind) else: internalError("Unexpected item kind: " & $item.kind)
@ -1152,21 +1153,21 @@ proc constructChild*[T](s: var YamlStream, c: ConstructionContext,
else: else:
case item.kind case item.kind
of yamlScalar: of yamlScalar:
if item.scalarTag notin [yTagQuestionMark, yTagExclamationMark, if item.scalarProperties.tag notin [yTagQuestionMark, yTagExclamationMark,
yamlTag(T)]: yamlTag(T)]:
raise s.constructionError("Wrong tag for " & typetraits.name(T)) raise s.constructionError(item.startPos, "Wrong tag for " & typetraits.name(T))
elif item.scalarAnchor != yAnchorNone: elif item.scalarProperties.anchor != yAnchorNone:
raise s.constructionError("Anchor on non-ref type") raise s.constructionError(item.startPos, "Anchor on non-ref type")
of yamlStartMap: of yamlStartMap:
if item.mapTag notin [yTagQuestionMark, yamlTag(T)]: if item.mapProperties.tag notin [yTagQuestionMark, yamlTag(T)]:
raise s.constructionError("Wrong tag for " & typetraits.name(T)) raise s.constructionError(item.startPos, "Wrong tag for " & typetraits.name(T))
elif item.mapAnchor != yAnchorNone: elif item.mapProperties.anchor != yAnchorNone:
raise s.constructionError("Anchor on non-ref type") raise s.constructionError(item.startPos, "Anchor on non-ref type")
of yamlStartSeq: of yamlStartSeq:
if item.seqTag notin [yTagQuestionMark, yamlTag(T)]: if item.seqProperties.tag notin [yTagQuestionMark, yamlTag(T)]:
raise s.constructionError("Wrong tag for " & typetraits.name(T)) raise s.constructionError(item.startPos, "Wrong tag for " & typetraits.name(T))
elif item.seqAnchor != yAnchorNone: elif item.seqProperties.anchor != yAnchorNone:
raise s.constructionError("Anchor on non-ref type") raise s.constructionError(item.startPos, "Anchor on non-ref type")
else: internalError("Unexpected item kind: " & $item.kind) else: internalError("Unexpected item kind: " & $item.kind)
constructObject(s, c, result) constructObject(s, c, result)
@ -1176,19 +1177,19 @@ proc constructChild*(s: var YamlStream, c: ConstructionContext,
if item.kind == yamlScalar: if item.kind == yamlScalar:
if item.scalarProperties.tag notin if item.scalarProperties.tag notin
[yTagQuestionMark, yTagExclamationMark, yamlTag(string)]: [yTagQuestionMark, yTagExclamationMark, yamlTag(string)]:
raise s.constructionError("Wrong tag for string") raise s.constructionError(item.startPos, "Wrong tag for string")
elif item.scalarProperties.anchor != yAnchorNone: elif item.scalarProperties.anchor != yAnchorNone:
raise s.constructionError("Anchor on non-ref type") raise s.constructionError(item.startPos, "Anchor on non-ref type")
constructObject(s, c, result) constructObject(s, c, result)
proc constructChild*[T](s: var YamlStream, c: ConstructionContext, proc constructChild*[T](s: var YamlStream, c: ConstructionContext,
result: var seq[T]) = result: var seq[T]) =
let item = s.peek() let item = s.peek()
if item.kind == yamlStartSeq: if item.kind == yamlStartSeq:
if item.seqTag notin [yTagQuestionMark, yamlTag(seq[T])]: if item.seqProperties.tag notin [yTagQuestionMark, yamlTag(seq[T])]:
raise s.constructionError("Wrong tag for " & typetraits.name(seq[T])) raise s.constructionError(item.startPos, "Wrong tag for " & typetraits.name(seq[T]))
elif item.seqAnchor != yAnchorNone: elif item.seqProperties.anchor != yAnchorNone:
raise s.constructionError("Anchor on non-ref type") raise s.constructionError(item.startPos, "Anchor on non-ref type")
constructObject(s, c, result) constructObject(s, c, result)
proc constructChild*[T](s: var YamlStream, c: ConstructionContext, proc constructChild*[T](s: var YamlStream, c: ConstructionContext,
@ -1211,21 +1212,22 @@ when defined(JS):
result: var Time) = result: var Time) =
let e = s.peek() let e = s.peek()
if e.kind == yamlScalar: if e.kind == yamlScalar:
if e.scalarTag notin [yTagQuestionMark, yTagTimestamp]: if e.scalarProperties.tag notin [yTagQuestionMark, yTagTimestamp]:
raise s.constructionError("Wrong tag for Time") raise s.constructionError(e.startPos, "Wrong tag for Time")
elif guessType(e.scalarContent) != yTypeTimestamp: elif guessType(e.scalarContent) != yTypeTimestamp:
raise s.constructionError("Invalid timestamp") raise s.constructionError(e.startPos, "Invalid timestamp")
elif e.scalarAnchor != yAnchorNone: elif e.scalarProperties.anchor != yAnchorNone:
raise s.constructionError("Anchor on non-ref type") raise s.constructionError(e.startPos, "Anchor on non-ref type")
constructObject(s, c, result) constructObject(s, c, result)
else: else:
raise s.constructionError("Unexpected structure, expected timestamp") raise s.constructionError(e.startPos, "Unexpected structure, expected timestamp")
proc constructChild*[O](s: var YamlStream, c: ConstructionContext, proc constructChild*[O](s: var YamlStream, c: ConstructionContext,
result: var ref O) = result: var ref O) =
var e = s.peek() var e = s.peek()
if e.kind == yamlScalar: if e.kind == yamlScalar:
if e.scalarTag == yTagNull or (e.scalarTag == yTagQuestionMark and let props = e.scalarProperties
if props.tag == yTagNull or (props.tag == yTagQuestionMark and
guessType(e.scalarContent) == yTypeNull): guessType(e.scalarContent) == yTypeNull):
result = nil result = nil
discard s.next() discard s.next()
@ -1359,11 +1361,15 @@ proc construct*[T](s: var YamlStream, target: var T)
var context = newConstructionContext() var context = newConstructionContext()
try: try:
var e = s.next() var e = s.next()
yAssert(e.kind == yamlStartStream)
e = s.next()
yAssert(e.kind == yamlStartDoc) yAssert(e.kind == yamlStartDoc)
constructChild(s, context, target) constructChild(s, context, target)
e = s.next() e = s.next()
yAssert(e.kind == yamlEndDoc) yAssert(e.kind == yamlEndDoc)
e = s.next()
yAssert(e.kind == yamlEndStream)
except YamlConstructionError: except YamlConstructionError:
raise (ref YamlConstructionError)(getCurrentException()) raise (ref YamlConstructionError)(getCurrentException())
except YamlStreamError: except YamlStreamError:
@ -1430,9 +1436,9 @@ proc represent*[T](value: T, ts: TagStyle = tsRootOnly,
if a == asTidy: if a == asTidy:
for item in bys.mitems(): for item in bys.mitems():
case item.kind case item.kind
of yamlStartMap: setAnchor(item.mapAnchor, context) of yamlStartMap: setAnchor(item.mapProperties.anchor, context)
of yamlStartSeq: setAnchor(item.seqAnchor, context) of yamlStartSeq: setAnchor(item.seqProperties.anchor, context)
of yamlScalar: setAnchor(item.scalarAnchor, context) of yamlScalar: setAnchor(item.scalarProperties.anchor, context)
else: discard else: discard
result = bys result = bys

14
yaml/stream.nim
View File

@ -33,8 +33,7 @@ type
## well-formed if they take it as input parameter. ## well-formed if they take it as input parameter.
nextImpl*: proc(s: YamlStream, e: var Event): bool nextImpl*: proc(s: YamlStream, e: var Event): bool
lastTokenContextImpl*: lastTokenContextImpl*:
proc(s: YamlStream, line, column: var int, proc(s: YamlStream, lineContent: var string): bool {.raises: [].}
lineContent: var string): bool {.raises: [].}
peeked: bool peeked: bool
cached: Event cached: Event
@ -43,13 +42,11 @@ type
## backend raises an exception. The error that has occurred is ## backend raises an exception. The error that has occurred is
## available from ``parent``. ## available from ``parent``.
proc noLastContext(s: YamlStream, line, column: var int, proc noLastContext(s: YamlStream, lineContent: var string): bool {.raises: [].} =
lineContent: var string): bool {.raises: [].} =
(line, column, lineContent) = (-1, -1, "")
result = false result = false
proc basicInit*(s: YamlStream, lastTokenContextImpl: proc basicInit*(s: YamlStream, lastTokenContextImpl:
proc(s: YamlStream, line, column: var int, lineContent: var string): bool proc(s: YamlStream, lineContent: var string): bool
{.raises: [].} = noLastContext) {.raises: [].} = {.raises: [].} = noLastContext) {.raises: [].} =
## initialize basic values of the YamlStream. Call this in your constructor ## initialize basic values of the YamlStream. Call this in your constructor
## if you subclass YamlStream. ## if you subclass YamlStream.
@ -121,12 +118,11 @@ proc `peek=`*(s: YamlStream, value: Event) {.raises: [].} =
s.cached = value s.cached = value
s.peeked = true s.peeked = true
proc getLastTokenContext*(s: YamlStream, line, column: var int, proc getLastTokenContext*(s: YamlStream, lineContent: var string): bool =
lineContent: var string): bool =
## ``true`` if source context information is available about the last returned ## ``true`` if source context information is available about the last returned
## token. If ``true``, line, column and lineContent are set to position and ## token. If ``true``, line, column and lineContent are set to position and
## line content where the last token has been read from. ## line content where the last token has been read from.
result = s.lastTokenContextImpl(s, line, column, lineContent) result = s.lastTokenContextImpl(s, lineContent)
iterator items*(s: YamlStream): Event iterator items*(s: YamlStream): Event
{.raises: [YamlStreamError].} = {.raises: [YamlStreamError].} =

42
yaml/taglib.nim
View File

@ -30,16 +30,21 @@ type
## `initExtendedTagLibrary <#initExtendedTagLibrary>`_. ## `initExtendedTagLibrary <#initExtendedTagLibrary>`_.
tags*: Table[string, TagId] tags*: Table[string, TagId]
nextCustomTagId*: TagId nextCustomTagId*: TagId
tagHandles: Table[string, string] prefixes: seq[tuple[prefix, uri: string]]
proc initTagLibrary*(): TagLibrary {.raises: [].} = proc initTagLibrary*(): TagLibrary {.raises: [].} =
## initializes the ``tags`` table and sets ``nextCustomTagId`` to ## initializes the ``tags`` table and sets ``nextCustomTagId`` to
## ``yFirstCustomTagId``. ## ``yFirstCustomTagId``.
new(result) new(result)
result.tags = initTable[string, TagId]() result.tags = initTable[string, TagId]()
result.tagHandles = {"!": "!", yamlTagRepositoryPrefix : "!!"}.toTable() result.prefixes = @[("!", "!"), ("!!", yamlTagRepositoryPrefix)]
result.nextCustomTagId = yFirstCustomTagId result.nextCustomTagId = yFirstCustomTagId
proc resetPrefixes*(tagLib: TagLibrary) {.raises: [].} =
## resets the registered prefixes in the given tag library, so that it
## only contains the prefixes `!` and `!!`.
tagLib.prefixes.setLen(2)
proc registerUri*(tagLib: TagLibrary, uri: string): TagId {.raises: [].} = proc registerUri*(tagLib: TagLibrary, uri: string): TagId {.raises: [].} =
## registers a custom tag URI with a ``TagLibrary``. The URI will get ## registers a custom tag URI with a ``TagLibrary``. The URI will get
## the ``TagId`` ``nextCustomTagId``, which will be incremented. ## the ``TagId`` ``nextCustomTagId``, which will be incremented.
@ -103,7 +108,7 @@ proc initExtendedTagLibrary*(): TagLibrary {.raises: [].} =
proc initSerializationTagLibrary*(): TagLibrary = proc initSerializationTagLibrary*(): TagLibrary =
result = initTagLibrary() result = initTagLibrary()
result.tagHandles[nimyamlTagRepositoryPrefix] = "!n!" result.prefixes.add(("!n!", nimyamlTagRepositoryPrefix))
result.tags["!"] = yTagExclamationMark result.tags["!"] = yTagExclamationMark
result.tags["?"] = yTagQuestionMark result.tags["?"] = yTagQuestionMark
result.tags[y"str"] = yTagString result.tags[y"str"] = yTagString
@ -190,11 +195,19 @@ setTagUri(uint64, n"system:uint64", yTagNimUInt64)
setTagUri(float32, n"system:float32", yTagNimFloat32) setTagUri(float32, n"system:float32", yTagNimFloat32)
setTagUri(float64, n"system:float64", yTagNimFloat64) setTagUri(float64, n"system:float64", yTagNimFloat64)
proc registerHandle*(tagLib: TagLibrary, handle, prefix: string) = proc registerHandle*(tagLib: TagLibrary, uri, prefix: string): bool =
## Registers a handle for a prefix. When presenting any tag that starts with ## Registers a handle for a prefix. When presenting any tag that starts with
## this prefix, the handle is used instead. Also causes the presenter to ## this prefix, the handle is used instead. Also causes the presenter to
## output a TAG directive for the handle. ## output a TAG directive for the handle.
taglib.tagHandles[prefix] = handle ##
## Returns true iff a new item has been created, false if an existing item
## has been updated.
for i in countup(0, len(tagLib.prefixes)-1):
if tagLib.prefixes[i].prefix == prefix:
tagLib.prefixes[i].uri = uri
return false
taglib.prefixes.add((prefix, uri))
return false
proc searchHandle*(tagLib: TagLibrary, tag: string): proc searchHandle*(tagLib: TagLibrary, tag: string):
tuple[handle: string, len: int] {.raises: [].} = tuple[handle: string, len: int] {.raises: [].} =
@ -203,22 +216,25 @@ proc searchHandle*(tagLib: TagLibrary, tag: string):
## longest prefix is returned. If no registered handle matches, (nil, 0) is ## longest prefix is returned. If no registered handle matches, (nil, 0) is
## returned. ## returned.
result.len = 0 result.len = 0
for key, value in tagLib.tagHandles: for item in tagLib.prefixes:
if key.len > result.len: if item.uri.len > result.len:
if tag.startsWith(key): if tag.startsWith(item.uri):
result.len = key.len result.len = item.uri.len
result.handle = value result.handle = item.prefix
proc resolve*(tagLib: TagLibrary, handle: string): string {.raises: [].} = proc resolve*(tagLib: TagLibrary, handle: string): string {.raises: [].} =
## try to resolve the given tag handle. ## try to resolve the given tag handle.
## return the registered URI if the tag handle is found. ## return the registered URI if the tag handle is found.
## if the handle is unknown, return the empty string. ## if the handle is unknown, return the empty string.
return tagLib.tagHandles.getOrDefault(handle, "") for item in tagLib.prefixes:
if item.prefix == handle:
return item.uri
return ""
iterator handles*(tagLib: TagLibrary): tuple[prefix, handle: string] = iterator handles*(tagLib: TagLibrary): tuple[prefix, uri: string] =
## iterate over registered tag handles that may be used as shortcuts ## iterate over registered tag handles that may be used as shortcuts
## (e.g. ``!n!`` for ``tag:nimyaml.org,2016:``) ## (e.g. ``!n!`` for ``tag:nimyaml.org,2016:``)
for key, value in tagLib.tagHandles: yield (key, value) for item in tagLib.prefixes.items(): yield item
proc nimTag*(suffix: string): string = proc nimTag*(suffix: string): string =
## prepends NimYAML's tag repository prefix to the given suffix. For example, ## prepends NimYAML's tag repository prefix to the given suffix. For example,