status-im
/
NimYAML
mirror of https://github.com/status-im/NimYAML.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

made lexer & parser tests compile (not succeed) again

This commit is contained in:
Felix Krause 2020-11-04 16:40:37 +01:00
parent 1d707b184e
commit 4c604b09df
9 changed files with 230 additions and 300 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
test/commonTestUtils.nim
View File

@ -4,7 +4,7 @@
# See the file "copying.txt", included in this # See the file "copying.txt", included in this
# distribution, for details about the copyright. # distribution, for details about the copyright.
import "../yaml", strutils import ../yaml, ../yaml/data
proc escapeNewlines(s: string): string = proc escapeNewlines(s: string): string =
result = "" result = ""
@ -15,46 +15,42 @@ proc escapeNewlines(s: string): string =
of '\\': result.add("\\\\") of '\\': result.add("\\\\")
else: result.add(c) else: result.add(c)
proc printDifference*(expected, actual: YamlStreamEvent) = proc printDifference(expected, actual: Properties): bool =
result = false
if expected.tag != actual.tag:
echo "[scalar.tag] expected", $expected.tag, ", got", $actual.tag
result = true
if expected.anchor != actual.anchor:
echo "[scalar.anchor] expected", $expected.anchor, ", got", $actual.anchor
result = true
proc printDifference*(expected, actual: Event) =
if expected.kind != actual.kind: if expected.kind != actual.kind:
echo "expected " & $expected.kind & ", got " & $actual.kind echo "expected " & $expected.kind & ", got " & $actual.kind
else: else:
case expected.kind case expected.kind
of yamlScalar: of yamlScalar:
if expected.scalarTag != actual.scalarTag: if not printDifference(expected.scalarProperties, actual.scalarProperties):
echo "[", escape(actual.scalarContent), ".tag] expected tag ", if expected.scalarContent != actual.scalarContent:
expected.scalarTag, ", got ", actual.scalarTag let msg = "[scalarEvent] content mismatch!\nexpected: " &
elif expected.scalarAnchor != actual.scalarAnchor: escapeNewlines(expected.scalarContent) &
echo "[scalarEvent] expected anchor ", expected.scalarAnchor, "\ngot : " & escapeNewlines(actual.scalarContent)
", got ", actual.scalarAnchor if expected.scalarContent.len != actual.scalarContent.len:
elif expected.scalarContent != actual.scalarContent: echo msg, "\n(length does not match)"
let msg = "[scalarEvent] content mismatch!\nexpected: " & else:
escapeNewlines(expected.scalarContent) & for i in 0..expected.scalarContent.high:
"\ngot : " & escapeNewlines(actual.scalarContent) if expected.scalarContent[i] != actual.scalarContent[i]:
if expected.scalarContent.len != actual.scalarContent.len: echo msg, "\n(first different char at pos ", i, ": expected ",
echo msg, "\n(length does not match)" cast[int](expected.scalarContent[i]), ", got ",
else: cast[int](actual.scalarContent[i]), ")"
for i in 0..expected.scalarContent.high: break
if expected.scalarContent[i] != actual.scalarContent[i]: else: echo "[scalar] Unknown difference"
echo msg, "\n(first different char at pos ", i, ": expected ",
cast[int](expected.scalarContent[i]), ", got ",
cast[int](actual.scalarContent[i]), ")"
break
else: echo "[scalarEvent] Unknown difference"
of yamlStartMap: of yamlStartMap:
if expected.mapTag != actual.mapTag: if not printDifference(expected.mapProperties, actual.mapProperties):
echo "[map.tag] expected ", expected.mapTag, ", got ", actual.mapTag echo "[map] Unknown difference"
elif expected.mapAnchor != actual.mapAnchor:
echo "[map.anchor] expected ", expected.mapAnchor, ", got ",
actual.mapAnchor
else: echo "[map.tag] Unknown difference"
of yamlStartSeq: of yamlStartSeq:
if expected.seqTag != actual.seqTag: if not printDifference(expected.seqProperties, actual.seqProperties):
echo "[seq.tag] expected ", expected.seqTag, ", got ", actual.seqTag echo "[seq] Unknown difference"
elif expected.seqAnchor != actual.seqAnchor:
echo "[seq.anchor] expected ", expected.seqAnchor, ", got ",
actual.seqAnchor
else: echo "[seq] Unknown difference"
of yamlAlias: of yamlAlias:
if expected.aliasTarget != actual.aliasTarget: if expected.aliasTarget != actual.aliasTarget:
echo "[alias] expected ", expected.aliasTarget, ", got ", echo "[alias] expected ", expected.aliasTarget, ", got ",
@ -63,7 +59,7 @@ proc printDifference*(expected, actual: YamlStreamEvent) =
else: echo "Unknown difference in event kind " & $expected.kind else: echo "Unknown difference in event kind " & $expected.kind
template ensure*(input: var YamlStream, template ensure*(input: var YamlStream,
expected: varargs[YamlStreamEvent]) {.dirty.} = expected: varargs[Event]) {.dirty.} =
var i = 0 var i = 0
for token in input: for token in input:
if i >= expected.len: if i >= expected.len:

68
test/testEventParser.nim
View File

@ -4,7 +4,7 @@
# See the file "copying.txt", included in this # See the file "copying.txt", included in this
# distribution, for details about the copyright. # distribution, for details about the copyright.
import "../yaml" import ../yaml, ../yaml/data, ../yaml/private/internal
import lexbase, streams, tables, strutils import lexbase, streams, tables, strutils
type type
@ -19,7 +19,7 @@ type
EventLexer = object of BaseLexer EventLexer = object of BaseLexer
content: string content: string
EventStreamError = object of Exception EventStreamError = object of ValueError
proc nextToken(lex: var EventLexer): LexerToken = proc nextToken(lex: var EventLexer): LexerToken =
while true: while true:
@ -116,75 +116,74 @@ template yieldEvent() {.dirty.} =
template setTag(t: TagId) {.dirty.} = template setTag(t: TagId) {.dirty.} =
case curEvent.kind case curEvent.kind
of yamlStartSeq: curEvent.seqTag = t of yamlStartSeq: curEvent.seqProperties.tag = t
of yamlStartMap: curEvent.mapTag = t of yamlStartMap: curEvent.mapProperties.tag = t
of yamlScalar: curEvent.scalarTag = t of yamlScalar: curEvent.scalarProperties.tag = t
else: discard else: discard
template setAnchor(a: AnchorId) {.dirty.} = template setAnchor(a: Anchor) {.dirty.} =
case curEvent.kind case curEvent.kind
of yamlStartSeq: curEvent.seqAnchor = a of yamlStartSeq: curEvent.seqProperties.anchor = a
of yamlStartMap: curEvent.mapAnchor = a of yamlStartMap: curEvent.mapProperties.anchor = a
of yamlScalar: curEvent.scalarAnchor = a of yamlScalar: curEvent.scalarProperties.anchor = a
of yamlAlias: curEvent.aliasTarget = a of yamlAlias: curEvent.aliasTarget = a
else: discard else: discard
template curTag(): TagId = template curTag(): TagId =
var foo: TagId var foo: TagId
case curEvent.kind case curEvent.kind
of yamlStartSeq: foo = curEvent.seqTag of yamlStartSeq: foo = curEvent.seqProperties.tag
of yamlStartMap: foo = curEvent.mapTag of yamlStartMap: foo = curEvent.mapProperties.tag
of yamlScalar: foo = curEvent.scalarTag of yamlScalar: foo = curEvent.scalarProperties.tag
else: raise newException(EventStreamError, else: raise newException(EventStreamError,
$curEvent.kind & " may not have a tag") $curEvent.kind & " may not have a tag")
foo foo
template setCurTag(val: TagId) = template setCurTag(val: TagId) =
case curEvent.kind case curEvent.kind
of yamlStartSeq: curEvent.seqTag = val of yamlStartSeq: curEvent.seqProperties.tag = val
of yamlStartMap: curEvent.mapTag = val of yamlStartMap: curEvent.mapProperties.tag = val
of yamlScalar: curEvent.scalarTag = val of yamlScalar: curEvent.scalarProperties.tag = val
else: raise newException(EventStreamError, else: raise newException(EventStreamError,
$curEvent.kind & " may not have a tag") $curEvent.kind & " may not have a tag")
template curAnchor(): AnchorId = template curAnchor(): Anchor =
var foo: AnchorId var foo: Anchor
case curEvent.kind case curEvent.kind
of yamlStartSeq: foo = curEvent.seqAnchor of yamlStartSeq: foo = curEvent.seqProperties.anchor
of yamlStartMap: foo = curEvent.mapAnchor of yamlStartMap: foo = curEvent.mapProperties.anchor
of yamlScalar: foo = curEvent.scalarAnchor of yamlScalar: foo = curEvent.scalarProperties.anchor
of yamlAlias: foo = curEvent.aliasTarget of yamlAlias: foo = curEvent.aliasTarget
else: raise newException(EventStreamError, else: raise newException(EventStreamError,
$curEvent.kind & "may not have an anchor") $curEvent.kind & "may not have an anchor")
foo foo
template setCurAnchor(val: AnchorId) = template setCurAnchor(val: Anchor) =
case curEvent.kind case curEvent.kind
of yamlStartSeq: curEvent.seqAnchor = val of yamlStartSeq: curEvent.seqProperties.anchor = val
of yamlStartMap: curEvent.mapAnchor = val of yamlStartMap: curEvent.mapProperties.anchor = val
of yamlScalar: curEvent.scalarAnchor = val of yamlScalar: curEvent.scalarProperties.anchor = val
of yamlAlias: curEvent.aliasTarget = val of yamlAlias: curEvent.aliasTarget = val
else: raise newException(EventStreamError, else: raise newException(EventStreamError,
$curEvent.kind & " may not have an anchor") $curEvent.kind & " may not have an anchor")
template eventStart(k: YamlStreamEventKind) {.dirty.} = template eventStart(k: EventKind) {.dirty.} =
assertInStream() assertInStream()
yieldEvent() yieldEvent()
curEvent = YamlStreamEvent(kind: k) curEvent = Event(kind: k)
setTag(yTagQuestionMark) setTag(yTagQuestionMark)
setAnchor(yAnchorNone) setAnchor(yAnchorNone)
inEvent = true inEvent = true
proc parseEventStream*(input: Stream, tagLib: TagLibrary): YamlStream = proc parseEventStream*(input: Stream, tagLib: TagLibrary): YamlStream =
var backend = iterator(): YamlStreamEvent = var backend = iterator(): Event =
var lex: EventLexer var lex: EventLexer
lex.open(input) lex.open(input)
var var
inEvent = false inEvent = false
curEvent: YamlStreamEvent curEvent: Event
streamPos: StreamPos = beforeStream streamPos: StreamPos = beforeStream
anchors = initTable[string, AnchorId]() nextAnchorId = "a"
nextAnchorId = 0.AnchorId
while lex.buf[lex.bufpos] != EndOfFile: while lex.buf[lex.bufpos] != EndOfFile:
let token = lex.nextToken() let token = lex.nextToken()
case token case token
@ -192,12 +191,14 @@ proc parseEventStream*(input: Stream, tagLib: TagLibrary): YamlStream =
if streamPos != beforeStream: if streamPos != beforeStream:
raise newException(EventStreamError, "Illegal +STR") raise newException(EventStreamError, "Illegal +STR")
streamPos = inStream streamPos = inStream
eventStart(yamlStartStream)
of minusStr: of minusStr:
if streamPos != inStream: if streamPos != inStream:
raise newException(EventStreamError, "Illegal -STR") raise newException(EventStreamError, "Illegal -STR")
if inEvent: yield curEvent if inEvent: yield curEvent
inEvent = false inEvent = false
streamPos = afterStream streamPos = afterStream
eventStart(yamlEndStream)
of plusDoc: eventStart(yamlStartDoc) of plusDoc: eventStart(yamlStartDoc)
of minusDoc: eventStart(yamlEndDoc) of minusDoc: eventStart(yamlEndDoc)
of plusMap: eventStart(yamlStartMap) of plusMap: eventStart(yamlStartMap)
@ -219,9 +220,8 @@ proc parseEventStream*(input: Stream, tagLib: TagLibrary): YamlStream =
if curAnchor() != yAnchorNone: if curAnchor() != yAnchorNone:
raise newException(EventStreamError, raise newException(EventStreamError,
"Duplicate anchor in " & $curEvent.kind) "Duplicate anchor in " & $curEvent.kind)
anchors[lex.content] = nextAnchorId setCurAnchor(nextAnchorId.Anchor)
setCurAnchor(nextAnchorId) nextAnchor(nextAnchorId, len(nextAnchorId))
nextAnchorId = (AnchorId)(((int)nextAnchorId) + 1)
of starAnchor: of starAnchor:
assertInEvent("alias") assertInEvent("alias")
if curEvent.kind != yamlAlias: if curEvent.kind != yamlAlias:
@ -231,7 +231,7 @@ proc parseEventStream*(input: Stream, tagLib: TagLibrary): YamlStream =
raise newException(EventStreamError, "Duplicate alias target: " & raise newException(EventStreamError, "Duplicate alias target: " &
escape(lex.content)) escape(lex.content))
else: else:
curEvent.aliasTarget = anchors[lex.content] curEvent.aliasTarget = lex.content.Anchor
of quotContent: of quotContent:
assertInEvent("scalar content") assertInEvent("scalar content")
if curTag() == yTagQuestionMark: setCurTag(yTagExclamationMark) if curTag() == yTagQuestionMark: setCurTag(yTagExclamationMark)

236
test/tlex.nim
View File

@ -3,36 +3,34 @@ import ../yaml/private/lex
import unittest, strutils import unittest, strutils
const tokensWithValue = const tokensWithValue =
{ltScalarPart, ltQuotedScalar, ltYamlVersion, ltTagShorthand, ltTagUri, {Token.Plain, Token.SingleQuoted, Token.DoubleQuoted, Token.Literal,
ltUnknownDirective, ltUnknownDirectiveParams, ltLiteralTag, ltAnchor, Token.Folded, Token.DirectiveParam,
ltAlias, ltBlockScalar} Token.TagHandle, Token.Suffix, Token.VerbatimTag,
Token.UnknownDirective, Token.Anchor, Token.Alias}
type type
TokenWithValue = object TokenWithValue = object
case kind: LexerToken case kind: Token
of tokensWithValue: of tokensWithValue:
value: string value: string
of ltIndentation: of Indentation:
indentation: int indentation: int
of ltTagHandle:
handle, suffix: string
else: discard else: discard
proc actualRepr(lex: YamlLexer, t: LexerToken): string = proc actualRepr(lex: Lexer, t: Token): string =
result = $t result = $t
case t case t
of tokensWithValue + {ltTagHandle}: of tokensWithValue + {Token.TagHandle}:
result.add("(" & escape(lex.buf) & ")") result.add("(" & escape(lex.evaluated) & ")")
of ltIndentation: of Indentation:
result.add("(" & $lex.indentation & ")") result.add("(" & $lex.indentation & ")")
else: discard else: discard
proc assertEquals(input: string, expected: varargs[TokenWithValue]) = proc assertEquals(input: string, expected: varargs[TokenWithValue]) =
let lex = newYamlLexer(input)
var var
lex: Lexer
i = 0 i = 0
blockScalarEnd = -1 lex.init(input)
flowDepth = 0
for expectedToken in expected: for expectedToken in expected:
inc(i) inc(i)
try: try:
@ -42,176 +40,135 @@ proc assertEquals(input: string, expected: varargs[TokenWithValue]) =
lex.actualRepr(lex.cur) lex.actualRepr(lex.cur)
case expectedToken.kind case expectedToken.kind
of tokensWithValue: of tokensWithValue:
doAssert lex.buf == expectedToken.value, "Wrong token content at #" & doAssert lex.evaluated == expectedToken.value, "Wrong token content at #" &
$i & ": Expected " & escape(expectedToken.value) & $i & ": Expected " & escape(expectedToken.value) &
", got " & escape(lex.buf) ", got " & escape(lex.evaluated)
lex.buf = "" of Indentation:
of ltIndentation:
doAssert lex.indentation == expectedToken.indentation, doAssert lex.indentation == expectedToken.indentation,
"Wrong indentation length at #" & $i & ": Expected " & "Wrong indentation length at #" & $i & ": Expected " &
$expectedToken.indentation & ", got " & $lex.indentation $expectedToken.indentation & ", got " & $lex.indentation
if lex.indentation <= blockScalarEnd:
lex.endBlockScalar()
blockScalarEnd = -1
of ltBraceOpen, ltBracketOpen:
inc(flowDepth)
if flowDepth == 1: lex.setFlow(true)
of ltBraceClose, ltBracketClose:
dec(flowDepth)
if flowDepth == 0: lex.setFlow(false)
of ltTagHandle:
let
handle = lex.buf.substr(0, lex.shorthandEnd)
suffix = lex.buf.substr(lex.shorthandEnd + 1)
doAssert handle == expectedToken.handle,
"Wrong handle at #" & $i & ": Expected " & expectedToken.handle &
", got " & handle
doAssert suffix == expectedToken.suffix,
"Wrong suffix at #" & $i & ": Expected " & expectedToken.suffix &
", got " & suffix
lex.buf = ""
else: discard else: discard
except YamlLexerError: except LexerError:
let e = (ref YamlLexerError)(getCurrentException()) let e = (ref LexerError)(getCurrentException())
echo "Error at line " & $e.line & ", column " & $e.column & ":" echo "Error at line " & $e.line & ", column " & $e.column & ":"
echo e.lineContent echo e.lineContent
assert false assert false
proc assertLookahead(input: string, expected: bool, tokensBefore: int = 1) =
let lex = newYamlLexer(input)
var flowDepth = 0
for i in 0..tokensBefore:
lex.next()
case lex.cur
of ltBraceOpen, ltBracketOpen:
inc(flowDepth)
if flowDepth == 1: lex.setFlow(true)
of ltBraceClose, ltBracketClose:
dec(flowDepth)
if flowDepth == 0: lex.setFlow(false)
else: discard
doAssert lex.isImplicitKeyStart() == expected
proc i(indent: int): TokenWithValue = proc i(indent: int): TokenWithValue =
TokenWithValue(kind: ltIndentation, indentation: indent) TokenWithValue(kind: Token.Indentation, indentation: indent)
proc sp(v: string): TokenWithValue = proc pl(v: string): TokenWithValue =
TokenWithValue(kind: ltScalarPart, value: v) TokenWithValue(kind: Token.Plain, value: v)
proc qs(v: string): TokenWithValue = proc sq(v: string): TokenWithValue =
TokenWithValue(kind: ltQuotedScalar, value: v) TokenWithValue(kind: Token.SingleQuoted, value: v)
proc se(): TokenWithValue = TokenWithValue(kind: ltStreamEnd) proc dq(v: string): TokenWithValue =
proc mk(): TokenWithValue = TokenWithValue(kind: ltMapKeyInd) TokenWithValue(kind: Token.DoubleQuoted, value: v)
proc mv(): TokenWithValue = TokenWithValue(kind: ltMapValInd) proc e(): TokenWithValue = TokenWithValue(kind: Token.StreamEnd)
proc si(): TokenWithValue = TokenWithValue(kind: ltSeqItemInd) proc mk(): TokenWithValue = TokenWithValue(kind: Token.MapKeyInd)
proc dy(): TokenWithValue = TokenWithValue(kind: ltYamlDirective) proc mv(): TokenWithValue = TokenWithValue(kind: Token.MapValueInd)
proc dt(): TokenWithValue = TokenWithValue(kind: ltTagDirective) proc si(): TokenWithValue = TokenWithValue(kind: Token.SeqItemInd)
proc dy(): TokenWithValue = TokenWithValue(kind: Token.YamlDirective)
proc dt(): TokenWithValue = TokenWithValue(kind: Token.TagDirective)
proc du(v: string): TokenWithValue = proc du(v: string): TokenWithValue =
TokenWithValue(kind: ltUnknownDirective, value: v) TokenWithValue(kind: Token.UnknownDirective, value: v)
proc dp(v: string): TokenWithValue = proc dp(v: string): TokenWithValue =
TokenWithValue(kind: ltUnknownDirectiveParams, value: v) TokenWithValue(kind: Token.DirectiveParam, value: v)
proc yv(v: string): TokenWithValue = proc th(v: string): TokenWithValue =
TokenWithValue(kind: ltYamlVersion, value: v) TokenWithValue(kind: Token.TagHandle, value: v)
proc ts(v: string): TokenWithValue = proc ts(v: string): TokenWithValue =
TokenWithValue(kind: ltTagShorthand, value: v) TokenWithValue(kind: Token.Suffix, value: v)
proc tu(v: string): TokenWithValue = proc tv(v: string): TokenWithValue =
TokenWithValue(kind: ltTagUri, value: v) TokenWithValue(kind: Token.VerbatimTag, value: v)
proc dirE(): TokenWithValue = TokenWithValue(kind: ltDirectivesEnd) proc dirE(): TokenWithValue = TokenWithValue(kind: Token.DirectivesEnd)
proc docE(): TokenWithValue = TokenWithValue(kind: ltDocumentEnd) proc docE(): TokenWithValue = TokenWithValue(kind: Token.DocumentEnd)
proc bsh(): TokenWithValue = TokenWithValue(kind: ltBlockScalarHeader) proc ls(v: string): TokenWithValue = TokenWithValue(kind: Token.Literal, value: v)
proc bs(v: string): TokenWithValue = proc fs(v: string): TokenWithValue = TokenWithValue(kind: Token.Folded, value: v)
TokenWithValue(kind: ltBlockScalar, value: v) proc ss(): TokenWithValue = TokenWithValue(kind: Token.SeqStart)
proc el(): TokenWithValue = TokenWithValue(kind: ltEmptyLine) proc se(): TokenWithValue = TokenWithValue(kind: Token.SeqEnd)
proc ao(): TokenWithValue = TokenWithValue(kind: ltBracketOpen) proc ms(): TokenWithValue = TokenWithValue(kind: Token.MapStart)
proc ac(): TokenWithValue = TokenWithValue(kind: ltBracketClose) proc me(): TokenWithValue = TokenWithValue(kind: Token.MapEnd)
proc oo(): TokenWithValue = TokenWithValue(kind: ltBraceOpen) proc sep(): TokenWithValue = TokenWithValue(kind: Token.SeqSep)
proc oc(): TokenWithValue = TokenWithValue(kind: ltBraceClose) proc an(v: string): TokenWithValue = TokenWithValue(kind: Token.Anchor, value: v)
proc c(): TokenWithValue = TokenWithValue(kind: ltComma) proc al(v: string): TokenWithValue = TokenWithValue(kind: Token.Alias, value: v)
proc th(handle, suffix: string): TokenWithValue =
TokenWithValue(kind: ltTagHandle, handle: handle, suffix: suffix)
proc lt(v: string): TokenWithValue =
TokenWithValue(kind: ltLiteralTag, value: v)
proc an(v: string): TokenWithValue = TokenWithValue(kind: ltAnchor, value: v)
proc al(v: string): TokenWithValue = TokenWithValue(kind: ltAlias, value: v)
suite "Lexer": suite "Lexer":
test "Empty document": test "Empty document":
assertEquals("", se()) assertEquals("", e())
test "Single-line scalar": test "Single-line scalar":
assertEquals("scalar", i(0), sp("scalar"), se()) assertEquals("scalar", i(0), pl("scalar"), e())
test "Multiline scalar": test "Multiline scalar":
assertEquals("scalar\l line two", i(0), sp("scalar"), i(2), assertEquals("scalar\l line two", i(0), pl("scalar line two"), e())
sp("line two"), se())
test "Single-line mapping": test "Single-line mapping":
assertEquals("key: value", i(0), sp("key"), mv(), sp("value"), se()) assertEquals("key: value", i(0), pl("key"), mv(), pl("value"), e())
test "Multiline mapping": test "Multiline mapping":
assertEquals("key:\n value", i(0), sp("key"), mv(), i(2), sp("value"), assertEquals("key:\n value", i(0), pl("key"), mv(), i(2), pl("value"),
se()) e())
test "Explicit mapping": test "Explicit mapping":
assertEquals("? key\n: value", i(0), mk(), sp("key"), i(0), mv(), assertEquals("? key\n: value", i(0), mk(), pl("key"), i(0), mv(),
sp("value"), se()) pl("value"), e())
test "Sequence": test "Sequence":
assertEquals("- a\n- b", i(0), si(), sp("a"), i(0), si(), sp("b"), se()) assertEquals("- a\n- b", i(0), si(), pl("a"), i(0), si(), pl("b"), e())
test "Single-line single-quoted scalar": test "Single-line single-quoted scalar":
assertEquals("'quoted scalar'", i(0), qs("quoted scalar"), se()) assertEquals("'quoted scalar'", i(0), sq("quoted scalar"), e())
test "Multiline single-quoted scalar": test "Multiline single-quoted scalar":
assertEquals("'quoted\l multi line \l\lscalar'", i(0), assertEquals("'quoted\l multi line \l\lscalar'", i(0),
qs("quoted multi line\lscalar"), se()) sq("quoted multi line\lscalar"), e())
test "Single-line double-quoted scalar": test "Single-line double-quoted scalar":
assertEquals("\"quoted scalar\"", i(0), qs("quoted scalar"), se()) assertEquals("\"quoted scalar\"", i(0), dq("quoted scalar"), e())
test "Multiline double-quoted scalar": test "Multiline double-quoted scalar":
assertEquals("\"quoted\l multi line \l\lscalar\"", i(0), assertEquals("\"quoted\l multi line \l\lscalar\"", i(0),
qs("quoted multi line\lscalar"), se()) dq("quoted multi line\lscalar"), e())
test "Escape sequences": test "Escape sequences":
assertEquals(""""\n\x31\u0032\U00000033"""", i(0), qs("\l123"), se()) assertEquals(""""\n\x31\u0032\U00000033"""", i(0), dq("\l123"), e())
test "Directives": test "Directives":
assertEquals("%YAML 1.2\n---\n%TAG\n...\n\n%TAG ! example.html", assertEquals("%YAML 1.2\n---\n%TAG\n...\n\n%TAG ! example.html",
dy(), yv("1.2"), dirE(), i(0), sp("%TAG"), i(0), docE(), dt(), dy(), dp("1.2"), dirE(), i(0), pl("%TAG"), i(0), docE(), dt(),
ts("!"), tu("example.html"), se()) th("!"), ts("example.html"), e())
test "Markers and Unknown Directive": test "Markers and Unknown Directive":
assertEquals("---\n---\n...\n%UNKNOWN warbl", dirE(), dirE(), i(0), assertEquals("---\n---\n...\n%UNKNOWN warbl", dirE(), dirE(), i(0),
docE(), du("UNKNOWN"), dp("warbl"), se()) docE(), du("UNKNOWN"), dp("warbl"), e())
test "Block scalar": test "Block scalar":
assertEquals("|\l a\l\l b\l # comment", i(0), bsh(), bs("a\l\lb\l"), se()) assertEquals("|\l a\l\l b\l # comment", i(0), ls("a\l\lb\l"), e())
test "Block Scalars": test "Block Scalars":
assertEquals("one : >2-\l foo\l bar\ltwo: |+\l bar\l baz", i(0), assertEquals("one : >2-\l foo\l bar\ltwo: |+\l bar\l baz", i(0),
sp("one"), mv(), bsh(), bs(" foo\lbar"), i(0), sp("two"), mv(), bsh(), pl("one"), mv(), fs(" foo\lbar"), i(0), pl("two"), mv(),
bs("bar\l baz"), se()) ls("bar\l baz"), e())
test "Flow indicators": test "Flow indicators":
assertEquals("bla]: {c: d, [e]: f}", i(0), sp("bla]"), mv(), oo(), sp("c"), assertEquals("bla]: {c: d, [e]: f}", i(0), pl("bla]"), mv(), ms(), pl("c"),
mv(), sp("d"), c(), ao(), sp("e"), ac(), mv(), sp("f"), oc(), se()) mv(), pl("d"), sep(), ss(), pl("e"), se(), mv(), pl("f"), me(), e())
test "Adjacent map values in flow style": test "Adjacent map values in flow style":
assertEquals("{\"foo\":bar, [1]\l:egg}", i(0), oo(), qs("foo"), mv(), assertEquals("{\"foo\":bar, [1]\l:egg}", i(0), ms(), dq("foo"), mv(),
sp("bar"), c(), ao(), sp("1"), ac(), mv(), sp("egg"), oc(), se()) pl("bar"), sep(), ss(), pl("1"), se(), mv(), pl("egg"), me(), e())
test "Tag handles": test "Tag handles":
assertEquals("- !!str string\l- !local local\l- !e! e", i(0), si(), assertEquals("- !!str string\l- !local local\l- !e! e", i(0), si(),
th("!!", "str"), sp("string"), i(0), si(), th("!", "local"), th("!!"), ts("str"), pl("string"), i(0), si(), th("!"), ts("local"),
sp("local"), i(0), si(), th("!e!", ""), sp("e"), se()) pl("local"), i(0), si(), th("!e!"), ts(""), pl("e"), e())
test "Literal tag handle": test "Literal tag handle":
assertEquals("!<tag:yaml.org,2002:str> string", i(0), assertEquals("!<tag:yaml.org,2002:str> string", i(0),
lt("tag:yaml.org,2002:str"), sp("string"), se()) tv("tag:yaml.org,2002:str"), pl("string"), e())
test "Anchors and aliases": test "Anchors and aliases":
assertEquals("&a foo: {&b b: *a, *b : c}", i(0), an("a"), sp("foo"), mv(), assertEquals("&a foo: {&b b: *a, *b : c}", i(0), an("a"), pl("foo"), mv(),
oo(), an("b"), sp("b"), mv(), al("a"), c(), al("b"), mv(), sp("c"), ms(), an("b"), pl("b"), mv(), al("a"), sep(), al("b"), mv(), pl("c"),
oc(), se()) me(), e())
test "Empty lines": test "Empty lines":
assertEquals("""block: foo assertEquals("""block: foo
@ -226,37 +183,6 @@ flow: {
mi mi
}""", i(0), sp("block"), mv(), sp("foo"), el(), i(2), sp("bar"), el(), i(4), }""", i(0), pl("block"), mv(), pl("foo\nbar\nbaz"),
sp("baz"), i(0), sp("flow"), mv(), oo(), sp("foo"), el(), sp("bar"), mv(), i(0), pl("flow"), mv(), ms(), pl("foo\nbar"), mv(),
sp("baz"), el(), el(), sp("mi"), oc(), se()) pl("baz\n\nmi"), me(), e())
suite "Lookahead":
test "Simple Scalar":
assertLookahead("abcde", false)
test "Simple Mapping":
assertLookahead("a: b", true)
test "Colon inside plain scalar":
assertLookahead("abc:de", false)
test "Colon inside quoted scalar":
assertLookahead("'abc: de'", false)
test "Quotes inside plain scalar":
assertLookahead("abc\'\"de: foo", true)
test "Flow indicator inside plain scalar":
assertLookahead("abc}]: de", true)
test "Complex key":
assertLookahead("[1, 2, \"3\"]: foo", true)
test "Flow value":
assertLookahead("{a: b}", false)
test "In flow context":
assertLookahead("[ abcde]: foo", false, 2)
test "Adjacent value":
assertLookahead("[\"abc\":de]", true, 2)

26
test/tparser.nim
View File

@ -4,9 +4,9 @@
# See the file "copying.txt", included in this # See the file "copying.txt", included in this
# distribution, for details about the copyright. # distribution, for details about the copyright.
import os, osproc, terminal, strutils, streams, macros, unittest, sets import os, terminal, strutils, streams, macros, unittest, sets
import testEventParser, commonTestUtils import testEventParser, commonTestUtils
import "../yaml" import ../yaml, ../yaml/data
const const
testSuiteFolder = "yaml-test-suite" testSuiteFolder = "yaml-test-suite"
@ -18,7 +18,9 @@ proc echoError(msg: string) =
proc parserTest(path: string, errorExpected : bool): bool = proc parserTest(path: string, errorExpected : bool): bool =
var var
tagLib = initExtendedTagLibrary() tagLib = initExtendedTagLibrary()
parser = newYamlParser(tagLib) parser: YamlParser
parser.init(tagLib)
var
actualIn = newFileStream(path / "in.yaml") actualIn = newFileStream(path / "in.yaml")
actual = parser.parse(actualIn) actual = parser.parse(actualIn)
expectedIn = newFileStream(path / "test.event") expectedIn = newFileStream(path / "test.event")
@ -28,14 +30,8 @@ proc parserTest(path: string, errorExpected : bool): bool =
expectedIn.close() expectedIn.close()
var i = 1 var i = 1
try: try:
while not actual.finished(): while true:
let actualEvent = actual.next() let actualEvent = actual.next()
if expected.finished():
result = errorExpected
if not result:
echoError("At token #" & $i & ": Expected stream end, got " &
$actualEvent.kind)
return
let expectedEvent = expected.next() let expectedEvent = expected.next()
if expectedEvent != actualEvent: if expectedEvent != actualEvent:
result = errorExpected result = errorExpected
@ -45,12 +41,8 @@ proc parserTest(path: string, errorExpected : bool): bool =
": Actual tokens do not match expected tokens") ": Actual tokens do not match expected tokens")
return return
i.inc() i.inc()
if not expected.finished(): if actualEvent.kind == yamlEndStream:
result = errorExpected break
if not result:
echoError("Got fewer tokens than expected, first missing " &
"token: " & $expected.next().kind)
return
result = not errorExpected result = not errorExpected
if not result: if not result:
echo "Expected error, but parsed without error." echo "Expected error, but parsed without error."
@ -60,7 +52,7 @@ proc parserTest(path: string, errorExpected : bool): bool =
let e = getCurrentException() let e = getCurrentException()
if e.parent of YamlParserError: if e.parent of YamlParserError:
let pe = (ref YamlParserError)(e.parent) let pe = (ref YamlParserError)(e.parent)
echo "line ", pe.line, ", column ", pe.column, ": ", pe.msg echo "line ", pe.mark.line, ", column ", pe.mark.column, ": ", pe.msg
echo pe.lineContent echo pe.lineContent
else: echo e.msg else: echo e.msg
echoError("Catched an exception at token #" & $i & echoError("Catched an exception at token #" & $i &

5
yaml/dom.nim
View File

@ -198,8 +198,9 @@ proc loadDom*(s: Stream | string): YamlDocument
{.raises: [IOError, YamlParserError, YamlConstructionError].} = {.raises: [IOError, YamlParserError, YamlConstructionError].} =
var var
tagLib = initExtendedTagLibrary() tagLib = initExtendedTagLibrary()
parser = newYamlParser(tagLib) parser: YamlParser
events = parser.parse(s) parser.init(tagLib)
var events = parser.parse(s)
try: result = compose(events, tagLib) try: result = compose(events, tagLib)
except YamlStreamError: except YamlStreamError:
let e = getCurrentException() let e = getCurrentException()

75
yaml/parser.nim
View File

@ -11,14 +11,14 @@
## This is the low-level parser API. A ``YamlParser`` enables you to parse any ## This is the low-level parser API. A ``YamlParser`` enables you to parse any
## non-nil string or Stream object as YAML character stream. ## non-nil string or Stream object as YAML character stream.
import tables, strutils, macros import tables, strutils, macros, streams
import taglib, stream, private/lex, private/internal, data import taglib, stream, private/lex, private/internal, data
when defined(nimNoNil): when defined(nimNoNil):
{.experimental: "notnil".} {.experimental: "notnil".}
type type
YamlParser* = ref object YamlParser* = object
## A parser object. Retains its ``TagLibrary`` across calls to ## A parser object. Retains its ``TagLibrary`` across calls to
## `parse <#parse,YamlParser,Stream>`_. Can be used ## `parse <#parse,YamlParser,Stream>`_. Can be used
## to access anchor names while parsing a YAML character stream, but ## to access anchor names while parsing a YAML character stream, but
@ -26,7 +26,6 @@ type
## ``yamlEndDocument`` is yielded). ## ``yamlEndDocument`` is yielded).
tagLib: TagLibrary tagLib: TagLibrary
issueWarnings: bool issueWarnings: bool
anchors: Table[string, Anchor]
State = proc(c: Context, e: var Event): bool {.locks: 0, gcSafe.} State = proc(c: Context, e: var Event): bool {.locks: 0, gcSafe.}
@ -83,28 +82,9 @@ type
## Some elements in this list are vague. For a detailed description of a ## Some elements in this list are vague. For a detailed description of a
## valid YAML character stream, see the YAML specification. ## valid YAML character stream, see the YAML specification.
# interface
proc newYamlParser*(tagLib: TagLibrary = initExtendedTagLibrary(),
issueWarnings: bool = false): YamlParser =
## Creates a YAML parser. if ``callback`` is not ``nil``, it will be called
## whenever the parser yields a warning.
new(result)
result.tagLib = tagLib
result.issueWarnings = issueWarnings
# implementation
template debug(message: string) {.dirty.} =
when defined(yamlDebug):
try: styledWriteLine(stdout, fgBlue, message)
except IOError: discard
const defaultProperties = (yAnchorNone, yTagQuestionMark) const defaultProperties = (yAnchorNone, yTagQuestionMark)
proc isEmpty(props: Properties): bool = # parser states
result = props.anchor == yAnchorNone and
props.tag == yTagQuestionMark
{.push gcSafe, locks: 0.} {.push gcSafe, locks: 0.}
proc atStreamStart(c: Context, e: var Event): bool proc atStreamStart(c: Context, e: var Event): bool
@ -139,11 +119,44 @@ proc afterFlowSeqItem(c: Context, e: var Event): bool
proc afterPairValue(c: Context, e: var Event): bool proc afterPairValue(c: Context, e: var Event): bool
{.pop.} {.pop.}
proc init[T](pc: Context, source: T) {.inline.} = proc init[T](c: Context, source: T) {.inline.} =
pc.levels.add(Level(state: atStreamStart, indentation: -2)) c.levels.add(Level(state: atStreamStart, indentation: -2))
pc.headerProps = defaultProperties c.nextImpl = proc(s: YamlStream, e: var Event): bool =
pc.inlineProps = defaultProperties let c = Context(s)
pc.lex.init(source) return c.levels[^1].state(c, e)
c.headerProps = defaultProperties
c.inlineProps = defaultProperties
c.lex.init(source)
# interface
proc init*(p: var YamlParser, tagLib: TagLibrary = initExtendedTagLibrary(),
issueWarnings: bool = false) =
## Creates a YAML parser. if ``callback`` is not ``nil``, it will be called
## whenever the parser yields a warning.
p.tagLib = tagLib
p.issueWarnings = issueWarnings
proc parse*(p: YamlParser, s: Stream): YamlStream =
let c = new(Context)
c.init(s)
return c
proc parse*(p: YamlParser, s: string): YamlStream =
let c = new(Context)
c.init(s)
return c
# implementation
template debug(message: string) {.dirty.} =
when defined(yamlDebug):
try: styledWriteLine(stdout, fgBlue, message)
except IOError: discard
proc isEmpty(props: Properties): bool =
result = props.anchor == yAnchorNone and
props.tag == yTagQuestionMark
proc generateError(c: Context, message: string): proc generateError(c: Context, message: string):
ref YamlParserError {.raises: [].} = ref YamlParserError {.raises: [].} =
@ -706,7 +719,7 @@ proc beforeBlockMapValue(c: Context, e: var Event): bool =
raise c.generateError("Unexpected token (expected mapping value): " & $c.lex.cur) raise c.generateError("Unexpected token (expected mapping value): " & $c.lex.cur)
proc beforeBlockIndentation(c: Context, e: var Event): bool = proc beforeBlockIndentation(c: Context, e: var Event): bool =
proc endBlockNode() = proc endBlockNode(e: var Event) =
if c.levels[^1].state == beforeBlockMapKey: if c.levels[^1].state == beforeBlockMapKey:
e = endMapEvent(c.lex.curStartPos, c.lex.curEndPos) e = endMapEvent(c.lex.curStartPos, c.lex.curEndPos)
elif c.levels[^1].state == beforeBlockMapValue: elif c.levels[^1].state == beforeBlockMapValue:
@ -729,7 +742,7 @@ proc beforeBlockIndentation(c: Context, e: var Event): bool =
of Indentation: of Indentation:
c.blockIndentation = c.lex.indentation c.blockIndentation = c.lex.indentation
if c.blockIndentation < c.levels[^1].indentation: if c.blockIndentation < c.levels[^1].indentation:
endBlockNode() endBlockNode(e)
return true return true
else: else:
c.lex.next() c.lex.next()
@ -737,7 +750,7 @@ proc beforeBlockIndentation(c: Context, e: var Event): bool =
of StreamEnd, DocumentEnd, DirectivesEnd: of StreamEnd, DocumentEnd, DirectivesEnd:
c.blockIndentation = 0 c.blockIndentation = 0
if c.levels[^1].state != beforeDocEnd: if c.levels[^1].state != beforeDocEnd:
endBlockNode() endBlockNode(e)
return true return true
else: else:
return false return false

5
yaml/presenter.nim
View File

@ -730,8 +730,9 @@ proc doTransform(input: Stream | string, output: PresenterTarget,
options: PresentationOptions, resolveToCoreYamlTags: bool) = options: PresentationOptions, resolveToCoreYamlTags: bool) =
var var
taglib = initExtendedTagLibrary() taglib = initExtendedTagLibrary()
parser = newYamlParser(tagLib) parser: YamlParser
events = parser.parse(input) parser.init(tagLib)
var events = parser.parse(input)
try: try:
if options.style == psCanonical: if options.style == psCanonical:
var bys: YamlStream = newBufferYamlStream() var bys: YamlStream = newBufferYamlStream()

12
yaml/serialization.nim
View File

@ -1380,9 +1380,9 @@ proc construct*[T](s: var YamlStream, target: var T)
proc load*[K](input: Stream | string, target: var K) proc load*[K](input: Stream | string, target: var K)
{.raises: [YamlConstructionError, IOError, YamlParserError].} = {.raises: [YamlConstructionError, IOError, YamlParserError].} =
## Loads a Nim value from a YAML character stream. ## Loads a Nim value from a YAML character stream.
var var parser: YamlParser
parser = newYamlParser(serializationTagLibrary) parser.init(serializationTagLibrary)
events = parser.parse(input) var events = parser.parse(input)
try: construct(events, target) try: construct(events, target)
except YamlStreamError: except YamlStreamError:
let e = (ref YamlStreamError)(getCurrentException()) let e = (ref YamlStreamError)(getCurrentException())
@ -1391,9 +1391,9 @@ proc load*[K](input: Stream | string, target: var K)
else: internalError("Unexpected exception: " & $e.parent.name) else: internalError("Unexpected exception: " & $e.parent.name)
proc loadMultiDoc*[K](input: Stream | string, target: var seq[K]) = proc loadMultiDoc*[K](input: Stream | string, target: var seq[K]) =
var var parser: YamlParser
parser = newYamlParser(serializationTagLibrary) parser.init(serializationTagLibrary)
events = parser.parse(input) var events = parser.parse(input)
try: try:
while not events.finished(): while not events.finished():
var item: K var item: K

37
yaml/tojson.nim
View File

@ -90,9 +90,10 @@ proc constructJson*(s: var YamlStream): seq[JsonNode]
var var
levels = newSeq[Level]() levels = newSeq[Level]()
anchors = initTable[AnchorId, JsonNode]() anchors = initTable[Anchor, JsonNode]()
for event in s: for event in s:
case event.kind case event.kind
of yamlStartStream, yamlEndStream: discard
of yamlStartDoc: of yamlStartDoc:
# we don't need to do anything here; root node will be created # we don't need to do anything here; root node will be created
# by first scalar, sequence or map event # by first scalar, sequence or map event
@ -106,13 +107,13 @@ proc constructJson*(s: var YamlStream): seq[JsonNode]
anchors[event.seqProperties.anchor] = levels[levels.high].node anchors[event.seqProperties.anchor] = levels[levels.high].node
of yamlStartMap: of yamlStartMap:
levels.add(initLevel(newJObject())) levels.add(initLevel(newJObject()))
if event.mapAnchor != yAnchorNone: if event.mapProperties.anchor != yAnchorNone:
anchors[event.mapAnchor] = levels[levels.high].node anchors[event.mapProperties.anchor] = levels[levels.high].node
of yamlScalar: of yamlScalar:
if levels.len == 0: if levels.len == 0:
# parser ensures that next event will be yamlEndDocument # parser ensures that next event will be yamlEndDocument
levels.add((node: jsonFromScalar(event.scalarContent, levels.add((node: jsonFromScalar(event.scalarContent,
event.scalarTag), event.scalarProperties.tag),
key: "", key: "",
expKey: true)) expKey: true))
continue continue
@ -120,25 +121,25 @@ proc constructJson*(s: var YamlStream): seq[JsonNode]
case levels[levels.high].node.kind case levels[levels.high].node.kind
of JArray: of JArray:
let jsonScalar = jsonFromScalar(event.scalarContent, let jsonScalar = jsonFromScalar(event.scalarContent,
event.scalarTag) event.scalarProperties.tag)
levels[levels.high].node.elems.add(jsonScalar) levels[levels.high].node.elems.add(jsonScalar)
if event.scalarAnchor != yAnchorNone: if event.scalarProperties.anchor != yAnchorNone:
anchors[event.scalarAnchor] = jsonScalar anchors[event.scalarProperties.anchor] = jsonScalar
of JObject: of JObject:
if levels[levels.high].expKey: if levels[levels.high].expKey:
levels[levels.high].expKey = false levels[levels.high].expKey = false
# JSON only allows strings as keys # JSON only allows strings as keys
levels[levels.high].key = event.scalarContent levels[levels.high].key = event.scalarContent
if event.scalarAnchor != yAnchorNone: if event.scalarProperties.anchor != yAnchorNone:
raise newException(YamlConstructionError, raise newException(YamlConstructionError,
"scalar keys may not have anchors in JSON") "scalar keys may not have anchors in JSON")
else: else:
let jsonScalar = jsonFromScalar(event.scalarContent, let jsonScalar = jsonFromScalar(event.scalarContent,
event.scalarTag) event.scalarProperties.tag)
levels[levels.high].node[levels[levels.high].key] = jsonScalar levels[levels.high].node[levels[levels.high].key] = jsonScalar
levels[levels.high].expKey = true levels[levels.high].expKey = true
if event.scalarAnchor != yAnchorNone: if event.scalarProperties.anchor != yAnchorNone:
anchors[event.scalarAnchor] = jsonScalar anchors[event.scalarProperties.anchor] = jsonScalar
else: else:
internalError("Unexpected node kind: " & $levels[levels.high].node.kind) internalError("Unexpected node kind: " & $levels[levels.high].node.kind)
of yamlEndSeq, yamlEndMap: of yamlEndSeq, yamlEndMap:
@ -177,13 +178,13 @@ proc constructJson*(s: var YamlStream): seq[JsonNode]
when not defined(JS): when not defined(JS):
proc loadToJson*(s: Stream): seq[JsonNode] proc loadToJson*(s: Stream): seq[JsonNode]
{.raises: [YamlParserError, YamlConstructionError, IOError].} = {.raises: [YamlParserError, YamlConstructionError, IOError, OSError].} =
## Uses `YamlParser <#YamlParser>`_ and ## Uses `YamlParser <#YamlParser>`_ and
## `constructJson <#constructJson>`_ to construct an in-memory JSON tree ## `constructJson <#constructJson>`_ to construct an in-memory JSON tree
## from a YAML character stream. ## from a YAML character stream.
var var parser: YamlParser
parser = newYamlParser(initCoreTagLibrary()) parser.init(initCoreTagLibrary())
events = parser.parse(s) var events = parser.parse(s)
try: try:
return constructJson(events) return constructJson(events)
except YamlStreamError: except YamlStreamError:
@ -199,9 +200,9 @@ proc loadToJson*(str: string): seq[JsonNode]
## Uses `YamlParser <#YamlParser>`_ and ## Uses `YamlParser <#YamlParser>`_ and
## `constructJson <#constructJson>`_ to construct an in-memory JSON tree ## `constructJson <#constructJson>`_ to construct an in-memory JSON tree
## from a YAML character stream. ## from a YAML character stream.
var var parser: YamlParser
parser = newYamlParser(initCoreTagLibrary()) parser.init(initCoreTagLibrary())
events = parser.parse(str) var events = parser.parse(str)
try: return constructJson(events) try: return constructJson(events)
except YamlStreamError: except YamlStreamError:
let e = getCurrentException() let e = getCurrentException()