NimYAML/test/tlex.nim

262 lines
9.0 KiB
Nim
Raw Normal View History

2017-02-06 19:39:04 +00:00
import ../yaml/private/lex
2016-09-10 08:30:40 +00:00
2016-09-10 10:38:03 +00:00
import unittest, strutils
const tokensWithValue =
2016-09-11 10:52:24 +00:00
{ltScalarPart, ltQuotedScalar, ltYamlVersion, ltTagShorthand, ltTagUri,
2016-09-11 11:04:10 +00:00
ltUnknownDirective, ltUnknownDirectiveParams, ltLiteralTag, ltAnchor,
2016-09-14 16:31:09 +00:00
ltAlias, ltBlockScalar}
2016-09-10 08:30:40 +00:00
type
TokenWithValue = object
case kind: LexerToken
2016-09-10 10:38:03 +00:00
of tokensWithValue:
2016-09-10 08:30:40 +00:00
value: string
2016-09-10 11:38:42 +00:00
of ltIndentation:
indentation: int
2016-09-11 10:52:24 +00:00
of ltTagHandle:
handle, suffix: string
2016-09-10 08:30:40 +00:00
else: discard
proc actualRepr(lex: YamlLexer, t: LexerToken): string =
result = $t
case t
2016-09-11 10:52:24 +00:00
of tokensWithValue + {ltTagHandle}:
result.add("(" & escape(lex.buf) & ")")
of ltIndentation:
result.add("(" & $lex.indentation & ")")
else: discard
2016-09-10 08:30:40 +00:00
proc assertEquals(input: string, expected: varargs[TokenWithValue]) =
2016-09-10 10:38:03 +00:00
let lex = newYamlLexer(input)
2016-09-10 15:33:58 +00:00
var
i = 0
blockScalarEnd = -1
flowDepth = 0
2016-09-10 10:38:03 +00:00
for expectedToken in expected:
inc(i)
try:
lex.next()
doAssert lex.cur == expectedToken.kind, "Wrong token kind at #" & $i &
": Expected " & $expectedToken.kind & ", got " &
lex.actualRepr(lex.cur)
2016-09-11 09:28:05 +00:00
case expectedToken.kind
of tokensWithValue:
doAssert lex.buf == expectedToken.value, "Wrong token content at #" &
$i & ": Expected " & escape(expectedToken.value) &
", got " & escape(lex.buf)
lex.buf = ""
of ltIndentation:
doAssert lex.indentation == expectedToken.indentation,
"Wrong indentation length at #" & $i & ": Expected " &
$expectedToken.indentation & ", got " & $lex.indentation
2016-09-11 09:28:05 +00:00
if lex.indentation <= blockScalarEnd:
lex.endBlockScalar()
blockScalarEnd = -1
2016-09-10 15:33:58 +00:00
of ltBraceOpen, ltBracketOpen:
inc(flowDepth)
if flowDepth == 1: lex.setFlow(true)
of ltBraceClose, ltBracketClose:
dec(flowDepth)
if flowDepth == 0: lex.setFlow(false)
2016-09-11 10:52:24 +00:00
of ltTagHandle:
let
handle = lex.buf.substr(0, lex.shorthandEnd)
suffix = lex.buf.substr(lex.shorthandEnd + 1)
doAssert handle == expectedToken.handle,
"Wrong handle at #" & $i & ": Expected " & expectedToken.handle &
", got " & handle
doAssert suffix == expectedToken.suffix,
"Wrong suffix at #" & $i & ": Expected " & expectedToken.suffix &
", got " & suffix
lex.buf = ""
else: discard
except YamlLexerError:
let e = (ref YamlLexerError)(getCurrentException())
echo "Error at line " & $e.line & ", column " & $e.column & ":"
echo e.lineContent
assert false
2016-09-10 10:38:03 +00:00
2016-09-11 16:23:47 +00:00
proc assertLookahead(input: string, expected: bool, tokensBefore: int = 1) =
let lex = newYamlLexer(input)
var flowDepth = 0
for i in 0..tokensBefore:
lex.next()
case lex.cur
of ltBraceOpen, ltBracketOpen:
inc(flowDepth)
if flowDepth == 1: lex.setFlow(true)
of ltBraceClose, ltBracketClose:
dec(flowDepth)
if flowDepth == 0: lex.setFlow(false)
else: discard
doAssert lex.isImplicitKeyStart() == expected
2016-09-10 11:38:42 +00:00
proc i(indent: int): TokenWithValue =
TokenWithValue(kind: ltIndentation, indentation: indent)
proc sp(v: string): TokenWithValue =
TokenWithValue(kind: ltScalarPart, value: v)
proc qs(v: string): TokenWithValue =
TokenWithValue(kind: ltQuotedScalar, value: v)
2016-09-10 10:38:03 +00:00
proc se(): TokenWithValue = TokenWithValue(kind: ltStreamEnd)
2016-09-10 11:38:42 +00:00
proc mk(): TokenWithValue = TokenWithValue(kind: ltMapKeyInd)
proc mv(): TokenWithValue = TokenWithValue(kind: ltMapValInd)
proc si(): TokenWithValue = TokenWithValue(kind: ltSeqItemInd)
proc dy(): TokenWithValue = TokenWithValue(kind: ltYamlDirective)
proc dt(): TokenWithValue = TokenWithValue(kind: ltTagDirective)
proc du(v: string): TokenWithValue =
TokenWithValue(kind: ltUnknownDirective, value: v)
proc dp(v: string): TokenWithValue =
TokenWithValue(kind: ltUnknownDirectiveParams, value: v)
proc yv(v: string): TokenWithValue =
TokenWithValue(kind: ltYamlVersion, value: v)
proc ts(v: string): TokenWithValue =
TokenWithValue(kind: ltTagShorthand, value: v)
proc tu(v: string): TokenWithValue =
TokenWithValue(kind: ltTagUri, value: v)
proc dirE(): TokenWithValue = TokenWithValue(kind: ltDirectivesEnd)
proc docE(): TokenWithValue = TokenWithValue(kind: ltDocumentEnd)
2016-09-14 16:31:09 +00:00
proc bsh(): TokenWithValue = TokenWithValue(kind: ltBlockScalarHeader)
proc bs(v: string): TokenWithValue =
TokenWithValue(kind: ltBlockScalar, value: v)
proc el(): TokenWithValue = TokenWithValue(kind: ltEmptyLine)
2016-09-10 15:33:58 +00:00
proc ao(): TokenWithValue = TokenWithValue(kind: ltBracketOpen)
proc ac(): TokenWithValue = TokenWithValue(kind: ltBracketClose)
proc oo(): TokenWithValue = TokenWithValue(kind: ltBraceOpen)
proc oc(): TokenWithValue = TokenWithValue(kind: ltBraceClose)
proc c(): TokenWithValue = TokenWithValue(kind: ltComma)
2016-09-11 10:52:24 +00:00
proc th(handle, suffix: string): TokenWithValue =
TokenWithValue(kind: ltTagHandle, handle: handle, suffix: suffix)
proc lt(v: string): TokenWithValue =
TokenWithValue(kind: ltLiteralTag, value: v)
2016-09-11 11:04:10 +00:00
proc an(v: string): TokenWithValue = TokenWithValue(kind: ltAnchor, value: v)
proc al(v: string): TokenWithValue = TokenWithValue(kind: ltAlias, value: v)
2016-09-10 08:30:40 +00:00
suite "Lexer":
test "Empty document":
2016-09-10 11:38:42 +00:00
assertEquals("", se())
2016-09-11 09:28:05 +00:00
2016-09-10 11:38:42 +00:00
test "Single-line scalar":
assertEquals("scalar", i(0), sp("scalar"), se())
2016-09-11 09:28:05 +00:00
2016-09-10 11:38:42 +00:00
test "Multiline scalar":
assertEquals("scalar\l line two", i(0), sp("scalar"), i(2),
sp("line two"), se())
2016-09-11 09:28:05 +00:00
2016-09-10 11:38:42 +00:00
test "Single-line mapping":
assertEquals("key: value", i(0), sp("key"), mv(), sp("value"), se())
2016-09-11 09:28:05 +00:00
2016-09-10 11:38:42 +00:00
test "Multiline mapping":
assertEquals("key:\n value", i(0), sp("key"), mv(), i(2), sp("value"),
se())
2016-09-11 09:28:05 +00:00
2016-09-10 11:38:42 +00:00
test "Explicit mapping":
assertEquals("? key\n: value", i(0), mk(), sp("key"), i(0), mv(),
sp("value"), se())
2016-09-11 09:28:05 +00:00
test "Sequence":
assertEquals("- a\n- b", i(0), si(), sp("a"), i(0), si(), sp("b"), se())
2016-09-11 09:28:05 +00:00
2016-09-10 11:38:42 +00:00
test "Single-line single-quoted scalar":
assertEquals("'quoted scalar'", i(0), qs("quoted scalar"), se())
2016-09-11 09:28:05 +00:00
2016-09-10 11:38:42 +00:00
test "Multiline single-quoted scalar":
assertEquals("'quoted\l multi line \l\lscalar'", i(0),
qs("quoted multi line\lscalar"), se())
2016-09-11 09:28:05 +00:00
2016-09-10 11:38:42 +00:00
test "Single-line double-quoted scalar":
assertEquals("\"quoted scalar\"", i(0), qs("quoted scalar"), se())
2016-09-11 09:28:05 +00:00
2016-09-10 11:38:42 +00:00
test "Multiline double-quoted scalar":
assertEquals("\"quoted\l multi line \l\lscalar\"", i(0),
qs("quoted multi line\lscalar"), se())
2016-09-11 09:28:05 +00:00
2016-09-10 11:38:42 +00:00
test "Escape sequences":
assertEquals(""""\n\x31\u0032\U00000033"""", i(0), qs("\l123"), se())
2016-09-11 09:28:05 +00:00
test "Directives":
assertEquals("%YAML 1.2\n---\n%TAG\n...\n\n%TAG ! example.html",
dy(), yv("1.2"), dirE(), i(0), sp("%TAG"), i(0), docE(), dt(),
ts("!"), tu("example.html"), se())
2016-09-11 09:28:05 +00:00
test "Markers and Unknown Directive":
assertEquals("---\n---\n...\n%UNKNOWN warbl", dirE(), dirE(), i(0),
docE(), du("UNKNOWN"), dp("warbl"), se())
2016-09-11 09:28:05 +00:00
test "Block scalar":
2016-09-14 16:31:09 +00:00
assertEquals("|\l a\l\l b\l # comment", i(0), bsh(), bs("a\l\lb\l"), se())
2016-09-11 09:28:05 +00:00
test "Block Scalars":
assertEquals("one : >2-\l foo\l bar\ltwo: |+\l bar\l baz", i(0),
2016-09-14 16:31:09 +00:00
sp("one"), mv(), bsh(), bs(" foo\lbar"), i(0), sp("two"), mv(), bsh(),
bs("bar\l baz"), se())
2016-09-11 09:28:05 +00:00
2016-09-10 15:33:58 +00:00
test "Flow indicators":
assertEquals("bla]: {c: d, [e]: f}", i(0), sp("bla]"), mv(), oo(), sp("c"),
mv(), sp("d"), c(), ao(), sp("e"), ac(), mv(), sp("f"), oc(), se())
test "Adjacent map values in flow style":
assertEquals("{\"foo\":bar, [1]\l:egg}", i(0), oo(), qs("foo"), mv(),
2016-09-11 10:52:24 +00:00
sp("bar"), c(), ao(), sp("1"), ac(), mv(), sp("egg"), oc(), se())
test "Tag handles":
assertEquals("- !!str string\l- !local local\l- !e! e", i(0), si(),
th("!!", "str"), sp("string"), i(0), si(), th("!", "local"),
sp("local"), i(0), si(), th("!e!", ""), sp("e"), se())
test "Literal tag handle":
assertEquals("!<tag:yaml.org,2002:str> string", i(0),
2016-09-11 11:04:10 +00:00
lt("tag:yaml.org,2002:str"), sp("string"), se())
test "Anchors and aliases":
assertEquals("&a foo: {&b b: *a, *b : c}", i(0), an("a"), sp("foo"), mv(),
oo(), an("b"), sp("b"), mv(), al("a"), c(), al("b"), mv(), sp("c"),
2016-09-11 16:23:47 +00:00
oc(), se())
2016-09-11 17:20:02 +00:00
test "Empty lines":
assertEquals("""block: foo
bar
baz
flow: {
foo
bar: baz
mi
}""", i(0), sp("block"), mv(), sp("foo"), el(), i(2), sp("bar"), el(), i(4),
sp("baz"), i(0), sp("flow"), mv(), oo(), sp("foo"), el(), sp("bar"), mv(),
sp("baz"), el(), el(), sp("mi"), oc(), se())
2016-09-11 16:23:47 +00:00
suite "Lookahead":
test "Simple Scalar":
assertLookahead("abcde", false)
test "Simple Mapping":
assertLookahead("a: b", true)
test "Colon inside plain scalar":
assertLookahead("abc:de", false)
test "Colon inside quoted scalar":
assertLookahead("'abc: de'", false)
test "Quotes inside plain scalar":
assertLookahead("abc\'\"de: foo", true)
test "Flow indicator inside plain scalar":
assertLookahead("abc}]: de", true)
test "Complex key":
assertLookahead("[1, 2, \"3\"]: foo", true)
test "Flow value":
assertLookahead("{a: b}", false)
test "In flow context":
assertLookahead("[ abcde]: foo", false, 2)
test "Adjacent value":
assertLookahead("[\"abc\":de]", true, 2)