mirror of https://github.com/status-im/NimYAML.git
Updated test suite, fixed lexer issues
This commit is contained in:
parent
f692a47820
commit
300dbce72a
|
@ -1,4 +1,4 @@
|
|||
[submodule "test/yaml-test-suite"]
|
||||
path = test/yaml-test-suite
|
||||
url = https://github.com/yaml/yaml-test-suite.git
|
||||
branch = data-2020-08-01
|
||||
branch = data-2022-01-17
|
||||
|
|
|
@ -71,23 +71,36 @@ macro genTests(): untyped =
|
|||
echo "[tparser] Generating tests from " & absolutePath
|
||||
discard staticExec("git submodule init && git submodule update --remote")
|
||||
|
||||
let errorTests = toHashSet(staticExec("cd " & (absolutePath / "tags" / "error") &
|
||||
" && ls -1d *").splitLines())
|
||||
var ignored = toHashSet([".git", "name", "tags", "meta"])
|
||||
|
||||
|
||||
proc genTest(target: var NimNode, dirPath: string, testId: string) {.compileTime.} =
|
||||
let title = slurp(dirPath / "===")
|
||||
|
||||
target.add(newCall("test",
|
||||
newLit(strip(title) & " [" &
|
||||
testId & ']'), newCall("doAssert", newCall("parserTest",
|
||||
newLit(dirPath), newLit(fileExists(dirPath / "error"))))))
|
||||
|
||||
result = newStmtList()
|
||||
|
||||
# walkDir for some crude reason does not work with travis build
|
||||
let dirItems = staticExec("ls -1d " & absolutePath / "*")
|
||||
for dirPath in dirItems.splitLines():
|
||||
if dirPath.strip.len == 0: continue
|
||||
let testId = dirPath[^4..^1]
|
||||
if ignored.contains(testId): continue
|
||||
let title = slurp(dirPath / "===")
|
||||
|
||||
result.add(newCall("test",
|
||||
newLit(strip(title) & " [" &
|
||||
testId & ']'), newCall("doAssert", newCall("parserTest",
|
||||
newLit(dirPath), newLit(errorTests.contains(testId))))))
|
||||
if fileExists(dirPath / "==="):
|
||||
genTest(result, dirPath, testId)
|
||||
else:
|
||||
let testItems = staticExec("ls -1d " & dirPath / "*")
|
||||
var start = len(dirPath) + 1
|
||||
for itemPath in testItems.splitLines():
|
||||
if itemPath.strip.len == 0: continue
|
||||
let testItemId = testId / itemPath[start..^1]
|
||||
if fileExists(itemPath / "==="):
|
||||
genTest(result, itemPath, testItemId)
|
||||
|
||||
|
||||
result = newCall("suite", newLit("Parser Tests (from yaml-test-suite)"), result)
|
||||
|
||||
genTests()
|
||||
|
|
|
@ -1 +1 @@
|
|||
Subproject commit 80f289435a56189bbc5ca2f318574e48dbeef8aa
|
||||
Subproject commit 6e6c296ae9c9d2d5c4134b4b64d01b29ac19ff6f
|
|
@ -492,6 +492,11 @@ proc atBlockIndentationProps(c: Context, e: var Event): bool =
|
|||
c.lex.next()
|
||||
c.transition(atBlockIndentation)
|
||||
return false
|
||||
of StreamEnd, DocumentEnd, DirectivesEnd:
|
||||
e = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos)
|
||||
c.inlineProps = defaultProperties
|
||||
c.popLevel()
|
||||
return true
|
||||
else:
|
||||
raise c.generateError("Unexpected token (expected block content): " & $c.lex.cur)
|
||||
|
||||
|
@ -558,11 +563,6 @@ proc afterCompactParentProps(c: Context, e: var Event): bool =
|
|||
c.transition(atBlockIndentation, c.levels[^3].indentation)
|
||||
c.pushLevel(beforeBlockIndentation)
|
||||
return false
|
||||
of StreamEnd, DocumentEnd, DirectivesEnd:
|
||||
e = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos)
|
||||
c.inlineProps = defaultProperties
|
||||
c.popLevel()
|
||||
return true
|
||||
of MapValueInd:
|
||||
c.keyCache.add(scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos))
|
||||
c.inlineProps = defaultProperties
|
||||
|
@ -599,7 +599,7 @@ proc afterCompactParentProps(c: Context, e: var Event): bool =
|
|||
else:
|
||||
c.popLevel()
|
||||
return true
|
||||
of MapStart, SeqStart:
|
||||
of MapStart, SeqStart, StreamEnd, DocumentEnd, DirectivesEnd:
|
||||
c.transition(atBlockIndentationProps)
|
||||
return false
|
||||
else:
|
||||
|
|
|
@ -303,6 +303,15 @@ proc startLine(lex: var Lexer): LineStartType =
|
|||
else: lsContent
|
||||
else:
|
||||
while lex.c == ' ': lex.advance()
|
||||
if lex.c == '\t':
|
||||
var peek = lex.source.bufpos
|
||||
while lex.source.buf[peek] in space:
|
||||
peek += 1
|
||||
if lex.source.buf[peek] in commentOrLineEnd:
|
||||
lex.source.bufpos = peek + 1
|
||||
lex.c = lex.source.buf[peek]
|
||||
else:
|
||||
return lsContent
|
||||
return case lex.c
|
||||
of '#': lsComment
|
||||
of '\l', '\c': lsNewline
|
||||
|
@ -448,6 +457,7 @@ proc readBlockScalar(lex: var Lexer) =
|
|||
indent = 0
|
||||
separationLines = 0
|
||||
contentStart: int
|
||||
hasBody = true
|
||||
lex.startToken()
|
||||
lex.cur = if lex.c == '>': Token.Folded else: Token.Literal
|
||||
lex.evaluated.setLen(0)
|
||||
|
@ -476,7 +486,10 @@ proc readBlockScalar(lex: var Lexer) =
|
|||
raise lex.generateError("Illegal character after block scalar header: " &
|
||||
escape("" & lex.c))
|
||||
break
|
||||
of lineEnd: break
|
||||
of EndOfFile:
|
||||
hasBody = false
|
||||
break
|
||||
of '\l', '\c': break
|
||||
else:
|
||||
raise lex.generateError("Illegal character in block scalar header: " &
|
||||
escape("" & lex.c))
|
||||
|
@ -503,6 +516,7 @@ proc readBlockScalar(lex: var Lexer) =
|
|||
of EndOfFile:
|
||||
lex.state = streamEnd
|
||||
lex.streamEndAfterBlock()
|
||||
if lex.source.getColNumber(lex.source.bufpos) > 1 and hasBody: separationLines += 1
|
||||
break body
|
||||
else:
|
||||
if indent == 0:
|
||||
|
@ -586,8 +600,7 @@ proc readBlockScalar(lex: var Lexer) =
|
|||
case chomp
|
||||
of ctStrip: discard
|
||||
of ctClip:
|
||||
if len(lex.evaluated) > 0:
|
||||
lex.evaluated.add('\l')
|
||||
if len(lex.evaluated) > 0: lex.evaluated.add('\l')
|
||||
of ctKeep:
|
||||
for i in countup(0, separationLines - 1):
|
||||
lex.evaluated.add('\l')
|
||||
|
@ -813,6 +826,14 @@ proc outsideDoc(lex: var Lexer): bool =
|
|||
if lex.c in commentOrLineEnd:
|
||||
lex.state = expectLineEnd
|
||||
return false
|
||||
if lex.c == '\t':
|
||||
var peek = lex.source.bufpos
|
||||
while lex.source.buf[peek] in space:
|
||||
peek += 1
|
||||
if lex.source.buf[peek] in commentOrLineEnd:
|
||||
lex.state = expectLineEnd
|
||||
lex.source.bufpos = peek
|
||||
return false
|
||||
lex.endToken()
|
||||
lex.cur = Token.Indentation
|
||||
lex.indentation = -1
|
||||
|
@ -914,6 +935,9 @@ proc flowLineStart(lex: var Lexer): bool =
|
|||
while lex.c == ' ': lex.advance()
|
||||
indent = lex.source.bufpos - lineStart
|
||||
while lex.c in space: lex.advance()
|
||||
if lex.c in commentOrLineEnd:
|
||||
lex.state = expectLineEnd
|
||||
return false
|
||||
if indent <= lex.indentation:
|
||||
raise lex.generateError("Too few indentation spaces (must surpass surrounding block level)")
|
||||
lex.state = insideLine
|
||||
|
|
Loading…
Reference in New Issue