Updated test suite, fixed lexer issues

This commit is contained in:
Felix Krause 2023-03-10 22:21:18 +01:00
parent f692a47820
commit 300dbce72a
5 changed files with 57 additions and 20 deletions

2
.gitmodules vendored
View File

@ -1,4 +1,4 @@
[submodule "test/yaml-test-suite"]
path = test/yaml-test-suite
url = https://github.com/yaml/yaml-test-suite.git
branch = data-2020-08-01
branch = data-2022-01-17

View File

@ -71,23 +71,36 @@ macro genTests(): untyped =
echo "[tparser] Generating tests from " & absolutePath
discard staticExec("git submodule init && git submodule update --remote")
let errorTests = toHashSet(staticExec("cd " & (absolutePath / "tags" / "error") &
" && ls -1d *").splitLines())
var ignored = toHashSet([".git", "name", "tags", "meta"])
proc genTest(target: var NimNode, dirPath: string, testId: string) {.compileTime.} =
let title = slurp(dirPath / "===")
target.add(newCall("test",
newLit(strip(title) & " [" &
testId & ']'), newCall("doAssert", newCall("parserTest",
newLit(dirPath), newLit(fileExists(dirPath / "error"))))))
result = newStmtList()
# walkDir for some crude reason does not work with travis build
let dirItems = staticExec("ls -1d " & absolutePath / "*")
for dirPath in dirItems.splitLines():
if dirPath.strip.len == 0: continue
let testId = dirPath[^4..^1]
if ignored.contains(testId): continue
let title = slurp(dirPath / "===")
result.add(newCall("test",
newLit(strip(title) & " [" &
testId & ']'), newCall("doAssert", newCall("parserTest",
newLit(dirPath), newLit(errorTests.contains(testId))))))
if fileExists(dirPath / "==="):
genTest(result, dirPath, testId)
else:
let testItems = staticExec("ls -1d " & dirPath / "*")
var start = len(dirPath) + 1
for itemPath in testItems.splitLines():
if itemPath.strip.len == 0: continue
let testItemId = testId / itemPath[start..^1]
if fileExists(itemPath / "==="):
genTest(result, itemPath, testItemId)
result = newCall("suite", newLit("Parser Tests (from yaml-test-suite)"), result)
genTests()

@ -1 +1 @@
Subproject commit 80f289435a56189bbc5ca2f318574e48dbeef8aa
Subproject commit 6e6c296ae9c9d2d5c4134b4b64d01b29ac19ff6f

View File

@ -492,6 +492,11 @@ proc atBlockIndentationProps(c: Context, e: var Event): bool =
c.lex.next()
c.transition(atBlockIndentation)
return false
of StreamEnd, DocumentEnd, DirectivesEnd:
e = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos)
c.inlineProps = defaultProperties
c.popLevel()
return true
else:
raise c.generateError("Unexpected token (expected block content): " & $c.lex.cur)
@ -558,11 +563,6 @@ proc afterCompactParentProps(c: Context, e: var Event): bool =
c.transition(atBlockIndentation, c.levels[^3].indentation)
c.pushLevel(beforeBlockIndentation)
return false
of StreamEnd, DocumentEnd, DirectivesEnd:
e = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos)
c.inlineProps = defaultProperties
c.popLevel()
return true
of MapValueInd:
c.keyCache.add(scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos))
c.inlineProps = defaultProperties
@ -599,7 +599,7 @@ proc afterCompactParentProps(c: Context, e: var Event): bool =
else:
c.popLevel()
return true
of MapStart, SeqStart:
of MapStart, SeqStart, StreamEnd, DocumentEnd, DirectivesEnd:
c.transition(atBlockIndentationProps)
return false
else:

View File

@ -303,6 +303,15 @@ proc startLine(lex: var Lexer): LineStartType =
else: lsContent
else:
while lex.c == ' ': lex.advance()
if lex.c == '\t':
var peek = lex.source.bufpos
while lex.source.buf[peek] in space:
peek += 1
if lex.source.buf[peek] in commentOrLineEnd:
lex.source.bufpos = peek + 1
lex.c = lex.source.buf[peek]
else:
return lsContent
return case lex.c
of '#': lsComment
of '\l', '\c': lsNewline
@ -448,6 +457,7 @@ proc readBlockScalar(lex: var Lexer) =
indent = 0
separationLines = 0
contentStart: int
hasBody = true
lex.startToken()
lex.cur = if lex.c == '>': Token.Folded else: Token.Literal
lex.evaluated.setLen(0)
@ -476,7 +486,10 @@ proc readBlockScalar(lex: var Lexer) =
raise lex.generateError("Illegal character after block scalar header: " &
escape("" & lex.c))
break
of lineEnd: break
of EndOfFile:
hasBody = false
break
of '\l', '\c': break
else:
raise lex.generateError("Illegal character in block scalar header: " &
escape("" & lex.c))
@ -503,6 +516,7 @@ proc readBlockScalar(lex: var Lexer) =
of EndOfFile:
lex.state = streamEnd
lex.streamEndAfterBlock()
if lex.source.getColNumber(lex.source.bufpos) > 1 and hasBody: separationLines += 1
break body
else:
if indent == 0:
@ -586,8 +600,7 @@ proc readBlockScalar(lex: var Lexer) =
case chomp
of ctStrip: discard
of ctClip:
if len(lex.evaluated) > 0:
lex.evaluated.add('\l')
if len(lex.evaluated) > 0: lex.evaluated.add('\l')
of ctKeep:
for i in countup(0, separationLines - 1):
lex.evaluated.add('\l')
@ -813,6 +826,14 @@ proc outsideDoc(lex: var Lexer): bool =
if lex.c in commentOrLineEnd:
lex.state = expectLineEnd
return false
if lex.c == '\t':
var peek = lex.source.bufpos
while lex.source.buf[peek] in space:
peek += 1
if lex.source.buf[peek] in commentOrLineEnd:
lex.state = expectLineEnd
lex.source.bufpos = peek
return false
lex.endToken()
lex.cur = Token.Indentation
lex.indentation = -1
@ -914,6 +935,9 @@ proc flowLineStart(lex: var Lexer): bool =
while lex.c == ' ': lex.advance()
indent = lex.source.bufpos - lineStart
while lex.c in space: lex.advance()
if lex.c in commentOrLineEnd:
lex.state = expectLineEnd
return false
if indent <= lex.indentation:
raise lex.generateError("Too few indentation spaces (must surpass surrounding block level)")
lex.state = insideLine