Updated test suite, fixed lexer issues

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "test/yaml-test-suite"]
 	path = test/yaml-test-suite
 	url = https://github.com/yaml/yaml-test-suite.git
-	branch = data-2020-08-01
+	branch = data-2022-01-17

test/tparser.nim

@@ -71,23 +71,36 @@ macro genTests(): untyped =
   echo "[tparser] Generating tests from " & absolutePath
   discard staticExec("git submodule init && git submodule update --remote")
 
-  let errorTests = toHashSet(staticExec("cd " & (absolutePath / "tags" / "error") &
-                         " && ls -1d *").splitLines())
   var ignored = toHashSet([".git", "name", "tags", "meta"])
-
+  
+  proc genTest(target: var NimNode, dirPath: string, testId: string) {.compileTime.} =
+    let title = slurp(dirPath / "===")
+    
+    target.add(newCall("test",
+        newLit(strip(title) & " [" &
+        testId & ']'), newCall("doAssert", newCall("parserTest",
+        newLit(dirPath), newLit(fileExists(dirPath / "error"))))))
+  
   result = newStmtList()
+  
   # walkDir for some crude reason does not work with travis build
   let dirItems = staticExec("ls -1d " & absolutePath / "*")
   for dirPath in dirItems.splitLines():
     if dirPath.strip.len == 0: continue
     let testId = dirPath[^4..^1]
     if ignored.contains(testId): continue
-    let title = slurp(dirPath / "===")
+    if fileExists(dirPath / "==="):
-
+      genTest(result, dirPath, testId)
-    result.add(newCall("test",
+    else:
-        newLit(strip(title) & " [" &
+      let testItems = staticExec("ls -1d " & dirPath / "*")
-        testId & ']'), newCall("doAssert", newCall("parserTest",
+      var start = len(dirPath) + 1
-        newLit(dirPath), newLit(errorTests.contains(testId))))))
+      for itemPath in testItems.splitLines():
+        if itemPath.strip.len == 0: continue
+        let testItemId = testId / itemPath[start..^1]
+        if fileExists(itemPath / "==="):
+          genTest(result, itemPath, testItemId)
+        
+    
   result = newCall("suite", newLit("Parser Tests (from yaml-test-suite)"), result)
 
 genTests()

test/yaml-test-suite

@@ -1 +1 @@
-Subproject commit 80f289435a56189bbc5ca2f318574e48dbeef8aa
+Subproject commit 6e6c296ae9c9d2d5c4134b4b64d01b29ac19ff6f

yaml/parser.nim

@@ -492,6 +492,11 @@ proc atBlockIndentationProps(c: Context, e: var Event): bool =
     c.lex.next()
     c.transition(atBlockIndentation)
     return false
+  of StreamEnd, DocumentEnd, DirectivesEnd:
+    e = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos)
+    c.inlineProps = defaultProperties
+    c.popLevel()
+    return true
   else:
     raise c.generateError("Unexpected token (expected block content): " & $c.lex.cur)
 
@@ -558,11 +563,6 @@ proc afterCompactParentProps(c: Context, e: var Event): bool =
     c.transition(atBlockIndentation, c.levels[^3].indentation)
     c.pushLevel(beforeBlockIndentation)
     return false
-  of StreamEnd, DocumentEnd, DirectivesEnd:
-    e = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos)
-    c.inlineProps = defaultProperties
-    c.popLevel()
-    return true
   of MapValueInd:
     c.keyCache.add(scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos))
     c.inlineProps = defaultProperties
@@ -599,7 +599,7 @@ proc afterCompactParentProps(c: Context, e: var Event): bool =
     else:
       c.popLevel()
     return true
-  of MapStart, SeqStart:
+  of MapStart, SeqStart, StreamEnd, DocumentEnd, DirectivesEnd:
     c.transition(atBlockIndentationProps)
     return false
   else:

yaml/private/lex.nim

@@ -303,6 +303,15 @@ proc startLine(lex: var Lexer): LineStartType =
            else: lsContent
   else:
     while lex.c == ' ': lex.advance()
+    if lex.c == '\t':
+      var peek = lex.source.bufpos
+      while lex.source.buf[peek] in space:
+        peek += 1
+      if lex.source.buf[peek] in commentOrLineEnd:
+        lex.source.bufpos = peek + 1
+        lex.c = lex.source.buf[peek]
+      else:
+        return lsContent
     return case lex.c
     of '#': lsComment
     of '\l', '\c': lsNewline
@@ -448,6 +457,7 @@ proc readBlockScalar(lex: var Lexer) =
     indent = 0
     separationLines = 0
     contentStart: int
+    hasBody = true
   lex.startToken()
   lex.cur = if lex.c == '>': Token.Folded else: Token.Literal
   lex.evaluated.setLen(0)
@@ -476,7 +486,10 @@ proc readBlockScalar(lex: var Lexer) =
         raise lex.generateError("Illegal character after block scalar header: " &
             escape("" & lex.c))
       break
-    of lineEnd: break
+    of EndOfFile:
+      hasBody = false
+      break
+    of '\l', '\c': break
     else:
       raise lex.generateError("Illegal character in block scalar header: " &
           escape("" & lex.c))
@@ -503,6 +516,7 @@ proc readBlockScalar(lex: var Lexer) =
       of EndOfFile:
         lex.state = streamEnd
         lex.streamEndAfterBlock()
+        if lex.source.getColNumber(lex.source.bufpos) > 1 and hasBody: separationLines += 1
         break body
       else:
         if indent == 0:
@@ -586,8 +600,7 @@ proc readBlockScalar(lex: var Lexer) =
   case chomp
   of ctStrip: discard
   of ctClip:
-    if len(lex.evaluated) > 0:
+    if len(lex.evaluated) > 0: lex.evaluated.add('\l')
-      lex.evaluated.add('\l')
   of ctKeep:
     for i in countup(0, separationLines - 1):
       lex.evaluated.add('\l')
@@ -813,6 +826,14 @@ proc outsideDoc(lex: var Lexer): bool =
     if lex.c in commentOrLineEnd:
       lex.state = expectLineEnd
       return false
+    if lex.c == '\t':
+      var peek = lex.source.bufpos
+      while lex.source.buf[peek] in space:
+        peek += 1
+      if lex.source.buf[peek] in commentOrLineEnd:
+        lex.state = expectLineEnd
+        lex.source.bufpos = peek
+        return false
     lex.endToken()
     lex.cur = Token.Indentation
     lex.indentation = -1
@@ -914,6 +935,9 @@ proc flowLineStart(lex: var Lexer): bool =
     while lex.c == ' ': lex.advance()
     indent = lex.source.bufpos - lineStart
     while lex.c in space: lex.advance()
+  if lex.c in commentOrLineEnd:
+    lex.state = expectLineEnd
+    return false
   if indent <= lex.indentation:
     raise lex.generateError("Too few indentation spaces (must surpass surrounding block level)")
   lex.state = insideLine