Started fixing yaml test suite errors

private/lex.nim

@@ -396,6 +396,7 @@ proc possibleDirectivesEnd[T](lex: YamlLexer): bool =
       lex.advance(T)
       if lex.c in spaceOrLineEnd:
         lex.cur = ltDirectivesEnd
+        while lex.c in space: lex.advance(T)
         lex.nextState = insideLine[T]
         return true
       lex.consumeNewlines()
@@ -415,7 +416,9 @@ proc possibleDirectivesEnd[T](lex: YamlLexer): bool =
 proc afterSeqInd[T](lex: YamlLexer): bool =
   result = true
   lex.cur = ltSeqItemInd
-  if lex.c notin lineEnd: lex.advance(T)
+  if lex.c notin lineEnd:
+    lex.advance(T)
+    while lex.c in space: lex.advance(T)
   lex.nextState = insideLine[T]
 
 proc possibleDocumentEnd[T](lex: YamlLexer): bool =
@@ -494,10 +497,13 @@ proc insideDoc[T](lex: YamlLexer): bool =
     while lex.c == ' ':
       lex.indentation.inc()
       lex.advance(T)
-    if lex.c in spaceOrLineEnd:
+    case lex.c
+    of lineEnd:
       lex.cur = ltEmptyLine
       lex.nextState = expectLineEnd[T]
       return true
+    of '\t':
+      raise generateError[T](lex, "'\\t' cannot start any token")
     else:
       lex.nextState = lex.inlineState
   else: lex.nextState = lex.inlineState
@@ -518,7 +524,7 @@ proc insideFlow[T](lex: YamlLexer): bool =
 proc possibleIndicatorChar[T](lex: YamlLexer, indicator: LexerToken,
     jsonContext: bool = false): bool =
   startToken[T](lex)
-  if not(jsonContext) and lex.nextIsPlainSafe(T, false):
+  if not(jsonContext) and lex.nextIsPlainSafe(T, lex.inFlow):
     lex.consumeNewlines()
     lex.nextState = plainScalarPart[T]
     result = false

private/parse.nim

@@ -48,6 +48,13 @@ proc generateError(c: ParserContext, message: string):
   (result.line, result.column) = c.lex.curStartPos
   result.lineContent = c.lex.getTokenLine()
 
+proc illegalToken(c: ParserContext, expected: string = ""):
+    ref YamlParserError {.raises: [].} =
+  var msg = "Illegal token"
+  if expected.len > 0: msg.add(" (expected " & expected & ")")
+  msg.add(": " & $c.lex.cur)
+  result = c.generateError(msg)
+
 proc callCallback(c: ParserContext, msg: string) {.raises: [YamlParserError].} =
   try:
     if not isNil(c.p.callback):
@@ -155,6 +162,7 @@ proc handleTagHandle(c: ParserContext) {.raises: [YamlParserError].} =
   else:
     try: c.tag = c.p.tagLib.tags[c.lex.buf]
     except KeyError: c.tag = c.p.tagLib.registerUri(c.lex.buf)
+  c.lex.buf.setLen(0)
   c.advance()
 
 proc handlePossibleMapStart(c: ParserContext, e: var YamlStreamEvent,
@@ -164,6 +172,7 @@ proc handlePossibleMapStart(c: ParserContext, e: var YamlStreamEvent,
     if c.lex.isImplicitKeyStart():
       e = c.objectStart(yamlStartMap, single)
       result = true
+    c.level.indentation = c.lex.indentation
 
 proc handleMapKeyIndicator(c: ParserContext, e: var YamlStreamEvent): bool =
   result = false
@@ -245,6 +254,7 @@ proc handleFlowItemStart(c: ParserContext, e: var YamlStreamEvent): bool =
   if c.level.kind == fplUnknown and
       c.ancestry[c.ancestry.high].kind == fplSequence:
     result = c.handlePossibleMapStart(e, true, true)
+  else: result = false
 
 proc handleFlowPlainScalar(c: ParserContext) =
   while c.lex.cur in {ltScalarPart, ltEmptyLine}:
@@ -558,7 +568,7 @@ parserState plainScalarEnd:
 
 parserState blockAfterObject:
   case c.lex.cur
-  of ltIndentation:
+  of ltIndentation, ltEmptyLine:
     c.advance()
     state = blockLineStart
   of ltMapValInd:
@@ -576,18 +586,19 @@ parserState blockAfterObject:
       c.level.kind = fplMapValue
       c.ancestry.add(c.level)
       c.level = initLevel(fplUnknown)
-    of fplSequence:
+    of fplSequence: raise c.illegalToken("sequence item")
-      raise c.generateError("Illegal token (expected sequence item)")
     of fplSinglePairKey, fplSinglePairValue, fplDocument:
       internalError("Unexpected level kind: " & $c.level.kind)
     c.advance()
     state = blockObjectStart
+  of ltDirectivesEnd:
+    c.closeEverything()
+    stored = startDoc
+    c.advance()
   of ltStreamEnd:
     c.closeEverything()
     stored = afterDocument
-  else:
+  else: raise c.illegalToken("':', comment or line end")
-    raise c.generateError(
-        "Illegal token (expected ':', comment or line end)")
 
 parserState objectEnd:
   if c.handleObjectEnd(true):
@@ -694,7 +705,9 @@ parserState alias:
   var id: AnchorId
   try: id = c.p.anchors[c.lex.buf]
   except KeyError: raise c.generateError("Unknown anchor")
+  c.lex.buf.setLen(0)
   e = aliasEvent(id)
+  c.advance()
   result = true
   state = objectEnd
 
@@ -706,6 +719,7 @@ parserState flow:
     result = true
     c.flowdepth.inc()
     c.explicitFlowKey = false
+    c.advance()
   of ltBracketOpen:
     if c.handleFlowItemStart(e): return true
     e = c.objectStart(yamlStartSeq)
@@ -786,6 +800,7 @@ parserState flow:
     result = true
     state = objectEnd
     stored = flowAfterObject
+    c.advance()
   of ltTagHandle:
     if c.handleFlowItemStart(e): return true
     c.handleTagHandle()
@@ -811,7 +826,6 @@ parserState flow:
   of ltScalarPart:
     if c.handleFlowItemStart(e): return true
     c.handleFlowPlainScalar()
-    if c.tag == yTagQuestionMark: c.tag = yTagExclamationMark
     e = c.currentScalar()
     result = true
     state = objectEnd