got parse.nim to compile again

* it is still pretty broken

config.nims

@@ -4,6 +4,7 @@ task build, "Compile the YAML module into a library":
   setCommand "c", "yaml"
 
 task tests, "Run all tests":
+  --d:yamlDebug
   --r
   --verbosity:0
   setCommand "c", "test/tests"

private/json.nim

@@ -161,9 +161,7 @@ proc loadToJson*(s: Stream): seq[JsonNode] =
     return constructJson(events)
   except YamlConstructionError:
     var e = (ref YamlConstructionError)(getCurrentException())
-    e.line = parser.getLineNumber()
-    e.column = parser.getColNumber()
-    e.lineContent = parser.getLineContent()
+    discard events.getLastTokenContext(e.line, e.column, e.lineContent)
     raise e
   except YamlStreamError:
     let e = getCurrentException()

private/lex.nim

@@ -42,15 +42,20 @@ type
     inFlow: bool
     literalEndIndent: int
     nextState, lineStartState, inlineState, insideLineImpl, insideDocImpl,
-        insideFlowImpl: LexerState
+        insideFlowImpl, outsideDocImpl: LexerState
     blockScalarIndent: int
     c: char
-    tokenLineGetter: proc(lex: YamlLexer): string
+    tokenLineGetter: proc(lex: YamlLexer, marker: bool): string {.raises: [].}
     searchColonImpl: proc(lex: YamlLexer): bool
 
   YamlLexer* = ref YamlLexerObj
 
-  LexerState = proc(lex: YamlLexer): bool
+  YamlLexerError* = object of Exception
+    line*, column*: int
+    lineContent*: string
+
+  LexerState = proc(lex: YamlLexer): bool {.raises: YamlLexerError, locks: 0,
+      gcSafe.}
 
   LexerToken* = enum
     ltYamlDirective, ltYamlVersion, ltTagDirective, ltTagShorthand,
@@ -60,10 +65,6 @@ type
     ltBraceOpen, ltBraceClose, ltBracketOpen, ltBracketClose, ltComma,
     ltLiteralTag, ltTagHandle, ltAnchor, ltAlias
 
-  YamlLexerError* = object of Exception
-    line*, column*: int
-    lineContent*: string
-
   ChompType* = enum
     ctKeep, ctClip, ctStrip
 
@@ -83,7 +84,7 @@ const
   UTF8LineSeparator      = toUTF8(0x2028.Rune)
   UTF8ParagraphSeparator = toUTF8(0x2029.Rune)
 
-  UnknownIndentation = int.low
+  UnknownIndentation* = int.low
 
 # lexer backend implementations
 
@@ -103,7 +104,12 @@ proc advance(lex: YamlLexer, t: typedesc[StringSource], step: int = 1)
   else: lex.c = lex.sSource.src[lex.sSource.pos]
 
 template lexCR(lex: YamlLexer, t: typedesc[BaseLexer]) =
-  lex.blSource.bufpos = lex.blSource.handleCR(lex.blSource.bufpos)
+  try: lex.blSource.bufpos = lex.blSource.handleCR(lex.blSource.bufpos)
+  except:
+    var e = generateError[T](lex, "Encountered stream error: " &
+        getCurrentExceptionMsg())
+    e.parent = getCurrentException()
+    raise e
   lex.c = lex.blSource.buf[lex.blSource.bufpos]
 
 template lexCR(lex: YamlLexer, t: typedesc[StringSource]) =
@@ -114,7 +120,12 @@ template lexCR(lex: YamlLexer, t: typedesc[StringSource]) =
   lex.c = lex.sSource.src[lex.sSource.pos]
 
 template lexLF(lex: YamlLexer, t: typedesc[BaseLexer]) =
-  lex.blSource.bufpos = lex.blSource.handleLF(lex.blSource.bufpos)
+  try: lex.blSource.bufpos = lex.blSource.handleLF(lex.blSource.bufpos)
+  except:
+    var e = generateError[T](lex, "Encountered stream error: " &
+        getCurrentExceptionMsg())
+    e.parent = getCurrentException()
+    raise e
   lex.c = lex.blSource.buf[lex.blSource.bufpos]
 
 template lexLF(lex: YamlLexer, t: typedesc[StringSource]) =
@@ -180,14 +191,15 @@ proc afterMark(lex: YamlLexer, t: typedesc[StringSource], m: int):
     int {.inline.} =
   lex.sSource.pos - m
 
-proc lineWithMarker(lex: YamlLexer, t: typedesc[BaseLexer]):
+proc lineWithMarker(lex: YamlLexer, t: typedesc[BaseLexer], marker: bool):
     string =
   if lex.curStartPos.line == lex.blSource.lineNumber:
-    result = lex.blSource.getCurrentLine(false) &
-        spaces(lex.curStartPos.column - 1) & "^\n"
+    result = lex.blSource.getCurrentLine(false)
+    if marker: result.add(spaces(lex.curStartPos.column - 1) & "^\n")
   else: result = nil
 
-proc lineWithMarker(lex: YamlLexer, t: typedesc[StringSource]): string =
+proc lineWithMarker(lex: YamlLexer, t: typedesc[StringSource], marker: bool):
+    string =
   var
     lineStartIndex = lex.sSource.pos
     lineEndIndex: int
@@ -209,11 +221,12 @@ proc lineWithMarker(lex: YamlLexer, t: typedesc[StringSource]): string =
       dec(lineEndIndex)
     dec(curLine)
   result = lex.sSource.src.substr(lineStartIndex,
-      lineEndIndex - lineStartIndex - 1) & "\n" &
-      spaces(lex.curStartPos.column - 1) & "^"
+      lineEndIndex - lineStartIndex - 1) & "\n"
+  if marker: result.add(spaces(lex.curStartPos.column - 1) & "^\n")
 
 # lexer states
 
+{.push raises: YamlLexerError, gcSafe, locks: 0.}
 proc outsideDoc[T](lex: YamlLexer): bool
 proc yamlVersion[T](lex: YamlLexer): bool
 proc tagShorthand[T](lex: YamlLexer): bool
@@ -223,7 +236,7 @@ proc expectLineEnd[T](lex: YamlLexer): bool
 proc possibleDirectivesEnd[T](lex: YamlLexer): bool
 proc possibleDocumentEnd[T](lex: YamlLexer): bool
 proc afterSeqInd[T](lex: YamlLexer): bool
-proc insideDoc[T](lex: YamlLexer): bool {.locks:0.}
+proc insideDoc[T](lex: YamlLexer): bool
 proc insideFlow[T](lex: YamlLexer): bool
 proc insideLine[T](lex: YamlLexer): bool
 proc plainScalarPart[T](lex: YamlLexer): bool
@@ -233,6 +246,7 @@ proc tagHandle[T](lex: YamlLexer): bool
 proc anchor[T](lex: YamlLexer): bool
 proc alias[T](lex: YamlLexer): bool
 proc streamEnd(lex: YamlLexer): bool
+{.pop.}
 
 # implementation
 
@@ -256,7 +270,7 @@ proc directiveName[T](lex: YamlLexer) =
     lex.buf.add(lex.c)
     lex.advance(T)
 
-proc consumeNewlines(lex: YamlLexer) =
+proc consumeNewlines(lex: YamlLexer) {.raises: [].} =
   case lex.newlines
   of 0: return
   of 1: lex.buf.add(' ')
@@ -286,6 +300,7 @@ proc yamlVersion[T](lex: YamlLexer): bool =
     lex.advance(T)
   if lex.c notin spaceOrLineEnd:
     raise generateError[T](lex, "Invalid YAML version number")
+  echo "Yaml version!"
   lex.cur = ltYamlVersion
   result = true
   lex.nextState = expectLineEnd[T]
@@ -368,12 +383,12 @@ proc expectLineEnd[T](lex: YamlLexer): bool =
       lex.nextState = lex.lineStartState
       break
     else:
-      raise generateError[T](lex, "Unexpected character (expected line end): " &
-          escape("" & lex.c))
+      raise generateError[T](lex,
+          "Unexpected character (expected line end): " & escape("" & lex.c))
 
 proc possibleDirectivesEnd[T](lex: YamlLexer): bool =
   debug("lex: possibleDirectivesEnd")
-  lex.lineStartState = insideDoc[T]
+  lex.lineStartState = lex.insideDocImpl  # could be insideDoc[T]
   lex.advance(T)
   if lex.c == '-':
     lex.advance(T)
@@ -388,6 +403,7 @@ proc possibleDirectivesEnd[T](lex: YamlLexer): bool =
     else: lex.consumeNewlines()
     lex.buf.add('-')
   elif lex.c in spaceOrLineEnd:
+    echo "spacey"
     lex.indentation = 0
     lex.cur = ltIndentation
     lex.nextState = afterSeqInd[T]
@@ -413,7 +429,7 @@ proc possibleDocumentEnd[T](lex: YamlLexer): bool =
       if lex.c in spaceOrLineEnd:
         lex.cur = ltDocumentEnd
         lex.nextState = expectLineEnd[T]
-        lex.lineStartState = outsideDoc[T]
+        lex.lineStartState = lex.outsideDocImpl
         return true
       lex.consumeNewlines()
       lex.buf.add('.')
@@ -427,6 +443,7 @@ proc possibleDocumentEnd[T](lex: YamlLexer): bool =
 proc outsideDoc[T](lex: YamlLexer): bool =
   debug("lex: outsideDoc")
   startToken[T](lex)
+  echo lex.c
   case lex.c
   of '%':
     lex.advance(T)
@@ -717,12 +734,12 @@ proc plainScalarPart[T](lex: YamlLexer): bool =
             if lex.nextIsPlainSafe(T, lex.inFlow): break
             else:
               lex.buf.setLen(lenBeforeSpace)
-              lex.nextState = insideLine[T]
+              lex.nextState = lex.insideLineImpl # could be insideLine[T]
               break outer
           of flowIndicators:
             if lex.inFlow:
               lex.buf.setLen(lenBeforeSpace)
-              lex.nextState = insideLine[T]
+              lex.nextState = lex.insideLineImpl # could be insideLine[T]
               break outer
             else:
               lex.buf.add(lex.c)
@@ -735,11 +752,11 @@ proc plainScalarPart[T](lex: YamlLexer): bool =
         break
       of flowIndicators:
         if lex.inFlow:
-          lex.nextState = insideLine[T]
+          lex.nextState = lex.insideLineImpl # could be insideLine[T]
           break
       of ':':
         if not lex.nextIsPlainSafe(T, lex.inFlow):
-          lex.nextState = insideLine[T]
+          lex.nextState = lex.insideLineImpl # could be insideLine[T]
           break outer
       else: discard
   lex.cur = ltScalarPart
@@ -861,7 +878,7 @@ proc tagHandle[T](lex: YamlLexer): bool =
     lex.cur = ltTagHandle
   while lex.c in space: lex.advance(T)
   if lex.c in lineEnd: lex.nextState = expectLineEnd[T]
-  else: lex.nextState = insideLine[T]
+  else: lex.nextState = lex.insideLineImpl # could be insideLine[T]
   result = true
 
 proc anchorName[T](lex: YamlLexer) =
@@ -874,7 +891,7 @@ proc anchorName[T](lex: YamlLexer) =
     else: lex.buf.add(lex.c)
   while lex.c in space: lex.advance(T)
   if lex.c in lineEnd: lex.nextState = expectLineEnd[T]
-  else: lex.nextState = insideLine[T]
+  else: lex.nextState = lex.insideLineImpl # could be insideLine[T]
 
 proc anchor[T](lex: YamlLexer): bool =
   debug("lex: anchor")
@@ -893,8 +910,8 @@ proc streamEnd(lex: YamlLexer): bool =
   lex.cur = ltStreamEnd
   result = true
 
-proc tokenLine[T](lex: YamlLexer): string =
-  result = lex.lineWithMarker(T)
+proc tokenLine[T](lex: YamlLexer, marker: bool): string =
+  result = lex.lineWithMarker(T, marker)
 
 proc searchColon[T](lex: YamlLexer): bool =
   var flowDepth = if lex.cur in [ltBraceOpen, ltBracketOpen]: 1 else: 0
@@ -977,6 +994,7 @@ proc init*[T](lex: YamlLexer) =
   lex.insideLineImpl = insideLine[T]
   lex.insideDocImpl = insideDoc[T]
   lex.insideFlowImpl = insideFlow[T]
+  lex.outsideDocImpl = outsideDoc[T] # only needed because of compiler checks
   lex.tokenLineGetter = tokenLine[T]
   lex.searchColonImpl = searchColon[T]
 
@@ -1020,8 +1038,8 @@ proc endBlockScalar*(lex: YamlLexer) =
   lex.inlineState = lex.insideLineImpl
   lex.nextState = lex.insideLineImpl
 
-proc getTokenLine*(lex: YamlLexer): string =
-  result = lex.tokenLineGetter(lex)
+proc getTokenLine*(lex: YamlLexer, marker: bool = true): string =
+  result = lex.tokenLineGetter(lex, marker)
 
 proc isImplicitKeyStart*(lex: YamlLexer): bool =
   result = lex.searchColonImpl(lex)

private/serialization.nim

@@ -762,9 +762,7 @@ proc load*[K](input: Stream, target: var K) =
   try: construct(events, target)
   except YamlConstructionError:
     var e = (ref YamlConstructionError)(getCurrentException())
-    e.line = parser.getLineNumber()
-    e.column = parser.getColNumber()
-    e.lineContent = parser.getLineContent()
+    discard events.getLastTokenContext(e.line, e.column, e.lineContent)
     raise e
   except YamlStreamError:
     let e = (ref YamlStreamError)(getCurrentException())

private/stream.nim

@@ -4,6 +4,11 @@
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 
+proc noLastContext(s: YamlStream, line, column: var int,
+    lineContent: var string): bool =
+  (line, column, lineContent) = (-1, -1, "")
+  result = false
+
 when not defined(JS):
   type IteratorYamlStream = ref object of YamlStream
     backend: iterator(): YamlStreamEvent
@@ -19,6 +24,7 @@ when not defined(JS):
         s.isFinished = true
         result = false
       else: result = true
+    result.lastTokenContextImpl = noLastContext
 
 type
   BufferYamlStream = ref object of YamlStream
@@ -27,6 +33,7 @@ type
 
 proc newBufferYamlStream(): BufferYamlStream not nil =
   BufferYamlStream(peeked: false, isFinished: false, buf: @[], pos: 0,
+      lastTokenContextImpl: noLastContext,
       nextImpl: proc(s: YamlStream, e: var YamlStreamEvent): bool =
         let bys = BufferYamlStream(s)
         if bys.pos == bys.buf.len:
@@ -88,3 +95,7 @@ proc finished*(s: YamlStream): bool =
       var e = newException(YamlStreamError, cur.msg)
       e.parent = cur
       raise e
+
+proc getLastTokenContext*(s: YamlStream, line, column: var int,
+    lineContent: var string): bool =
+  result = s.lastTokenContextImpl(s, line, column, lineContent)

test/constructingJson.nim

@@ -18,7 +18,8 @@ proc ensureEqual(yamlIn, jsonIn: string) =
     yamlResult = constructJson(s)
     jsonResult = parseJson(jsonIn)
   assert yamlResult.len == 1
-  assert(jsonResult == yamlResult[0])
+  assert(jsonResult == yamlResult[0], "Expected: " & $jsonResult & ", got: " &
+      $yamlResult[0])
 
 suite "Constructing JSON":
   test "Constructing JSON: Simple Sequence":

yaml.nim

@@ -113,6 +113,9 @@ type
     ## always yield a well-formed ``YamlStream`` and expect it to be
     ## well-formed if they take it as input parameter.
     nextImpl: proc(s: YamlStream, e: var YamlStreamEvent): bool
+    lastTokenContextImpl:
+        proc(s: YamlStream, line, column: var int,
+             lineContent: var string): bool {.raises: [].}
     isFinished: bool
     peeked: bool
     cached: YamlStreamEvent
@@ -459,6 +462,11 @@ proc `peek=`*(s: YamlStream, value: YamlStreamEvent) {.raises: [].}
 proc finished*(s: YamlStream): bool {.raises: [YamlStreamError].}
   ## ``true`` if no more items are available in the stream. Handles exceptions
   ## of the backend like ``next()``.
+proc getLastTokenContext*(s: YamlStream, line, column: var int,
+    lineContent: var string): bool
+  ## ``true`` if source context information is available about the last returned
+  ## token. If ``true``, line, column and lineContent are set to position and
+  ## line content where the last token has been read from.
 iterator items*(s: YamlStream): YamlStreamEvent
     {.raises: [YamlStreamError].} =
   ## Iterate over all items of the stream. You may not use ``peek()`` on the
@@ -511,21 +519,6 @@ proc newYamlParser*(tagLib: TagLibrary = initExtendedTagLibrary(),
   ## Creates a YAML parser. if ``callback`` is not ``nil``, it will be called
   ## whenever the parser yields a warning.
 
-proc getLineNumber*(p: YamlParser): int {.raises: [].}
-  ## Get the line number (1-based) of the recently yielded parser token.
-  ## Useful for error reporting at later loading stages.
-
-proc getColNumber*(p: YamlParser): int {.raises: [].}
-  ## Get the column number (1-based) of the recently yielded parser token.
-  ## Useful for error reporting at later parsing stages.
-
-proc getLineContent*(p: YamlParser, marker: bool = true): string {.raises: [].}
-  ## Get the content of the input line containing the recently yielded parser
-  ## token. Useful for error reporting at later parsing stages. The line will
-  ## be terminated by ``"\n"``. If ``marker`` is ``true``, a second line will
-  ## be returned containing a ``^`` at the position of the recent parser
-  ## token.
-
 proc parse*(p: YamlParser, s: Stream): YamlStream {.raises: [YamlParserError].}
   ## Parse the given stream as YAML character stream.
   ## The only Exception that can be raised comes from opening the Stream.