From c3810fbb22b82406c5dd093ec40cac17ea29c130 Mon Sep 17 00:00:00 2001
From: Felix Krause <contact@flyx.org>
Date: Thu, 12 May 2016 15:35:29 +0200
Subject: [PATCH 1/4] Made server output syntax-highlighted YAML

---
 doc/testing.html  |  5 +----
 server/server.nim | 24 +++++++++++++++++++++---
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/doc/testing.html b/doc/testing.html
index 553668f..99ac2b8 100644
--- a/doc/testing.html
+++ b/doc/testing.html
@@ -88,10 +88,7 @@
   </section>
   <script type="text/javascript">
     function setTextContent(element, text) {
-      while (element.firstChild!==null) {
-        element.removeChild(element.firstChild); // remove all existing content
-      }
-      element.appendChild(document.createTextNode(text));
+      element.innerHTML = text;
     }
     function parse() {
       var r = new XMLHttpRequest();
diff --git a/server/server.nim b/server/server.nim
index c0cc0ba..335006d 100644
--- a/server/server.nim
+++ b/server/server.nim
@@ -4,7 +4,8 @@
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 
-import jester, asyncdispatch, json, streams
+import jester, asyncdispatch, json, streams, strutils
+import packages.docutils.rstgen, packages.docutils.highlite
 import yaml
 
 routes:
@@ -38,10 +39,27 @@ routes:
         resp resultNode.pretty, "application/json"
         tokens = true
       if not tokens:
-        var output = newStringStream()
+        var
+          output = newStringStream()
+          highlighted = ""
         transform(newStringStream(@"input"), output, defineOptions(style))
+        
+        # syntax highlighting (stolen and modified from stlib's rstgen)
+        var g: GeneralTokenizer
+        g.initGeneralTokenizer(output.data)
+        while true:
+          g.getNextToken(langYaml)
+          case g.kind
+          of gtEof: break
+          of gtNone, gtWhitespace:
+            highlighted.add(substr(output.data, g.start, g.length + g.start - 1))
+          else:
+            highlighted.addf("<span class=\"$2\">$1</span>", "\\span$2{$1}", [
+              esc(outHtml, substr(output.data, g.start, g.length+g.start-1)),
+              tokenClassToStr[g.kind]])
+
         resultNode["code"] = %0
-        resultNode["output"] = %output.data
+        resultNode["output"] = %highlighted
         resp resultNode.pretty, "application/json"
     except YamlParserError:
       let e = (ref YamlParserError)(getCurrentException())

From 10fec5fb1f3b4f1c3262600db73dd6a0ff9301a0 Mon Sep 17 00:00:00 2001
From: Felix Krause <contact@flyx.org>
Date: Sat, 21 May 2016 22:08:19 +0200
Subject: [PATCH 2/4] fastparse refactoring, part I

 * Replaced lots of templates with procs
 * Some minor improvements
---
 private/fastparse.nim | 471 ++++++++++++++++++++++--------------------
 1 file changed, 245 insertions(+), 226 deletions(-)

diff --git a/private/fastparse.nim b/private/fastparse.nim
index 05c10b6..d7b9b4f 100644
--- a/private/fastparse.nim
+++ b/private/fastparse.nim
@@ -67,28 +67,28 @@ template debugFail() {.dirty.} =
     echo "internal error at line: ", instantiationInfo().line
   assert(false)
 
-template parserError(message: string) {.dirty.} =
-  var e = newException(YamlParserError, message)
-  e.line = p.lexer.lineNumber
-  e.column = p.tokenstart + 1
-  e.lineContent = p.getLineContent(true)
-  raise e
+proc generateError(p: YamlParser, message: string):
+    ref YamlParserError {.raises: [].} =
+  result = newException(YamlParserError, message)
+  result.line = p.lexer.lineNumber
+  result.column = p.tokenstart + 1
+  result.lineContent = p.getLineContent(true)
 
-template lexerError(lx: BaseLexer, message: string) {.dirty.} =
-  var e = newException(YamlParserError, message)
-  e.line = lx.lineNumber
-  e.column = lx.bufpos + 1
-  e.lineContent = lx.getCurrentLine(false) &
+proc generateError(lx: BaseLexer, message: string):
+    ref YamlParserError {.raises: [].} =
+  result = newException(YamlParserError, message)
+  result.line = lx.lineNumber
+  result.column = lx.bufpos + 1
+  result.lineContent = lx.getCurrentLine(false) &
       repeat(' ', lx.getColNumber(lx.bufpos)) & "^\n"
-  raise e
 
-template addMultiple(s: var string, c: char, num: int) =
+proc addMultiple(s: var string, c: char, num: int) {.raises: [], inline.} =
   for i in 1..num:
     s.add(c)
 
-template startContent() {.dirty.} = content.setLen(0)
+proc reset(content: var string) {.raises: [], inline.} = content.setLen(0)
 
-template initLevel(k: FastParseLevelKind): FastParseLevel =
+proc initLevel(k: FastParseLevelKind): FastParseLevel {.raises: [], inline.} =
   FastParseLevel(kind: k, indentation: UnknownIndentation)
 
 template yieldEmptyScalar() {.dirty.} =
@@ -205,44 +205,45 @@ template closeEverything() {.dirty.} =
   yield endDocEvent()
 
 template handleBlockSequenceIndicator() {.dirty.} =
-  startToken()
+  p.startToken()
   case level.kind
   of fplUnknown: handleObjectStart(yamlStartSeq)
   of fplSequence:
     if level.indentation != indentation:
-      parserError("Invalid indentation of block sequence indicator")
+      raise p.generateError("Invalid indentation of block sequence indicator")
     ancestry.add(level)
     level = initLevel(fplUnknown)
-  else: parserError("Illegal sequence item in map")
+  else: raise p.generateError("Illegal sequence item in map")
   p.lexer.skipWhitespace()
   indentation = p.lexer.getColNumber(p.lexer.bufpos)
 
 template handleMapKeyIndicator() {.dirty.} =
-  startToken()
+  p.startToken()
   case level.kind
   of fplUnknown: handleObjectStart(yamlStartMap)
   of fplMapValue:
     if level.indentation != indentation:
-      parserError("Invalid indentation of map key indicator")
+      raise p.generateError("Invalid indentation of map key indicator")
     yield scalarEvent("", yTagQuestionMark, yAnchorNone)
     level.kind = fplMapKey
     ancestry.add(level)
     level = initLevel(fplUnknown)
   of fplMapKey:
     if level.indentation != indentation:
-      parserError("Invalid indentation of map key indicator")
+      raise p.generateError("Invalid indentation of map key indicator")
     ancestry.add(level)
     level = initLevel(fplUnknown)
   of fplSequence:
-    parserError("Unexpected map key indicator (expected '- ')")
+    raise p.generateError("Unexpected map key indicator (expected '- ')")
   of fplScalar:
-    parserError("Unexpected map key indicator (expected multiline scalar end)")
+    raise p.generateError(
+        "Unexpected map key indicator (expected multiline scalar end)")
   of fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail()
   p.lexer.skipWhitespace()
   indentation = p.lexer.getColNumber(p.lexer.bufpos)
 
 template handleMapValueIndicator() {.dirty.} =
-  startToken()
+  p.startToken()
   case level.kind
   of fplUnknown:
     if level.indentation == UnknownIndentation:
@@ -252,20 +253,20 @@ template handleMapValueIndicator() {.dirty.} =
     ancestry[ancestry.high].kind = fplMapValue
   of fplMapKey:
     if level.indentation != indentation:
-      parserError("Invalid indentation of map key indicator")
+      raise p.generateError("Invalid indentation of map key indicator")
     yield scalarEvent("", yTagQuestionMark, yAnchorNone)
     level.kind = fplMapValue
     ancestry.add(level)
     level = initLevel(fplUnknown)
   of fplMapValue:
     if level.indentation != indentation:
-      parserError("Invalid indentation of map key indicator")
+      raise p.generateError("Invalid indentation of map key indicator")
     ancestry.add(level)
     level = initLevel(fplUnknown)
   of fplSequence:
-    parserError("Unexpected map value indicator (expected '- ')")
+    raise p.generateError("Unexpected map value indicator (expected '- ')")
   of fplScalar:
-    parserError(
+    raise p.generateError(
         "Unexpected map value indicator (expected multiline scalar end)")
   of fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail()
   p.lexer.skipWhitespace()
@@ -283,11 +284,11 @@ template initDocValues() {.dirty.} =
   ancestry.add(FastParseLevel(kind: fplDocument, indentation: -1))
 
 template handleTagHandle() {.dirty.} =
-  startToken()
-  if level.kind != fplUnknown: parserError("Unexpected tag handle")
+  p.startToken()
+  if level.kind != fplUnknown: raise p.generateError("Unexpected tag handle")
   if tag != yTagQuestionMark:
-    parserError("Only one tag handle is allowed per node")
-  startContent()
+    raise p.generateError("Only one tag handle is allowed per node")
+  content.reset()
   var
     shorthandEnd: int
   p.lexer.tagHandle(content, shorthandEnd)
@@ -297,7 +298,8 @@ template handleTagHandle() {.dirty.} =
       tagUri.add(shorthands[content[0..shorthandEnd]])
       tagUri.add(content[shorthandEnd + 1 .. ^1])
     except KeyError:
-      parserError("Undefined tag shorthand: " & content[0..shorthandEnd])
+      raise p.generateError(
+          "Undefined tag shorthand: " & content[0..shorthandEnd])
     try: tag = p.tagLib.tags[tagUri]
     except KeyError: tag = p.tagLib.registerUri(tagUri)
   else:
@@ -305,26 +307,26 @@ template handleTagHandle() {.dirty.} =
     except KeyError: tag = p.tagLib.registerUri(content)
 
 template handleAnchor() {.dirty.} =
-  startToken()
-  if level.kind != fplUnknown: parserError("Unexpected token")
+  p.startToken()
+  if level.kind != fplUnknown: raise p.generateError("Unexpected token")
   if anchor != yAnchorNone:
-    parserError("Only one anchor is allowed per node")
-  startContent()
+    raise p.generateError("Only one anchor is allowed per node")
+  content.reset()
   p.lexer.anchorName(content)
   anchor = nextAnchorId
   anchors[content] = anchor
-  nextAnchorId = cast[AnchorId](cast[int](nextAnchorId) + 1)
+  nextAnchorId = AnchorId(int(nextAnchorId) + 1)
 
 template handleAlias() {.dirty.} =
-  startToken()
-  if level.kind != fplUnknown: parserError("Unexpected token")
+  p.startToken()
+  if level.kind != fplUnknown: raise p.generateError("Unexpected token")
   if anchor != yAnchorNone or tag != yTagQuestionMark:
-    parserError("Alias may not have anchor or tag")
-  startContent()
+    raise p.generateError("Alias may not have anchor or tag")
+  content.reset()
   p.lexer.anchorName(content)
   var id: AnchorId
   try: id = anchors[content]
-  except KeyError: parserError("Unknown anchor")
+  except KeyError: raise p.generateError("Unknown anchor")
   yield aliasEvent(id)
   handleObjectEnd(fpBlockAfterObject)
 
@@ -385,7 +387,8 @@ template handleBlockItemStart() {.dirty.} =
   case level.kind
   of fplUnknown: handlePossibleMapStart()
   of fplSequence:
-    parserError("Unexpected token (expected block sequence indicator)")
+    raise p.generateError(
+        "Unexpected token (expected block sequence indicator)")
   of fplMapKey:
     ancestry.add(level)
     level = FastParseLevel(kind: fplUnknown, indentation: indentation)
@@ -400,19 +403,20 @@ template handleFlowItemStart() {.dirty.} =
   if level.kind == fplUnknown and ancestry[ancestry.high].kind == fplSequence:
     handlePossibleMapStart(true, true)
 
-template startToken() {.dirty.} =
+proc startToken(p: YamlParser) {.raises: [], inline.} =
   p.tokenstart = p.lexer.getColNumber(p.lexer.bufpos)
 
-template finishLine(lexer: BaseLexer) =
+proc finishLine(lexer: var BaseLexer) {.raises: [], inline.} =
   debug("lex: finishLine")
   while lexer.buf[lexer.bufpos] notin lineEnd:
     lexer.bufpos.inc()
 
-template skipWhitespace(lexer: BaseLexer) =
+proc skipWhitespace(lexer: var BaseLexer) {.raises: [], inline.} =
   debug("lex: skipWhitespace")
   while lexer.buf[lexer.bufpos] in space: lexer.bufpos.inc()
 
-template skipWhitespaceCommentsAndNewlines(lexer: BaseLexer) =
+# TODO: {.raises: [].}
+proc skipWhitespaceCommentsAndNewlines(lexer: var BaseLexer) {.inline.} =
   debug("lex: skipWhitespaceCommentsAndNewlines")
   if lexer.buf[lexer.bufpos] != '#':
     while true:
@@ -426,11 +430,12 @@ template skipWhitespaceCommentsAndNewlines(lexer: BaseLexer) =
           lexer.bufpos.inc()
       else: break
 
-template skipIndentation(lexer: BaseLexer) =
+proc skipIndentation(lexer: var BaseLexer) {.raises: [], inline.} =
   debug("lex: skipIndentation")
   while lexer.buf[lexer.bufpos] == ' ': lexer.bufpos.inc()
 
-template directiveName(lexer: BaseLexer, directive: var LexedDirective) =
+proc directiveName(lexer: var BaseLexer, directive: var LexedDirective)
+    {.raises: [].} =
   debug("lex: directiveName")
   directive = ldUnknown
   lexer.bufpos.inc()
@@ -455,11 +460,12 @@ template directiveName(lexer: BaseLexer, directive: var LexedDirective) =
   while lexer.buf[lexer.bufpos] notin spaceOrLineEnd:
     lexer.bufpos.inc()
 
-template yamlVersion(lexer: BaseLexer, o: var string) =
+proc yamlVersion(lexer: var BaseLexer, o: var string)
+    {.raises: [YamlParserError], inline.} =
   debug("lex: yamlVersion")
   while lexer.buf[lexer.bufpos] in space: lexer.bufpos.inc()
   var c = lexer.buf[lexer.bufpos]
-  if c notin digits: lexerError(lexer, "Invalid YAML version number")
+  if c notin digits: raise lexer.generateError("Invalid YAML version number")
   o.add(c)
   lexer.bufpos.inc()
   c = lexer.buf[lexer.bufpos]
@@ -468,11 +474,11 @@ template yamlVersion(lexer: BaseLexer, o: var string) =
     o.add(c)
     c = lexer.buf[lexer.bufpos]
   if lexer.buf[lexer.bufpos] != '.':
-    lexerError(lexer, "Invalid YAML version number")
+    raise lexer.generateError("Invalid YAML version number")
   o.add('.')
   lexer.bufpos.inc()
   c = lexer.buf[lexer.bufpos]
-  if c notin digits: lexerError(lexer, "Invalid YAML version number")
+  if c notin digits: raise lexer.generateError("Invalid YAML version number")
   o.add(c)
   lexer.bufpos.inc()
   c = lexer.buf[lexer.bufpos]
@@ -481,20 +487,20 @@ template yamlVersion(lexer: BaseLexer, o: var string) =
     lexer.bufpos.inc()
     c = lexer.buf[lexer.bufpos]
   if lexer.buf[lexer.bufpos] notin spaceOrLineEnd:
-    lexerError(lexer, "Invalid YAML version number")
+    raise lexer.generateError("Invalid YAML version number")
 
-template lineEnding(lexer: BaseLexer) =
+proc lineEnding(p: YamlParser) {.raises: [YamlParserError], inline.} =
   debug("lex: lineEnding")
-  if lexer.buf[lexer.bufpos] notin lineEnd:
-    while lexer.buf[lexer.bufpos] in space: lexer.bufpos.inc()
-    if lexer.buf[lexer.bufpos] in lineEnd: discard
-    elif lexer.buf[lexer.bufpos] == '#':
-      while lexer.buf[lexer.bufpos] notin lineEnd: lexer.bufpos.inc()
+  if p.lexer.buf[p.lexer.bufpos] notin lineEnd:
+    while p.lexer.buf[p.lexer.bufpos] in space: p.lexer.bufpos.inc()
+    if p.lexer.buf[p.lexer.bufpos] in lineEnd: discard
+    elif p.lexer.buf[p.lexer.bufpos] == '#':
+      while p.lexer.buf[p.lexer.bufpos] notin lineEnd: p.lexer.bufpos.inc()
     else:
-      startToken()
-      parserError("Unexpected token (expected comment or line end)")
+      p.startToken()
+      raise p.generateError("Unexpected token (expected comment or line end)")
 
-template tagShorthand(lexer: BaseLexer, shorthand: var string) =
+proc tagShorthand(lexer: var BaseLexer, shorthand: var string) {.inline.} =
   debug("lex: tagShorthand")
   while lexer.buf[lexer.bufpos] in space: lexer.bufpos.inc()
   assert lexer.buf[lexer.bufpos] == '!'
@@ -509,13 +515,14 @@ template tagShorthand(lexer: BaseLexer, shorthand: var string) =
         shorthand.add(c)
         lexer.bufpos.inc()
         c = lexer.buf[lexer.bufpos]
-      else: lexerError(lexer, "Illegal character in tag shorthand")
+      else: raise lexer.generateError("Illegal character in tag shorthand")
     shorthand.add(c)
     lexer.bufpos.inc()
   if lexer.buf[lexer.bufpos] notin spaceOrLineEnd:
-    lexerError(lexer, "Missing space after tag shorthand")
+    raise lexer.generateError("Missing space after tag shorthand")
 
-template tagUriMapping(lexer: BaseLexer, uri: var string) =
+proc tagUriMapping(lexer: var BaseLexer, uri: var string)
+    {.raises: [YamlParserError].} =
   debug("lex: tagUriMapping")
   while lexer.buf[lexer.bufpos] in space:
     lexer.bufpos.inc()
@@ -531,9 +538,10 @@ template tagUriMapping(lexer: BaseLexer, uri: var string) =
       uri.add(c)
       lexer.bufpos.inc()
       c = lexer.buf[lexer.bufpos]
-    else: lexerError(lexer, "Invalid tag uri")
+    else: raise lexer.generateError("Invalid tag uri")
 
-template directivesEndMarker(lexer: var BaseLexer, success: var bool) =
+proc directivesEndMarker(lexer: var BaseLexer, success: var bool)
+    {.raises: [].} =
   debug("lex: directivesEndMarker")
   success = true
   for i in 0..2:
@@ -542,7 +550,7 @@ template directivesEndMarker(lexer: var BaseLexer, success: var bool) =
       break
   if success: success = lexer.buf[lexer.bufpos + 3] in spaceOrLineEnd
 
-template documentEndMarker(lexer: var BaseLexer, success: var bool) =
+proc documentEndMarker(lexer: var BaseLexer, success: var bool) {.raises: [].} =
   debug("lex: documentEndMarker")
   success = true
   for i in 0..2:
@@ -562,18 +570,17 @@ proc unicodeSequence(lexer: var BaseLexer, length: int):
       c = lexer.buf[lexer.bufpos]
     case c
     of EndOFFile, '\l', '\c':
-        lexerError(lexer, "Unfinished unicode escape sequence")
+      raise lexer.generateError("Unfinished unicode escape sequence")
     of '0' .. '9':
-        unicodeChar = unicodechar or
-                (cast[int](c) - 0x30) shl (digitPosition * 4)
+      unicodeChar = unicodechar or (int(c) - 0x30) shl (digitPosition * 4)
     of 'A' .. 'F':
-        unicodeChar = unicodechar or
-                (cast[int](c) - 0x37) shl (digitPosition * 4)
+      unicodeChar = unicodechar or (int(c) - 0x37) shl (digitPosition * 4)
     of 'a' .. 'f':
-        unicodeChar = unicodechar or
-                (cast[int](c) - 0x57) shl (digitPosition * 4)
-    else: lexerError(lexer, "Invalid character in unicode escape sequence")
-  return toUTF8(cast[Rune](unicodeChar))
+      unicodeChar = unicodechar or (int(c) - 0x57) shl (digitPosition * 4)
+    else:
+      raise lexer.generateError(
+          "Invalid character in unicode escape sequence")
+  return toUTF8(Rune(unicodeChar))
 
 proc byteSequence(lexer: var BaseLexer): char {.raises: [YamlParserError].} =
   debug("lex: byteSequence")
@@ -585,17 +592,20 @@ proc byteSequence(lexer: var BaseLexer): char {.raises: [YamlParserError].} =
       c = lexer.buf[lexer.bufpos]
     case c
     of EndOfFile, '\l', 'r':
-      lexerError(lexer, "Unfinished octet escape sequence")
+      raise lexer.generateError("Unfinished octet escape sequence")
     of '0' .. '9':
       charCode = charCode or (int8(c) - 0x30.int8) shl (digitPosition * 4)
     of 'A' .. 'F':
       charCode = charCode or (int8(c) - 0x37.int8) shl (digitPosition * 4)
     of 'a' .. 'f':
       charCode = charCode or (int8(c) - 0x57.int8) shl (digitPosition * 4)
-    else: lexerError(lexer, "Invalid character in octet escape sequence")
+    else:
+      raise lexer.generateError("Invalid character in octet escape sequence")
   return char(charCode)
 
-template processQuotedWhitespace(newlines: var int) {.dirty.} =
+# TODO: {.raises: [].}
+proc processQuotedWhitespace(p: YamlParser, content, after: var string,
+                             newlines: var int) =
   after.setLen(0)
   block outer:
     while true:
@@ -629,19 +639,20 @@ template processQuotedWhitespace(newlines: var int) {.dirty.} =
         break
       p.lexer.bufpos.inc()
 
-template doubleQuotedScalar(lexer: BaseLexer, content: var string) =
+# TODO: {.raises: [YamlParserError].}
+proc doubleQuotedScalar(p: YamlParser, content, after: var string) =
   debug("lex: doubleQuotedScalar")
-  lexer.bufpos.inc()
+  p.lexer.bufpos.inc()
   while true:
-    var c = lexer.buf[lexer.bufpos]
+    var c = p.lexer.buf[p.lexer.bufpos]
     case c
     of EndOfFile:
-      lexerError(lexer, "Unfinished double quoted string")
+      raise p.lexer.generateError("Unfinished double quoted string")
     of '\\':
-      lexer.bufpos.inc()
-      case lexer.buf[lexer.bufpos]
+      p.lexer.bufpos.inc()
+      case p.lexer.buf[p.lexer.bufpos]
       of EndOfFile:
-        lexerError(lexer, "Unfinished escape sequence")
+        raise p.lexer.generateError("Unfinished escape sequence")
       of '0':       content.add('\0')
       of 'a':       content.add('\x07')
       of 'b':       content.add('\x08')
@@ -659,41 +670,42 @@ template doubleQuotedScalar(lexer: BaseLexer, content: var string) =
       of '_':       content.add(UTF8NonBreakingSpace)
       of 'L':       content.add(UTF8LineSeparator)
       of 'P':       content.add(UTF8ParagraphSeparator)
-      of 'x':       content.add(lexer.unicodeSequence(2))
-      of 'u':       content.add(lexer.unicodeSequence(4))
-      of 'U':       content.add(lexer.unicodeSequence(8))
+      of 'x':       content.add(p.lexer.unicodeSequence(2))
+      of 'u':       content.add(p.lexer.unicodeSequence(4))
+      of 'U':       content.add(p.lexer.unicodeSequence(8))
       of '\l', '\c':
         var newlines = 0
-        processQuotedWhitespace(newlines)
+        p.processQuotedWhitespace(content, after, newlines)
         continue
-      else: lexerError(lexer, "Illegal character in escape sequence")
+      else: raise p.lexer.generateError("Illegal character in escape sequence")
     of '"':
-      lexer.bufpos.inc()
+      p.lexer.bufpos.inc()
       break
     of '\l', '\c', '\t', ' ':
       var newlines = 1
-      processQuotedWhitespace(newlines)
+      p.processQuotedWhitespace(content, after, newlines)
       continue
     else:
       content.add(c)
-    lexer.bufpos.inc()
+    p.lexer.bufpos.inc()
 
-template singleQuotedScalar(lexer: BaseLexer, content: var string) =
+# TODO: {.raises: [].}
+proc singleQuotedScalar(p: YamlParser, content, after: var string) =
   debug("lex: singleQuotedScalar")
-  lexer.bufpos.inc()
+  p.lexer.bufpos.inc()
   while true:
-    case lexer.buf[lexer.bufpos]
+    case p.lexer.buf[p.lexer.bufpos]
     of '\'':
-      lexer.bufpos.inc()
-      if lexer.buf[lexer.bufpos] == '\'': content.add('\'')
+      p.lexer.bufpos.inc()
+      if p.lexer.buf[p.lexer.bufpos] == '\'': content.add('\'')
       else: break
-    of EndOfFile: lexerError(lexer, "Unfinished single quoted string")
+    of EndOfFile: raise p.lexer.generateError("Unfinished single quoted string")
     of '\l', '\c', '\t', ' ':
       var newlines = 1
-      processQuotedWhitespace(newlines)
+      p.processQuotedWhitespace(content, after, newlines)
       continue
-    else: content.add(lexer.buf[lexer.bufpos])
-    lexer.bufpos.inc()
+    else: content.add(p.lexer.buf[p.lexer.bufpos])
+    p.lexer.bufpos.inc()
 
 proc isPlainSafe(lexer: BaseLexer, index: int, context: YamlContext): bool =
   case lexer.buf[lexer.bufpos + 1]
@@ -712,8 +724,8 @@ proc isPlainSafe(lexer: BaseLexer, index: int, context: YamlContext): bool =
 #  when context == cBlock: c in plainCharOut
 #  else: c in plainCharIn
 
-template plainScalar(lexer: BaseLexer, content: var string,
-                     context: YamlContext) =
+proc plainScalar(lexer: var BaseLexer, content, after: var string,
+                 context: static[YamlContext]) {.raises: [].} =
   debug("lex: plainScalar")
   content.add(lexer.buf[lexer.bufpos])
   block outer:
@@ -758,14 +770,14 @@ template plainScalar(lexer: BaseLexer, content: var string,
 
 template continueMultilineScalar() {.dirty.} =
   content.add(if newlines == 1: " " else: repeat('\l', newlines - 1))
-  startToken()
-  p.lexer.plainScalar(content, cBlock)
+  p.startToken()
+  p.lexer.plainScalar(content, after, cBlock)
   state = fpBlockAfterPlainScalar
 
 template handleFlowPlainScalar() {.dirty.} =
-  startContent()
-  startToken()
-  p.lexer.plainScalar(content, cFlow)
+  content.reset()
+  p.startToken()
+  p.lexer.plainScalar(content, after, cFlow)
   if p.lexer.buf[p.lexer.bufpos] in {'{', '}', '[', ']', ',', ':', '#'}:
     discard
   else:
@@ -780,7 +792,7 @@ template handleFlowPlainScalar() {.dirty.} =
           elif newlines > 1:
             content.addMultiple(' ', newlines - 1)
             newlines = 0
-          p.lexer.plainScalar(content, cFlow)
+          p.lexer.plainScalar(content, after, cFlow)
         break
       of '#', EndOfFile: break
       of '\l':
@@ -798,12 +810,12 @@ template handleFlowPlainScalar() {.dirty.} =
         elif newlines > 1:
           content.addMultiple(' ', newlines - 1)
           newlines = 0
-        p.lexer.plainScalar(content, cFlow)
+        p.lexer.plainScalar(content, after, cFlow)
   yieldShallowScalar(content)
   handleObjectEnd(fpFlowAfterObject)
 
-template tagHandle(lexer: var BaseLexer, content: var string,
-                   shorthandEnd: var int) =
+proc tagHandle(lexer: var BaseLexer, content: var string,
+                shorthandEnd: var int) {.raises: [YamlParserError].} =
   debug("lex: tagHandle")
   shorthandEnd = 0
   content.add(lexer.buf[lexer.bufpos])
@@ -814,14 +826,14 @@ template tagHandle(lexer: var BaseLexer, content: var string,
     let c = lexer.buf[lexer.bufpos]
     case c
     of spaceOrLineEnd:
-      if shorthandEnd == -1: lexerError(lexer, "Unclosed verbatim tag")
+      if shorthandEnd == -1: raise lexer.generateError("Unclosed verbatim tag")
       break
     of '!':
       if shorthandEnd == -1 and i == 2:
         content.add(c)
         continue
       elif shorthandEnd != 0:
-        lexerError(lexer, "Illegal character in tag suffix")
+        raise lexer.generateError("Illegal character in tag suffix")
       shorthandEnd = i
       content.add(c)
     of 'a' .. 'z', 'A' .. 'Z', '0' .. '9', '#', ';', '/', '?', ':', '@', '&',
@@ -833,21 +845,21 @@ template tagHandle(lexer: var BaseLexer, content: var string,
     of '<':
       if i == 1:
         shorthandEnd = -1
-        startContent()
-      else: lexerError(lexer, "Illegal character in tag handle")
+        content.reset()
+      else: raise lexer.generateError("Illegal character in tag handle")
     of '>':
       if shorthandEnd == -1:
         lexer.bufpos.inc()
         if lexer.buf[lexer.bufpos] notin spaceOrLineEnd:
-          lexerError(lexer, "Missing space after verbatim tag handle")
+          raise lexer.generateError("Missing space after verbatim tag handle")
         break
-      else: lexerError(lexer, "Illegal character in tag handle")
+      else: raise lexer.generateError("Illegal character in tag handle")
     of '%':
       if shorthandEnd != 0: content.add(lexer.byteSequence())
-      else: lexerError(lexer, "Illegal character in tag handle")
-    else: lexerError(lexer, "Illegal character in tag handle")
+      else: raise lexer.generateError("Illegal character in tag handle")
+    else: raise lexer.generateError("Illegal character in tag handle")
 
-template anchorName(lexer: BaseLexer, content: var string) =
+proc anchorName(lexer: var BaseLexer, content: var string) {.raises: [].} =
   debug("lex: anchorName")
   while true:
     lexer.bufpos.inc()
@@ -856,26 +868,25 @@ template anchorName(lexer: BaseLexer, content: var string) =
     of spaceOrLineEnd, '[', ']', '{', '}', ',': break
     else: content.add(c)
 
-template consumeLineIfEmpty(lex: BaseLexer): bool =
-  var result = true
+proc consumeLineIfEmpty(p: YamlParser, newlines: var int): bool =
+  result = true
   while true:
-    lex.bufpos.inc()
-    case lex.buf[lex.bufpos]
+    p.lexer.bufpos.inc()
+    case p.lexer.buf[p.lexer.bufpos]
     of ' ', '\t': discard
     of '\l':
-      lex.bufpos = lex.handleLF(lex.bufpos)
+      p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos)
       break
     of '\c':
-      lex.bufpos = lex.handleCR(lex.bufpos)
+      p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos)
       break
     of '#', EndOfFile:
-      lex.lineEnding()
+      p.lineEnding()
       handleLineEnd(false)
       break
     else:
       result = false
       break
-  result
 
 template startScalar(t: ScalarType) {.dirty.} =
   newlines = 0
@@ -893,25 +904,27 @@ template blockScalarHeader() {.dirty.} =
     case p.lexer.buf[p.lexer.bufpos]
     of '+':
       if chomp != ctClip:
-        lexerError(p.lexer, "Only one chomping indicator is allowed")
+        raise p.lexer.generateError("Only one chomping indicator is allowed")
       chomp = ctKeep
     of '-':
       if chomp != ctClip:
-        lexerError(p.lexer, "Only one chomping indicator is allowed")
+        raise p.lexer.generateError("Only one chomping indicator is allowed")
       chomp = ctStrip
     of '1'..'9':
       if level.indentation != UnknownIndentation:
-        lexerError(p.lexer, "Only one indentation indicator is allowed")
+        raise p.lexer.generateError("Only one indentation indicator is allowed")
       level.indentation = ancestry[ancestry.high].indentation +
           ord(p.lexer.buf[p.lexer.bufpos]) - ord('\x30')
     of spaceOrLineEnd: break
-    else: lexerError(p.lexer, "Illegal character in block scalar header: '" &
-                     p.lexer.buf[p.lexer.bufpos] & "'")
+    else:
+      raise p.lexer.generateError(
+          "Illegal character in block scalar header: '" &
+          p.lexer.buf[p.lexer.bufpos] & "'")
   recentWasMoreIndented = false
-  p.lexer.lineEnding()
+  p.lineEnding()
   handleLineEnd(true)
   startScalar(t)
-  startContent()
+  content.reset()
 
 template blockScalarLine() {.dirty.} =
   debug("lex: blockScalarLine")
@@ -919,16 +932,17 @@ template blockScalarLine() {.dirty.} =
     if p.lexer.buf[p.lexer.bufpos] == '#':
       # skip all following comment lines
       while indentation > ancestry[ancestry.high].indentation:
-        p.lexer.lineEnding()
+        p.lineEnding()
         handleLineEnd(true)
         newlines.inc(-1)
         p.lexer.skipIndentation()
         indentation = p.lexer.getColNumber(p.lexer.bufpos)
       if indentation > ancestry[ancestry.high].indentation:
-        p.lexer.lexerError("Invalid content in block scalar after comments")
+        raise p.lexer.generateError(
+            "Invalid content in block scalar after comments")
       closeMoreIndentedLevels()
     else:
-      p.lexer.lexerError(
+      raise p.lexer.generateError(
           "Invalid indentation (expected indentation of at least " &
           $level.indentation & " spaces)")    
   else:
@@ -989,12 +1003,12 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
         case p.lexer.buf[p.lexer.bufpos]
         of '%':
           var ld: LexedDirective
-          startToken()
+          p.startToken()
           p.lexer.directiveName(ld)
           case ld
           of ldYaml:
             var version = ""
-            startToken()
+            p.startToken()
             p.lexer.yamlVersion(version)
             if version != "1.2":
               if p.callback != nil:
@@ -1002,16 +1016,16 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
                              p.getLineContent(),
                              "Version is not 1.2, but " & version)
               discard
-            p.lexer.lineEnding()
+            p.lineEnding()
             handleLineEnd(false)
           of ldTag:
             var shorthand = ""
             tagUri.setLen(0)
-            startToken()
+            p.startToken()
             p.lexer.tagShorthand(shorthand)
             p.lexer.tagUriMapping(tagUri)
             shorthands[shorthand] = tagUri
-            p.lexer.lineEnding()
+            p.lineEnding()
             handleLineEnd(false)
           of ldUnknown:
             if p.callback != nil:
@@ -1020,7 +1034,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             p.lexer.finishLine()
             handleLineEnd(false)
         of ' ', '\t':
-          if not p.lexer.consumeLineIfEmpty():
+          if not p.consumeLineIfEmpty(newlines):
             indentation = p.lexer.getColNumber(p.lexer.bufpos)
             yield startDocEvent()
             state = fpBlockObjectStart
@@ -1028,11 +1042,11 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
         of '\c': p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos)
         of EndOfFile: return
         of '#':
-          p.lexer.lineEnding()
+          p.lineEnding()
           handleLineEnd(false)
         of '-':
           var success: bool
-          startToken()
+          p.startToken()
           p.lexer.directivesEndMarker(success)
           yield startDocEvent()
           if success:
@@ -1047,16 +1061,16 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
         case p.lexer.buf[p.lexer.bufpos]
         of '\l':
           if level.kind notin {fplUnknown, fplScalar}:
-            startToken()
-            parserError("Unexpected scalar")
+            p.startToken()
+            raise p.generateError("Unexpected scalar")
           startScalar(stFlow)
           p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos)
           newlines.inc()
           state = fpBlockObjectStart
         of '\c':
           if level.kind notin {fplUnknown, fplScalar}:
-            startToken()
-            parserError("Unexpected scalar")
+            p.startToken()
+            raise p.generateError("Unexpected scalar")
           startScalar(stFlow)
           p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos)
           newlines.inc()
@@ -1091,23 +1105,25 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             ancestry.add(level)
             level = initLevel(fplUnknown)
           of fplSequence:
-            startToken()
-            parserError("Illegal token (expected sequence item)")
+            p.startToken()
+            raise p.generateError("Illegal token (expected sequence item)")
           of fplScalar:
-            startToken()
-            parserError("Multiline scalars may not be implicit map keys")
+            p.startToken()
+            raise p.generateError(
+                "Multiline scalars may not be implicit map keys")
           of fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail()
           p.lexer.bufpos.inc()
           p.lexer.skipWhitespace()
           indentation = p.lexer.getColNumber(p.lexer.bufpos)
           state = fpBlockObjectStart
         of '#':
-          p.lexer.lineEnding()
+          p.lineEnding()
           handleLineEnd(true)
           state = fpBlockObjectStart
         else:
-          startToken()
-          parserError("Illegal token (expected ':', comment or line end)")
+          p.startToken()
+          raise p.generateError(
+              "Illegal token (expected ':', comment or line end)")
       of fpBlockObjectStart:
         debug("state: blockObjectStart")
         p.lexer.skipIndentation()
@@ -1125,7 +1141,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           if success:
             closeEverything()
             p.lexer.bufpos.inc(3)
-            p.lexer.lineEnding()
+            p.lineEnding()
             handleLineEnd(false)
             state = fpAfterDocument
             continue
@@ -1134,7 +1150,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             handleLineEnd(true)
             continue
           elif p.lexer.buf[p.lexer.bufpos] == '#':
-            p.lexer.lineEnding()
+            p.lineEnding()
             handleLineEnd(true)
             continue
           elif p.lexer.buf[p.lexer.bufpos] == '-' and not
@@ -1161,22 +1177,22 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           closeEverything()
           return
         of '#':
-          p.lexer.lineEnding()
+          p.lineEnding()
           handleLineEnd(true)
           if level.kind == fplUnknown: level.indentation = UnknownIndentation
         of '\'':
           handleBlockItemStart()
-          startContent()
-          startToken()
-          p.lexer.singleQuotedScalar(content)
+          content.reset()
+          p.startToken()
+          p.singleQuotedScalar(content, after)
           if tag == yTagQuestionMark: tag = yTagExclamationMark
           yieldShallowScalar(content)
           handleObjectEnd(fpBlockAfterObject)
         of '"':
           handleBlockItemStart()
-          startContent()
-          startToken()
-          p.lexer.doubleQuotedScalar(content)
+          content.reset()
+          p.startToken()
+          p.doubleQuotedScalar(content, after)
           if tag == yTagQuestionMark: tag = yTagExclamationMark
           yieldShallowScalar(content)
           handleObjectEnd(fpBlockAfterObject)
@@ -1188,9 +1204,9 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             if level.kind == fplScalar: continueMultilineScalar()
             else:
               handleBlockItemStart()
-              startContent()
-              startToken()
-              p.lexer.plainScalar(content, cBlock)
+              content.reset()
+              p.startToken()
+              p.lexer.plainScalar(content, after, cBlock)
               state = fpBlockAfterPlainScalar
           else:
             p.lexer.bufpos.inc()
@@ -1212,9 +1228,9 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             if level.kind == fplScalar: continueMultilineScalar()
             else:
               handleBlockItemStart()
-              startContent()
-              startToken()
-              p.lexer.plainScalar(content, cBlock)
+              content.reset()
+              p.startToken()
+              p.lexer.plainScalar(content, after, cBlock)
               state = fpBlockAfterPlainScalar
           else:
             p.lexer.bufpos.inc()
@@ -1224,27 +1240,28 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             if level.kind == fplScalar: continueMultilineScalar()
             else:
               handleBlockItemStart()
-              startContent()
-              startToken()
-              p.lexer.plainScalar(content, cBlock)
+              content.reset()
+              p.startToken()
+              p.lexer.plainScalar(content, after, cBlock)
               state = fpBlockAfterPlainScalar
           else:
             p.lexer.bufpos.inc()
             handleMapValueIndicator()
         of '@', '`':
-          lexerError(p.lexer, "Reserved characters cannot start a plain scalar")
+          raise p.lexer.generateError(
+              "Reserved characters cannot start a plain scalar")
         of '\t':
           if level.kind == fplScalar:
             p.lexer.skipWhitespace()
             continueMultilineScalar()
-          else: lexerError(p.lexer, "\\t cannot start any token")
+          else: raise p.lexer.generateError("\\t cannot start any token")
         else:
           if level.kind == fplScalar: continueMultilineScalar()
           else:
             handleBlockItemStart()
-            startContent()
-            startToken()
-            p.lexer.plainScalar(content, cBlock)
+            content.reset()
+            p.startToken()
+            p.lexer.plainScalar(content, after, cBlock)
             state = fpBlockAfterPlainScalar
       of fpExpectDocEnd:
         debug("state: expectDocEnd")
@@ -1259,20 +1276,22 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             initDocValues()
             yield startDocEvent()
             state = fpBlockObjectStart
-          else: parserError("Unexpected content (expected document end)")
+          else:
+            raise p.generateError("Unexpected content (expected document end)")
         of '.':
           var isDocumentEnd: bool
-          startToken()
+          p.startToken()
           p.lexer.documentEndMarker(isDocumentEnd)
           if isDocumentEnd:
             closeEverything()
             p.lexer.bufpos.inc(3)
-            p.lexer.lineEnding()
+            p.lineEnding()
             handleLineEnd(false)
             state = fpAfterDocument
-          else: parserError("Unexpected content (expected document end)")
+          else:
+            raise p.generateError("Unexpected content (expected document end)")
         of ' ', '\t', '#':
-          p.lexer.lineEnding()
+          p.lineEnding()
           handleLineEnd(true)
         of '\l': p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos)
         of '\c': p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos)
@@ -1280,28 +1299,28 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           yield endDocEvent()
           break
         else:
-          startToken()
-          parserError("Unexpected content (expected document end)")
+          p.startToken()
+          raise p.generateError("Unexpected content (expected document end)")
       of fpAfterDocument:
         debug("state: afterDocument")
         case p.lexer.buf[p.lexer.bufpos]
         of '.':
           var isDocumentEnd: bool
-          startToken()
+          p.startToken()
           p.lexer.documentEndMarker(isDocumentEnd)
           if isDocumentEnd:
             p.lexer.bufpos.inc(3)
-            p.lexer.lineEnding()
+            p.lineEnding()
             handleLineEnd(false)
           else:
             initDocValues()
             yield startDocEvent()
             state = fpBlockObjectStart
         of '#':
-          p.lexer.lineEnding()
+          p.lineEnding()
           handleLineEnd(false)
         of '\t', ' ':
-          if not p.lexer.consumeLineIfEmpty():
+          if not p.consumeLineIfEmpty(newlines):
             indentation = p.lexer.getColNumber(p.lexer.bufpos)
             initDocValues()
             yield startDocEvent()
@@ -1338,11 +1357,11 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
               yieldEmptyScalar()
               yield scalarEvent("", yTagQuestionMark, yAnchorNone)
           of fplSequence:
-            startToken()
-            parserError("Unexpected token (expected ']')")
+            p.startToken()
+            raise p.generateError("Unexpected token (expected ']')")
           of fplSinglePairValue:
-            startToken()
-            parserError("Unexpected token (expected ']')")
+            p.startToken()
+            raise p.generateError("Unexpected token (expected ']')")
           of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail()
           p.lexer.bufpos.inc()
           leaveFlowLevel()
@@ -1359,8 +1378,8 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             yield endMapEvent()
             assert(level.kind == fplSequence)
           of fplMapKey, fplMapValue:
-            startToken()
-            parserError("Unexpected token (expected '}')")
+            p.startToken()
+            raise p.generateError("Unexpected token (expected '}')")
           of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail()
           p.lexer.bufpos.inc()
           leaveFlowLevel()
@@ -1406,8 +1425,8 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
               level = initLevel(fplSinglePairValue)
               yield scalarEvent("")
             of fplMapValue, fplSinglePairValue:
-              startToken()
-              parserError("Unexpected token (expected ',')")
+              p.startToken()
+              raise p.generateError("Unexpected token (expected ',')")
             of fplMapKey:
               yieldEmptyScalar()
               level.kind = fplMapValue
@@ -1420,17 +1439,17 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             p.lexer.bufpos.inc()
         of '\'':
           handleFlowItemStart()
-          startContent()
-          startToken()
-          p.lexer.singleQuotedScalar(content)
+          content.reset()
+          p.startToken()
+          p.singleQuotedScalar(content, after)
           if tag == yTagQuestionMark: tag = yTagExclamationMark
           yieldShallowScalar(content)
           handleObjectEnd(fpFlowAfterObject)
         of '"':
           handleFlowItemStart()
-          startContent()
-          startToken()
-          p.lexer.doubleQuotedScalar(content)
+          content.reset()
+          p.startToken()
+          p.doubleQuotedScalar(content, after)
           if tag == yTagQuestionMark: tag = yTagExclamationMark
           yieldShallowScalar(content)
           handleObjectEnd(fpFlowAfterObject)
@@ -1448,15 +1467,15 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             handleFlowItemStart()
             handleFlowPlainScalar()
           elif explicitFlowKey:
-            startToken()
-            parserError("Duplicate '?' in flow mapping")
+            p.startToken()
+            raise p.generateError("Duplicate '?' in flow mapping")
           elif level.kind == fplUnknown:
             case ancestry[ancestry.high].kind
             of fplMapKey, fplMapValue, fplDocument: discard
             of fplSequence: handleObjectStart(yamlStartMap, true)
             else:
-              startToken()
-              parserError("Unexpected token")
+              p.startToken()
+              raise p.generateError("Unexpected token")
             explicitFlowKey = true
             p.lexer.bufpos.inc()
           else:
@@ -1473,8 +1492,8 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           case level.kind
           of fplSequence: discard
           of fplMapKey, fplMapValue:
-            startToken()
-            parserError("Unexpected token (expected '}')")
+            p.startToken()
+            raise p.generateError("Unexpected token (expected '}')")
           of fplSinglePairValue:
             level = ancestry.pop()
             assert(level.kind == fplSequence)
@@ -1486,8 +1505,8 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           case level.kind
           of fplMapKey, fplMapValue: discard
           of fplSequence, fplSinglePairValue:
-            startToken()
-            parserError("Unexpected token (expected ']')")
+            p.startToken()
+            raise p.generateError("Unexpected token (expected ']')")
           of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail()
           p.lexer.bufpos.inc()
           leaveFlowLevel()
@@ -1511,8 +1530,8 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
         of ':':
           case level.kind
           of fplSequence, fplMapKey:
-            startToken()
-            parserError("Unexpected token (expected ',')")
+            p.startToken()
+            raise p.generateError("Unexpected token (expected ',')")
           of fplMapValue, fplSinglePairValue: discard
           of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail()
           ancestry.add(level)
@@ -1520,13 +1539,13 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           state = fpFlow
           p.lexer.bufpos.inc()
         of '#':
-          p.lexer.lineEnding()
+          p.lineEnding()
           handleLineEnd(true)
         of EndOfFile:
-          startToken()
-          parserError("Unclosed flow content")
+          p.startToken()
+          raise p.generateError("Unclosed flow content")
         else:
-          startToken()
-          parserError("Unexpected content (expected flow indicator)")
+          p.startToken()
+          raise p.generateError("Unexpected content (expected flow indicator)")
   try: result = initYamlStream(backend)
   except Exception: debugFail() # compiler error
\ No newline at end of file

From 78fac63af5e3cc254f3f0658b7b9bbbee6998a71 Mon Sep 17 00:00:00 2001
From: Felix Krause <contact@flyx.org>
Date: Sat, 21 May 2016 23:26:39 +0200
Subject: [PATCH 3/4] fastparse refactoring, part II

 * Transformed quite some templates to procs
 * Moved quite some local vars from parse() to YamlParser
---
 private/fastparse.nim | 795 +++++++++++++++++++++---------------------
 yaml.nim              |  19 +-
 2 files changed, 417 insertions(+), 397 deletions(-)

diff --git a/private/fastparse.nim b/private/fastparse.nim
index d7b9b4f..765bce2 100644
--- a/private/fastparse.nim
+++ b/private/fastparse.nim
@@ -9,17 +9,9 @@ type
     fpInitial, fpBlockAfterObject, fpBlockAfterPlainScalar, fpBlockObjectStart,
     fpExpectDocEnd, fpFlow, fpFlowAfterObject, fpAfterDocument
   
-  FastParseLevelKind = enum
-    fplUnknown, fplSequence, fplMapKey, fplMapValue, fplSinglePairKey,
-    fplSinglePairValue, fplScalar, fplDocument
-  
   ScalarType = enum
     stFlow, stLiteral, stFolded
   
-  FastParseLevel = object
-    kind: FastParseLevelKind
-    indentation: int
-  
   LexedDirective = enum
     ldYaml, ldTag, ldUnknown
     
@@ -44,18 +36,21 @@ const
     
 proc newYamlParser*(tagLib: TagLibrary = initExtendedTagLibrary(),
                     callback: WarningCallback = nil): YamlParser =
-    new(result)
-    result.tagLib = tagLib
-    result.callback = callback
+  new(result)
+  result.tagLib = tagLib
+  result.callback = callback
+  result.content = ""
+  result.after = ""
+  result.tagUri = ""
+  result.ancestry = newSeq[FastParseLevel]()
 
 proc getLineNumber*(p: YamlParser): int = p.lexer.lineNumber
     
 proc getColNumber*(p: YamlParser): int = p.tokenstart + 1 # column is 1-based
 
 proc getLineContent*(p: YamlParser, marker: bool = true): string =
-    result = p.lexer.getCurrentLine(false)
-    if marker:
-        result.add(repeat(' ', p.tokenstart) & "^\n")
+  result = p.lexer.getCurrentLine(false)
+  if marker: result.add(repeat(' ', p.tokenstart) & "^\n")
 
 template debug(message: string) {.dirty.} =
   when defined(yamlDebug):
@@ -86,24 +81,24 @@ proc addMultiple(s: var string, c: char, num: int) {.raises: [], inline.} =
   for i in 1..num:
     s.add(c)
 
-proc reset(content: var string) {.raises: [], inline.} = content.setLen(0)
+proc reset(buffer: var string) {.raises: [], inline.} = buffer.setLen(0)
 
 proc initLevel(k: FastParseLevelKind): FastParseLevel {.raises: [], inline.} =
   FastParseLevel(kind: k, indentation: UnknownIndentation)
 
 template yieldEmptyScalar() {.dirty.} =
-  yield scalarEvent("", tag, anchor)
-  tag = yTagQuestionMark
-  anchor = yAnchorNone
+  yield scalarEvent("", p.tag, p.anchor)
+  p.tag = yTagQuestionMark
+  p.anchor = yAnchorNone
 
 template yieldShallowScalar(content: string) {.dirty.} =
-  var e = YamlStreamEvent(kind: yamlScalar, scalarTag: tag,
-                          scalarAnchor: anchor)
+  var e = YamlStreamEvent(kind: yamlScalar, scalarTag: p.tag,
+                          scalarAnchor: p.anchor)
   shallowCopy(e.scalarContent, content)
   yield e
 
 template yieldLevelEnd() {.dirty.} =
-  case level.kind
+  case p.level.kind
   of fplSequence: yield endSeqEvent()
   of fplMapKey: yield endMapEvent()
   of fplMapValue, fplSinglePairValue:
@@ -113,16 +108,16 @@ template yieldLevelEnd() {.dirty.} =
     if scalarType != stFlow:
       case chomp
       of ctKeep:
-        if content.len == 0: newlines.inc(-1)
-        content.addMultiple('\l', newlines)
+        if p.content.len == 0: p.newlines.inc(-1)
+        p.content.addMultiple('\l', p.newlines)
       of ctClip:
-        if content.len != 0: content.add('\l')
+        if p.content.len != 0: p.content.add('\l')
       of ctStrip: discard
-    yieldShallowScalar(content)
-    tag = yTagQuestionMark
-    anchor = yAnchorNone
+    yieldShallowScalar(p.content)
+    p.tag = yTagQuestionMark
+    p.anchor = yAnchorNone
   of fplUnknown:
-    if ancestry.len > 1: yieldEmptyScalar() # don't yield scalar for empty doc
+    if p.ancestry.len > 1: yieldEmptyScalar() # don't yield scalar for empty doc
   of fplSinglePairKey, fplDocument: debugFail()
 
 template handleLineEnd(insideDocument: bool) {.dirty.} =
@@ -133,57 +128,59 @@ template handleLineEnd(insideDocument: bool) {.dirty.} =
     when insideDocument: closeEverything()
     return
   else: discard
-  newlines.inc()
+  p.newlines.inc()
 
 template handleObjectEnd(nextState: FastParseState) {.dirty.} =
-  level = ancestry.pop()
-  if level.kind == fplSinglePairValue:
+  p.level = p.ancestry.pop()
+  if p.level.kind == fplSinglePairValue:
     yield endMapEvent()
-    level = ancestry.pop()
-  state = if level.kind == fplDocument: fpExpectDocEnd else: nextState
-  tag = yTagQuestionMark
-  anchor = yAnchorNone
-  case level.kind
-  of fplMapKey: level.kind = fplMapValue
-  of fplSinglePairKey: level.kind = fplSinglePairValue
-  of fplMapValue: level.kind = fplMapKey
+    p.level = p.ancestry.pop()
+  state = if p.level.kind == fplDocument: fpExpectDocEnd else: nextState
+  p.tag = yTagQuestionMark
+  p.anchor = yAnchorNone
+  case p.level.kind
+  of fplMapKey: p.level.kind = fplMapValue
+  of fplSinglePairKey: p.level.kind = fplSinglePairValue
+  of fplMapValue: p.level.kind = fplMapKey
   of fplSequence, fplDocument: discard
   of fplUnknown, fplScalar, fplSinglePairValue: debugFail()
 
 template handleObjectStart(k: YamlStreamEventKind, single: bool = false)
     {.dirty.} =
-  assert(level.kind == fplUnknown)
+  assert(p.level.kind == fplUnknown)
   when k == yamlStartMap:
-    yield startMapEvent(tag, anchor)
+    yield startMapEvent(p.tag, p.anchor)
     if single:
       debug("started single-pair map at " &
-          (if level.indentation == UnknownIndentation: $indentation else:
-           $level.indentation))
-      level.kind = fplSinglePairKey
+          (if p.level.indentation == UnknownIndentation: $indentation else:
+           $p.level.indentation))
+      p.level.kind = fplSinglePairKey
     else:
       debug("started map at " &
-          (if level.indentation == UnknownIndentation: $indentation else:
-           $level.indentation))
-      level.kind = fplMapKey
+          (if p.level.indentation == UnknownIndentation: $indentation else:
+           $p.level.indentation))
+      p.level.kind = fplMapKey
   else:
-    yield startSeqEvent(tag, anchor)
+    yield startSeqEvent(p.tag, p.anchor)
     debug("started sequence at " &
-        (if level.indentation == UnknownIndentation: $indentation else:
-         $level.indentation))
-    level.kind = fplSequence
-  tag = yTagQuestionMark
-  anchor = yAnchorNone
-  if level.indentation == UnknownIndentation: level.indentation = indentation
-  ancestry.add(level)
-  level = initLevel(fplUnknown)
+        (if p.level.indentation == UnknownIndentation: $indentation else:
+         $p.level.indentation))
+    p.level.kind = fplSequence
+  p.tag = yTagQuestionMark
+  p.anchor = yAnchorNone
+  if p.level.indentation == UnknownIndentation:
+    p.level.indentation = indentation
+  p.ancestry.add(p.level)
+  p.level = initLevel(fplUnknown)
   
 template closeMoreIndentedLevels(atSequenceItem: bool = false) {.dirty.} =
-  while level.kind != fplDocument:
-    let parent = ancestry[ancestry.high]
+  while p.level.kind != fplDocument:
+    let parent = p.ancestry[p.ancestry.high]
     if parent.indentation >= indentation:
       when atSequenceItem:
-        if (indentation == level.indentation and level.kind == fplSequence) or
-           (indentation == parent.indentation and level.kind == fplUnknown and
+        if (indentation == p.level.indentation and
+            p.level.kind == fplSequence) or
+           (indentation == parent.indentation and p.level.kind == fplUnknown and
             parent.kind != fplSequence):
           break
       debug("Closing because parent.indentation (" & $parent.indentation &
@@ -191,48 +188,48 @@ template closeMoreIndentedLevels(atSequenceItem: bool = false) {.dirty.} =
       yieldLevelEnd()
       handleObjectEnd(state)
     else: break
-  if level.kind == fplDocument: state = fpExpectDocEnd
+  if p.level.kind == fplDocument: state = fpExpectDocEnd
 
 template closeEverything() {.dirty.} =
   indentation = 0
   closeMoreIndentedLevels()
-  case level.kind
-  of fplUnknown: discard ancestry.pop()
+  case p.level.kind
+  of fplUnknown: discard p.ancestry.pop()
   of fplDocument: discard
   else:
     yieldLevelEnd()
-    discard ancestry.pop()
+    discard p.ancestry.pop()
   yield endDocEvent()
 
 template handleBlockSequenceIndicator() {.dirty.} =
   p.startToken()
-  case level.kind
+  case p.level.kind
   of fplUnknown: handleObjectStart(yamlStartSeq)
   of fplSequence:
-    if level.indentation != indentation:
+    if p.level.indentation != indentation:
       raise p.generateError("Invalid indentation of block sequence indicator")
-    ancestry.add(level)
-    level = initLevel(fplUnknown)
+    p.ancestry.add(p.level)
+    p.level = initLevel(fplUnknown)
   else: raise p.generateError("Illegal sequence item in map")
   p.lexer.skipWhitespace()
   indentation = p.lexer.getColNumber(p.lexer.bufpos)
 
 template handleMapKeyIndicator() {.dirty.} =
   p.startToken()
-  case level.kind
+  case p.level.kind
   of fplUnknown: handleObjectStart(yamlStartMap)
   of fplMapValue:
-    if level.indentation != indentation:
+    if p.level.indentation != indentation:
       raise p.generateError("Invalid indentation of map key indicator")
     yield scalarEvent("", yTagQuestionMark, yAnchorNone)
-    level.kind = fplMapKey
-    ancestry.add(level)
-    level = initLevel(fplUnknown)
+    p.level.kind = fplMapKey
+    p.ancestry.add(p.level)
+    p.level = initLevel(fplUnknown)
   of fplMapKey:
-    if level.indentation != indentation:
+    if p.level.indentation != indentation:
       raise p.generateError("Invalid indentation of map key indicator")
-    ancestry.add(level)
-    level = initLevel(fplUnknown)
+    p.ancestry.add(p.level)
+    p.level = initLevel(fplUnknown)
   of fplSequence:
     raise p.generateError("Unexpected map key indicator (expected '- ')")
   of fplScalar:
@@ -244,25 +241,25 @@ template handleMapKeyIndicator() {.dirty.} =
 
 template handleMapValueIndicator() {.dirty.} =
   p.startToken()
-  case level.kind
+  case p.level.kind
   of fplUnknown:
-    if level.indentation == UnknownIndentation:
+    if p.level.indentation == UnknownIndentation:
       handleObjectStart(yamlStartMap)
       yield scalarEvent("", yTagQuestionMark, yAnchorNone)
     else: yieldEmptyScalar()
-    ancestry[ancestry.high].kind = fplMapValue
+    p.ancestry[p.ancestry.high].kind = fplMapValue
   of fplMapKey:
-    if level.indentation != indentation:
+    if p.level.indentation != indentation:
       raise p.generateError("Invalid indentation of map key indicator")
     yield scalarEvent("", yTagQuestionMark, yAnchorNone)
-    level.kind = fplMapValue
-    ancestry.add(level)
-    level = initLevel(fplUnknown)
+    p.level.kind = fplMapValue
+    p.ancestry.add(p.level)
+    p.level = initLevel(fplUnknown)
   of fplMapValue:
-    if level.indentation != indentation:
+    if p.level.indentation != indentation:
       raise p.generateError("Invalid indentation of map key indicator")
-    ancestry.add(level)
-    level = initLevel(fplUnknown)
+    p.ancestry.add(p.level)
+    p.level = initLevel(fplUnknown)
   of fplSequence:
     raise p.generateError("Unexpected map value indicator (expected '- ')")
   of fplScalar:
@@ -273,59 +270,48 @@ template handleMapValueIndicator() {.dirty.} =
   indentation = p.lexer.getColNumber(p.lexer.bufpos)
 
 template initDocValues() {.dirty.} =
-  shorthands = initTable[string, string]()
-  anchors = initTable[string, AnchorId]()
-  shorthands["!"] = "!"
-  shorthands["!!"] = "tag:yaml.org,2002:"
-  nextAnchorId = 0.AnchorId
-  level = initLevel(fplUnknown)
-  tag = yTagQuestionMark
-  anchor = yAnchorNone
-  ancestry.add(FastParseLevel(kind: fplDocument, indentation: -1))
+  p.shorthands = initTable[string, string]()
+  p.anchors = initTable[string, AnchorId]()
+  p.shorthands["!"] = "!"
+  p.shorthands["!!"] = "tag:yaml.org,2002:"
+  p.nextAnchorId = 0.AnchorId
+  p.level = initLevel(fplUnknown)
+  p.tag = yTagQuestionMark
+  p.anchor = yAnchorNone
+  p.ancestry.add(FastParseLevel(kind: fplDocument, indentation: -1))
 
-template handleTagHandle() {.dirty.} =
-  p.startToken()
-  if level.kind != fplUnknown: raise p.generateError("Unexpected tag handle")
-  if tag != yTagQuestionMark:
-    raise p.generateError("Only one tag handle is allowed per node")
-  content.reset()
-  var
-    shorthandEnd: int
-  p.lexer.tagHandle(content, shorthandEnd)
-  if shorthandEnd != -1:
-    try:
-      tagUri.setLen(0)
-      tagUri.add(shorthands[content[0..shorthandEnd]])
-      tagUri.add(content[shorthandEnd + 1 .. ^1])
-    except KeyError:
-      raise p.generateError(
-          "Undefined tag shorthand: " & content[0..shorthandEnd])
-    try: tag = p.tagLib.tags[tagUri]
-    except KeyError: tag = p.tagLib.registerUri(tagUri)
-  else:
-    try: tag = p.tagLib.tags[content]
-    except KeyError: tag = p.tagLib.registerUri(content)
+proc startToken(p: YamlParser) {.raises: [], inline.} =
+  p.tokenstart = p.lexer.getColNumber(p.lexer.bufpos)
 
-template handleAnchor() {.dirty.} =
+proc anchorName(p: YamlParser) {.raises: [].} =
+  debug("lex: anchorName")
+  while true:
+    p.lexer.bufpos.inc()
+    let c = p.lexer.buf[p.lexer.bufpos]
+    case c
+    of spaceOrLineEnd, '[', ']', '{', '}', ',': break
+    else: p.content.add(c)
+
+proc handleAnchor(p: YamlParser) {.raises: [YamlParserError].} =
   p.startToken()
-  if level.kind != fplUnknown: raise p.generateError("Unexpected token")
-  if anchor != yAnchorNone:
+  if p.level.kind != fplUnknown: raise p.generateError("Unexpected token")
+  if p.anchor != yAnchorNone:
     raise p.generateError("Only one anchor is allowed per node")
-  content.reset()
-  p.lexer.anchorName(content)
-  anchor = nextAnchorId
-  anchors[content] = anchor
-  nextAnchorId = AnchorId(int(nextAnchorId) + 1)
+  p.content.reset()
+  p.anchorName()
+  p.anchor = p.nextAnchorId
+  p.anchors[p.content] = p.anchor
+  p.nextAnchorId = AnchorId(int(p.nextAnchorId) + 1)
 
 template handleAlias() {.dirty.} =
   p.startToken()
-  if level.kind != fplUnknown: raise p.generateError("Unexpected token")
-  if anchor != yAnchorNone or tag != yTagQuestionMark:
+  if p.level.kind != fplUnknown: raise p.generateError("Unexpected token")
+  if p.anchor != yAnchorNone or p.tag != yTagQuestionMark:
     raise p.generateError("Alias may not have anchor or tag")
-  content.reset()
-  p.lexer.anchorName(content)
+  p.content.reset()
+  p.anchorName()
   var id: AnchorId
-  try: id = anchors[content]
+  try: id = p.anchors[p.content]
   except KeyError: raise p.generateError("Unknown anchor")
   yield aliasEvent(id)
   handleObjectEnd(fpBlockAfterObject)
@@ -341,7 +327,7 @@ template leaveFlowLevel() {.dirty.} =
   
 template handlePossibleMapStart(flow: bool = false,
                                 single: bool = false) {.dirty.} =
-  if level.indentation == UnknownIndentation:
+  if p.level.indentation == UnknownIndentation:
     var flowDepth = 0
     var pos = p.lexer.bufpos
     var recentJsonStyle = false
@@ -381,31 +367,30 @@ template handlePossibleMapStart(flow: bool = false,
       if flow and p.lexer.buf[pos] notin space:
         recentJsonStyle = p.lexer.buf[pos] in {']', '}', '\'', '"'}
       pos.inc()
-    if level.indentation == UnknownIndentation: level.indentation = indentation
+    if p.level.indentation == UnknownIndentation:
+      p.level.indentation = indentation
 
 template handleBlockItemStart() {.dirty.} =
-  case level.kind
+  case p.level.kind
   of fplUnknown: handlePossibleMapStart()
   of fplSequence:
     raise p.generateError(
         "Unexpected token (expected block sequence indicator)")
   of fplMapKey:
-    ancestry.add(level)
-    level = FastParseLevel(kind: fplUnknown, indentation: indentation)
+    p.ancestry.add(p.level)
+    p.level = FastParseLevel(kind: fplUnknown, indentation: indentation)
   of fplMapValue:
     yieldEmptyScalar()
-    level.kind = fplMapKey
-    ancestry.add(level)
-    level = FastParseLevel(kind: fplUnknown, indentation: indentation)
+    p.level.kind = fplMapKey
+    p.ancestry.add(p.level)
+    p.level = FastParseLevel(kind: fplUnknown, indentation: indentation)
   of fplScalar, fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail()
 
 template handleFlowItemStart() {.dirty.} =
-  if level.kind == fplUnknown and ancestry[ancestry.high].kind == fplSequence:
+  if p.level.kind == fplUnknown and
+      p.ancestry[p.ancestry.high].kind == fplSequence:
     handlePossibleMapStart(true, true)
 
-proc startToken(p: YamlParser) {.raises: [], inline.} =
-  p.tokenstart = p.lexer.getColNumber(p.lexer.bufpos)
-
 proc finishLine(lexer: var BaseLexer) {.raises: [], inline.} =
   debug("lex: finishLine")
   while lexer.buf[lexer.bufpos] notin lineEnd:
@@ -604,13 +589,12 @@ proc byteSequence(lexer: var BaseLexer): char {.raises: [YamlParserError].} =
   return char(charCode)
 
 # TODO: {.raises: [].}
-proc processQuotedWhitespace(p: YamlParser, content, after: var string,
-                             newlines: var int) =
-  after.setLen(0)
+proc processQuotedWhitespace(p: YamlParser, newlines: var int) =
+  p.after.reset()
   block outer:
     while true:
       case p.lexer.buf[p.lexer.bufpos]
-      of ' ', '\t': after.add(p.lexer.buf[p.lexer.bufpos])
+      of ' ', '\t': p.after.add(p.lexer.buf[p.lexer.bufpos])
       of '\l':
         p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos)
         break
@@ -618,7 +602,7 @@ proc processQuotedWhitespace(p: YamlParser, content, after: var string,
         p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos)
         break
       else:
-        content.add(after)
+        p.content.add(p.after)
         break outer
       p.lexer.bufpos.inc()
     while true:
@@ -634,13 +618,13 @@ proc processQuotedWhitespace(p: YamlParser, content, after: var string,
         continue
       else:
         if newlines == 0: discard
-        elif newlines == 1: content.add(' ')
-        else: content.addMultiple('\l', newlines - 1)
+        elif newlines == 1: p.content.add(' ')
+        else: p.content.addMultiple('\l', newlines - 1)
         break
       p.lexer.bufpos.inc()
 
 # TODO: {.raises: [YamlParserError].}
-proc doubleQuotedScalar(p: YamlParser, content, after: var string) =
+proc doubleQuotedScalar(p: YamlParser) =
   debug("lex: doubleQuotedScalar")
   p.lexer.bufpos.inc()
   while true:
@@ -653,29 +637,29 @@ proc doubleQuotedScalar(p: YamlParser, content, after: var string) =
       case p.lexer.buf[p.lexer.bufpos]
       of EndOfFile:
         raise p.lexer.generateError("Unfinished escape sequence")
-      of '0':       content.add('\0')
-      of 'a':       content.add('\x07')
-      of 'b':       content.add('\x08')
-      of '\t', 't': content.add('\t')
-      of 'n':       content.add('\l')
-      of 'v':       content.add('\v')
-      of 'f':       content.add('\f')
-      of 'r':       content.add('\c')
-      of 'e':       content.add('\e')
-      of ' ':       content.add(' ')
-      of '"':       content.add('"')
-      of '/':       content.add('/')
-      of '\\':      content.add('\\')
-      of 'N':       content.add(UTF8NextLine)
-      of '_':       content.add(UTF8NonBreakingSpace)
-      of 'L':       content.add(UTF8LineSeparator)
-      of 'P':       content.add(UTF8ParagraphSeparator)
-      of 'x':       content.add(p.lexer.unicodeSequence(2))
-      of 'u':       content.add(p.lexer.unicodeSequence(4))
-      of 'U':       content.add(p.lexer.unicodeSequence(8))
+      of '0':       p.content.add('\0')
+      of 'a':       p.content.add('\x07')
+      of 'b':       p.content.add('\x08')
+      of '\t', 't': p.content.add('\t')
+      of 'n':       p.content.add('\l')
+      of 'v':       p.content.add('\v')
+      of 'f':       p.content.add('\f')
+      of 'r':       p.content.add('\c')
+      of 'e':       p.content.add('\e')
+      of ' ':       p.content.add(' ')
+      of '"':       p.content.add('"')
+      of '/':       p.content.add('/')
+      of '\\':      p.content.add('\\')
+      of 'N':       p.content.add(UTF8NextLine)
+      of '_':       p.content.add(UTF8NonBreakingSpace)
+      of 'L':       p.content.add(UTF8LineSeparator)
+      of 'P':       p.content.add(UTF8ParagraphSeparator)
+      of 'x':       p.content.add(p.lexer.unicodeSequence(2))
+      of 'u':       p.content.add(p.lexer.unicodeSequence(4))
+      of 'U':       p.content.add(p.lexer.unicodeSequence(8))
       of '\l', '\c':
         var newlines = 0
-        p.processQuotedWhitespace(content, after, newlines)
+        p.processQuotedWhitespace(newlines)
         continue
       else: raise p.lexer.generateError("Illegal character in escape sequence")
     of '"':
@@ -683,31 +667,31 @@ proc doubleQuotedScalar(p: YamlParser, content, after: var string) =
       break
     of '\l', '\c', '\t', ' ':
       var newlines = 1
-      p.processQuotedWhitespace(content, after, newlines)
+      p.processQuotedWhitespace(newlines)
       continue
-    else:
-      content.add(c)
+    else: p.content.add(c)
     p.lexer.bufpos.inc()
 
 # TODO: {.raises: [].}
-proc singleQuotedScalar(p: YamlParser, content, after: var string) =
+proc singleQuotedScalar(p: YamlParser) =
   debug("lex: singleQuotedScalar")
   p.lexer.bufpos.inc()
   while true:
     case p.lexer.buf[p.lexer.bufpos]
     of '\'':
       p.lexer.bufpos.inc()
-      if p.lexer.buf[p.lexer.bufpos] == '\'': content.add('\'')
+      if p.lexer.buf[p.lexer.bufpos] == '\'': p.content.add('\'')
       else: break
     of EndOfFile: raise p.lexer.generateError("Unfinished single quoted string")
     of '\l', '\c', '\t', ' ':
       var newlines = 1
-      p.processQuotedWhitespace(content, after, newlines)
+      p.processQuotedWhitespace(newlines)
       continue
-    else: content.add(p.lexer.buf[p.lexer.bufpos])
+    else: p.content.add(p.lexer.buf[p.lexer.bufpos])
     p.lexer.bufpos.inc()
 
-proc isPlainSafe(lexer: BaseLexer, index: int, context: YamlContext): bool =
+proc isPlainSafe(lexer: BaseLexer, index: int, context: YamlContext): bool
+    {.raises: [].} =
   case lexer.buf[lexer.bufpos + 1]
   of spaceOrLineEnd: result = false
   of flowIndicators: result = context == cBlock
@@ -724,149 +708,162 @@ proc isPlainSafe(lexer: BaseLexer, index: int, context: YamlContext): bool =
 #  when context == cBlock: c in plainCharOut
 #  else: c in plainCharIn
 
-proc plainScalar(lexer: var BaseLexer, content, after: var string,
-                 context: static[YamlContext]) {.raises: [].} =
+proc plainScalar(p: YamlParser, context: static[YamlContext]) {.raises: [].} =
   debug("lex: plainScalar")
-  content.add(lexer.buf[lexer.bufpos])
+  p.content.add(p.lexer.buf[p.lexer.bufpos])
   block outer:
     while true:
-      lexer.bufpos.inc()
-      let c = lexer.buf[lexer.bufpos]
+      p.lexer.bufpos.inc()
+      let c = p.lexer.buf[p.lexer.bufpos]
       case c
       of ' ', '\t':
-        after.setLen(1)
-        after[0] = c
+        p.after.setLen(1)
+        p.after[0] = c
         while true:
-          lexer.bufpos.inc()
-          let c2 = lexer.buf[lexer.bufpos]
+          p.lexer.bufpos.inc()
+          let c2 = p.lexer.buf[p.lexer.bufpos]
           case c2
-          of ' ', '\t': after.add(c2)
+          of ' ', '\t': p.after.add(c2)
           of lineEnd: break outer
           of ':':
-            if lexer.isPlainSafe(lexer.bufpos + 1, context):
-              content.add(after & ':')
+            if p.lexer.isPlainSafe(p.lexer.bufpos + 1, context):
+              p.content.add(p.after & ':')
               break
             else: break outer
           of '#': break outer
           of flowIndicators:
             if context == cBlock:
-              content.add(after)
-              content.add(c2)
+              p.content.add(p.after)
+              p.content.add(c2)
               break
             else: break outer
           else:
-            content.add(after)
-            content.add(c2)
+            p.content.add(p.after)
+            p.content.add(c2)
             break
       of flowIndicators:
         when context == cFlow: break
-        else: content.add(c)
+        else: p.content.add(c)
       of lineEnd: break
       of ':':
-        if lexer.isPlainSafe(lexer.bufpos + 1, context): content.add(':')
+        if p.lexer.isPlainSafe(p.lexer.bufpos + 1, context): p.content.add(':')
         else: break outer
-      else: content.add(c)
-  debug("lex: \"" & content & '\"')
+      else: p.content.add(c)
+  debug("lex: \"" & p.content & '\"')
 
-template continueMultilineScalar() {.dirty.} =
-  content.add(if newlines == 1: " " else: repeat('\l', newlines - 1))
+proc continueMultilineScalar(p: YamlParser) {.raises: [].} =
+  p.content.add(if p.newlines == 1: " " else: repeat('\l', p.newlines - 1))
   p.startToken()
-  p.lexer.plainScalar(content, after, cBlock)
-  state = fpBlockAfterPlainScalar
+  p.plainScalar(cBlock)
 
 template handleFlowPlainScalar() {.dirty.} =
-  content.reset()
+  p.content.reset()
   p.startToken()
-  p.lexer.plainScalar(content, after, cFlow)
+  p.plainScalar(cFlow)
   if p.lexer.buf[p.lexer.bufpos] in {'{', '}', '[', ']', ',', ':', '#'}:
     discard
   else:
-    newlines = 0
+    p.newlines = 0
     while true:
       case p.lexer.buf[p.lexer.bufpos]
       of ':':
         if p.lexer.isPlainSafe(p.lexer.bufpos + 1, cFlow):
-          if newlines == 1:
-            content.add(' ')
-            newlines = 0
-          elif newlines > 1:
-            content.addMultiple(' ', newlines - 1)
-            newlines = 0
-          p.lexer.plainScalar(content, after, cFlow)
+          if p.newlines == 1:
+            p.content.add(' ')
+            p.newlines = 0
+          elif p.newlines > 1:
+            p.content.addMultiple(' ', p.newlines - 1)
+            p.newlines = 0
+          p.plainScalar(cFlow)
         break
       of '#', EndOfFile: break
       of '\l':
         p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos)
-        newlines.inc()
+        p.newlines.inc()
       of '\c':
         p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos)
-        newlines.inc()
+        p.newlines.inc()
       of flowIndicators: break
       of ' ', '\t': p.lexer.skipWhitespace()
       else:
-        if newlines == 1:
-          content.add(' ')
-          newlines = 0
-        elif newlines > 1:
-          content.addMultiple(' ', newlines - 1)
-          newlines = 0
-        p.lexer.plainScalar(content, after, cFlow)
-  yieldShallowScalar(content)
+        if p.newlines == 1:
+          p.content.add(' ')
+          p.newlines = 0
+        elif p.newlines > 1:
+          p.content.addMultiple(' ', p.newlines - 1)
+          p.newlines = 0
+        p.plainScalar(cFlow)
+  yieldShallowScalar(p.content)
   handleObjectEnd(fpFlowAfterObject)
 
-proc tagHandle(lexer: var BaseLexer, content: var string,
-                shorthandEnd: var int) {.raises: [YamlParserError].} =
+proc tagHandle(p: YamlParser, shorthandEnd: var int)
+    {.raises: [YamlParserError].} =
   debug("lex: tagHandle")
   shorthandEnd = 0
-  content.add(lexer.buf[lexer.bufpos])
+  p.content.add(p.lexer.buf[p.lexer.bufpos])
   var i = 0
   while true:
-    lexer.bufpos.inc()
+    p.lexer.bufpos.inc()
     i.inc()
-    let c = lexer.buf[lexer.bufpos]
+    let c = p.lexer.buf[p.lexer.bufpos]
     case c
     of spaceOrLineEnd:
-      if shorthandEnd == -1: raise lexer.generateError("Unclosed verbatim tag")
+      if shorthandEnd == -1:
+        raise p.lexer.generateError("Unclosed verbatim tag")
       break
     of '!':
       if shorthandEnd == -1 and i == 2:
-        content.add(c)
+        p.content.add(c)
         continue
       elif shorthandEnd != 0:
-        raise lexer.generateError("Illegal character in tag suffix")
+        raise p.lexer.generateError("Illegal character in tag suffix")
       shorthandEnd = i
-      content.add(c)
+      p.content.add(c)
     of 'a' .. 'z', 'A' .. 'Z', '0' .. '9', '#', ';', '/', '?', ':', '@', '&',
        '-', '=', '+', '$', '_', '.', '~', '*', '\'', '(', ')':
-      content.add(c)
+      p.content.add(c)
     of ',':
       if shortHandEnd > 0: break # ',' after shorthand is flow indicator
-      content.add(c)
+      p.content.add(c)
     of '<':
       if i == 1:
         shorthandEnd = -1
-        content.reset()
-      else: raise lexer.generateError("Illegal character in tag handle")
+        p.content.reset()
+      else: raise p.lexer.generateError("Illegal character in tag handle")
     of '>':
       if shorthandEnd == -1:
-        lexer.bufpos.inc()
-        if lexer.buf[lexer.bufpos] notin spaceOrLineEnd:
-          raise lexer.generateError("Missing space after verbatim tag handle")
+        p.lexer.bufpos.inc()
+        if p.lexer.buf[p.lexer.bufpos] notin spaceOrLineEnd:
+          raise p.lexer.generateError("Missing space after verbatim tag handle")
         break
-      else: raise lexer.generateError("Illegal character in tag handle")
+      else: raise p.lexer.generateError("Illegal character in tag handle")
     of '%':
-      if shorthandEnd != 0: content.add(lexer.byteSequence())
-      else: raise lexer.generateError("Illegal character in tag handle")
-    else: raise lexer.generateError("Illegal character in tag handle")
+      if shorthandEnd != 0: p.content.add(p.lexer.byteSequence())
+      else: raise p.lexer.generateError("Illegal character in tag handle")
+    else: raise p.lexer.generateError("Illegal character in tag handle")
 
-proc anchorName(lexer: var BaseLexer, content: var string) {.raises: [].} =
-  debug("lex: anchorName")
-  while true:
-    lexer.bufpos.inc()
-    let c = lexer.buf[lexer.bufpos]
-    case c
-    of spaceOrLineEnd, '[', ']', '{', '}', ',': break
-    else: content.add(c)
+proc handleTagHandle(p: YamlParser) {.raises: [YamlParserError].} =
+  p.startToken()
+  if p.level.kind != fplUnknown: raise p.generateError("Unexpected tag handle")
+  if p.tag != yTagQuestionMark:
+    raise p.generateError("Only one tag handle is allowed per node")
+  p.content.reset()
+  var
+    shorthandEnd: int
+  p.tagHandle(shorthandEnd)
+  if shorthandEnd != -1:
+    try:
+      p.tagUri.reset()
+      p.tagUri.add(p.shorthands[p.content[0..shorthandEnd]])
+      p.tagUri.add(p.content[shorthandEnd + 1 .. ^1])
+    except KeyError:
+      raise p.generateError(
+          "Undefined tag shorthand: " & p.content[0..shorthandEnd])
+    try: p.tag = p.tagLib.tags[p.tagUri]
+    except KeyError: p.tag = p.tagLib.registerUri(p.tagUri)
+  else:
+    try: p.tag = p.tagLib.tags[p.content]
+    except KeyError: p.tag = p.tagLib.registerUri(p.content)
 
 proc consumeLineIfEmpty(p: YamlParser, newlines: var int): bool =
   result = true
@@ -889,15 +886,15 @@ proc consumeLineIfEmpty(p: YamlParser, newlines: var int): bool =
       break
 
 template startScalar(t: ScalarType) {.dirty.} =
-  newlines = 0
-  level.kind = fplScalar
+  p.newlines = 0
+  p.level.kind = fplScalar
   scalarType = t
 
 template blockScalarHeader() {.dirty.} =
   debug("lex: blockScalarHeader")
   chomp = ctClip
-  level.indentation = UnknownIndentation
-  if tag == yTagQuestionMark: tag = yTagExclamationMark
+  p.level.indentation = UnknownIndentation
+  if p.tag == yTagQuestionMark: p.tag = yTagExclamationMark
   let t = if p.lexer.buf[p.lexer.bufpos] == '|': stLiteral else: stFolded
   while true:
     p.lexer.bufpos.inc()
@@ -911,9 +908,9 @@ template blockScalarHeader() {.dirty.} =
         raise p.lexer.generateError("Only one chomping indicator is allowed")
       chomp = ctStrip
     of '1'..'9':
-      if level.indentation != UnknownIndentation:
+      if p.level.indentation != UnknownIndentation:
         raise p.lexer.generateError("Only one indentation indicator is allowed")
-      level.indentation = ancestry[ancestry.high].indentation +
+      p.level.indentation = p.ancestry[p.ancestry.high].indentation +
           ord(p.lexer.buf[p.lexer.bufpos]) - ord('\x30')
     of spaceOrLineEnd: break
     else:
@@ -924,69 +921,63 @@ template blockScalarHeader() {.dirty.} =
   p.lineEnding()
   handleLineEnd(true)
   startScalar(t)
-  content.reset()
+  p.content.reset()
 
 template blockScalarLine() {.dirty.} =
   debug("lex: blockScalarLine")
-  if indentation < level.indentation:
+  if indentation < p.level.indentation:
     if p.lexer.buf[p.lexer.bufpos] == '#':
       # skip all following comment lines
-      while indentation > ancestry[ancestry.high].indentation:
+      while indentation > p.ancestry[p.ancestry.high].indentation:
         p.lineEnding()
         handleLineEnd(true)
-        newlines.inc(-1)
+        p.newlines.inc(-1)
         p.lexer.skipIndentation()
         indentation = p.lexer.getColNumber(p.lexer.bufpos)
-      if indentation > ancestry[ancestry.high].indentation:
+      if indentation > p.ancestry[p.ancestry.high].indentation:
         raise p.lexer.generateError(
             "Invalid content in block scalar after comments")
       closeMoreIndentedLevels()
     else:
       raise p.lexer.generateError(
           "Invalid indentation (expected indentation of at least " &
-          $level.indentation & " spaces)")    
+          $p.level.indentation & " spaces)")    
   else:
-    if level.indentation == UnknownIndentation:
+    if p.level.indentation == UnknownIndentation:
       if p.lexer.buf[p.lexer.bufpos] in lineEnd:
         handleLineEnd(true)
         continue
       else:
-        level.indentation = indentation
-        content.addMultiple('\l', newlines)
-    elif indentation > level.indentation or p.lexer.buf[p.lexer.bufpos] == '\t':
-      content.addMultiple('\l', newlines)
+        p.level.indentation = indentation
+        p.content.addMultiple('\l', p.newlines)
+    elif indentation > p.level.indentation or
+        p.lexer.buf[p.lexer.bufpos] == '\t':
+      p.content.addMultiple('\l', p.newlines)
       recentWasMoreIndented = true
-      content.addMultiple(' ', indentation - level.indentation)
+      p.content.addMultiple(' ', indentation - p.level.indentation)
     elif scalarType == stFolded:
       if recentWasMoreIndented:
         recentWasMoreIndented = false
-        newlines.inc()
-      if newlines == 0: discard
-      elif newlines == 1: content.add(' ')
-      else: content.addMultiple('\l', newlines - 1)    
-    else: content.addMultiple('\l', newlines)
-    newlines = 0
+        p.newlines.inc()
+      if p.newlines == 0: discard
+      elif p.newlines == 1: p.content.add(' ')
+      else: p.content.addMultiple('\l', p.newlines - 1)    
+    else: p.content.addMultiple('\l', p.newlines)
+    p.newlines = 0
     while p.lexer.buf[p.lexer.bufpos] notin lineEnd:
-      content.add(p.lexer.buf[p.lexer.bufpos])
+      p.content.add(p.lexer.buf[p.lexer.bufpos])
       p.lexer.bufpos.inc()
     handleLineEnd(true)
 
 proc parse*(p: YamlParser, s: Stream): YamlStream =
+  p.content.reset()
+  p.after.reset()
+  p.tagUri.reset()
+  p.ancestry.setLen(0)
   var backend = iterator(): YamlStreamEvent =
     var
       state = fpInitial
-      shorthands: Table[string, string]
-      anchors: Table[string, AnchorId]
-      nextAnchorId: AnchorId
-      content: string = ""      
-      after: string = ""
-      tagUri: string = ""
-      tag: TagId
-      anchor: AnchorId
-      ancestry = newSeq[FastParseLevel]()
-      level: FastParseLevel
       indentation: int
-      newlines: int
       flowdepth: int = 0
       explicitFlowKey: bool
       scalarType: ScalarType
@@ -1020,11 +1011,11 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             handleLineEnd(false)
           of ldTag:
             var shorthand = ""
-            tagUri.setLen(0)
+            p.tagUri.reset()
             p.startToken()
             p.lexer.tagShorthand(shorthand)
-            p.lexer.tagUriMapping(tagUri)
-            shorthands[shorthand] = tagUri
+            p.lexer.tagUriMapping(p.tagUri)
+            p.shorthands[shorthand] = p.tagUri
             p.lineEnding()
             handleLineEnd(false)
           of ldUnknown:
@@ -1034,7 +1025,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             p.lexer.finishLine()
             handleLineEnd(false)
         of ' ', '\t':
-          if not p.consumeLineIfEmpty(newlines):
+          if not p.consumeLineIfEmpty(p.newlines):
             indentation = p.lexer.getColNumber(p.lexer.bufpos)
             yield startDocEvent()
             state = fpBlockObjectStart
@@ -1060,23 +1051,23 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
         p.lexer.skipWhitespace()
         case p.lexer.buf[p.lexer.bufpos]
         of '\l':
-          if level.kind notin {fplUnknown, fplScalar}:
+          if p.level.kind notin {fplUnknown, fplScalar}:
             p.startToken()
             raise p.generateError("Unexpected scalar")
           startScalar(stFlow)
           p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos)
-          newlines.inc()
+          p.newlines.inc()
           state = fpBlockObjectStart
         of '\c':
-          if level.kind notin {fplUnknown, fplScalar}:
+          if p.level.kind notin {fplUnknown, fplScalar}:
             p.startToken()
             raise p.generateError("Unexpected scalar")
           startScalar(stFlow)
           p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos)
-          newlines.inc()
+          p.newlines.inc()
           state = fpBlockObjectStart
         else:
-          yieldShallowScalar(content)
+          yieldShallowScalar(p.content)
           handleObjectEnd(fpBlockAfterObject)
       of fpBlockAfterObject:
         debug("state: blockAfterObject")
@@ -1092,18 +1083,18 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           state = fpBlockObjectStart
           p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos)
         of ':':
-          case level.kind
+          case p.level.kind
           of fplUnknown:
             handleObjectStart(yamlStartMap)
           of fplMapKey:
             yield scalarEvent("", yTagQuestionMark, yAnchorNone)
-            level.kind = fplMapValue
-            ancestry.add(level)
-            level = initLevel(fplUnknown)
+            p.level.kind = fplMapValue
+            p.ancestry.add(p.level)
+            p.level = initLevel(fplUnknown)
           of fplMapValue:
-            level.kind = fplMapValue
-            ancestry.add(level)
-            level = initLevel(fplUnknown)
+            p.level.kind = fplMapValue
+            p.ancestry.add(p.level)
+            p.level = initLevel(fplUnknown)
           of fplSequence:
             p.startToken()
             raise p.generateError("Illegal token (expected sequence item)")
@@ -1145,7 +1136,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             handleLineEnd(false)
             state = fpAfterDocument
             continue
-        if indentation <= ancestry[ancestry.high].indentation:
+        if indentation <= p.ancestry[p.ancestry.high].indentation:
           if p.lexer.buf[p.lexer.bufpos] in lineEnd:
             handleLineEnd(true)
             continue
@@ -1157,66 +1148,71 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
               p.lexer.isPlainSafe(p.lexer.bufpos + 1, cBlock):
             closeMoreIndentedLevels(true)
           else: closeMoreIndentedLevels()
-        elif indentation <= level.indentation and
+        elif indentation <= p.level.indentation and
             p.lexer.buf[p.lexer.bufpos] in lineEnd:
           handleLineEnd(true)
           continue
-        if level.kind == fplScalar and scalarType != stFlow:
+        if p.level.kind == fplScalar and scalarType != stFlow:
           blockScalarLine()
           continue
         case p.lexer.buf[p.lexer.bufpos]
         of '\l':
           p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos)
-          newlines.inc()
-          if level.kind == fplUnknown: level.indentation = UnknownIndentation
+          p.newlines.inc()
+          if p.level.kind == fplUnknown:
+            p.level.indentation = UnknownIndentation
         of '\c':
           p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos)
-          newlines.inc()
-          if level.kind == fplUnknown: level.indentation = UnknownIndentation
+          p.newlines.inc()
+          if p.level.kind == fplUnknown:
+            p.level.indentation = UnknownIndentation
         of EndOfFile:
           closeEverything()
           return
         of '#':
           p.lineEnding()
           handleLineEnd(true)
-          if level.kind == fplUnknown: level.indentation = UnknownIndentation
+          if p.level.kind == fplUnknown:
+            p.level.indentation = UnknownIndentation
         of '\'':
           handleBlockItemStart()
-          content.reset()
+          p.content.reset()
           p.startToken()
-          p.singleQuotedScalar(content, after)
-          if tag == yTagQuestionMark: tag = yTagExclamationMark
-          yieldShallowScalar(content)
+          p.singleQuotedScalar()
+          if p.tag == yTagQuestionMark: p.tag = yTagExclamationMark
+          yieldShallowScalar(p.content)
           handleObjectEnd(fpBlockAfterObject)
         of '"':
           handleBlockItemStart()
-          content.reset()
+          p.content.reset()
           p.startToken()
-          p.doubleQuotedScalar(content, after)
-          if tag == yTagQuestionMark: tag = yTagExclamationMark
-          yieldShallowScalar(content)
+          p.doubleQuotedScalar()
+          if p.tag == yTagQuestionMark: p.tag = yTagExclamationMark
+          yieldShallowScalar(p.content)
           handleObjectEnd(fpBlockAfterObject)
         of '|', '>':
           blockScalarHeader()
           continue
         of '-':
           if p.lexer.isPlainSafe(p.lexer.bufpos + 1, cBlock):
-            if level.kind == fplScalar: continueMultilineScalar()
+            if p.level.kind == fplScalar:
+              p.continueMultilineScalar()
+              state = fpBlockAfterPlainScalar
             else:
               handleBlockItemStart()
-              content.reset()
+              p.content.reset()
               p.startToken()
-              p.lexer.plainScalar(content, after, cBlock)
+              p.plainScalar(cBlock)
               state = fpBlockAfterPlainScalar
           else:
             p.lexer.bufpos.inc()
             handleBlockSequenceIndicator()
         of '!':
           handleBlockItemStart()
-          handleTagHandle()
+          p.handleTagHandle()
         of '&':
           handleBlockItemStart()
-          handleAnchor()
+          p.handleAnchor()
         of '*':
           handleBlockItemStart()
           handleAlias()
@@ -1225,24 +1221,28 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           state = fpFlow
         of '?':
           if p.lexer.isPlainSafe(p.lexer.bufpos + 1, cBlock):
-            if level.kind == fplScalar: continueMultilineScalar()
+            if p.level.kind == fplScalar:
+              p.continueMultilineScalar()
+              state = fpBlockAfterPlainScalar
             else:
               handleBlockItemStart()
-              content.reset()
+              p.content.reset()
               p.startToken()
-              p.lexer.plainScalar(content, after, cBlock)
+              p.plainScalar(cBlock)
               state = fpBlockAfterPlainScalar
           else:
             p.lexer.bufpos.inc()
             handleMapKeyIndicator()
         of ':':
           if p.lexer.isPlainSafe(p.lexer.bufpos + 1, cBlock):
-            if level.kind == fplScalar: continueMultilineScalar()
+            if p.level.kind == fplScalar:
+              p.continueMultilineScalar()
+              state = fpBlockAfterPlainScalar
             else:
               handleBlockItemStart()
-              content.reset()
+              p.content.reset()
               p.startToken()
-              p.lexer.plainScalar(content, after, cBlock)
+              p.plainScalar(cBlock)
               state = fpBlockAfterPlainScalar
           else:
             p.lexer.bufpos.inc()
@@ -1251,17 +1251,20 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           raise p.lexer.generateError(
               "Reserved characters cannot start a plain scalar")
         of '\t':
-          if level.kind == fplScalar:
+          if p.level.kind == fplScalar:
             p.lexer.skipWhitespace()
-            continueMultilineScalar()
+            p.continueMultilineScalar()
+            state = fpBlockAfterPlainScalar
           else: raise p.lexer.generateError("\\t cannot start any token")
         else:
-          if level.kind == fplScalar: continueMultilineScalar()
+          if p.level.kind == fplScalar:
+            p.continueMultilineScalar()
+            state = fpBlockAfterPlainScalar
           else:
             handleBlockItemStart()
-            content.reset()
+            p.content.reset()
             p.startToken()
-            p.lexer.plainScalar(content, after, cBlock)
+            p.plainScalar(cBlock)
             state = fpBlockAfterPlainScalar
       of fpExpectDocEnd:
         debug("state: expectDocEnd")
@@ -1272,7 +1275,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           if success:
             p.lexer.bufpos.inc(3)
             yield endDocEvent()
-            discard ancestry.pop()
+            discard p.ancestry.pop()
             initDocValues()
             yield startDocEvent()
             state = fpBlockObjectStart
@@ -1320,7 +1323,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           p.lineEnding()
           handleLineEnd(false)
         of '\t', ' ':
-          if not p.consumeLineIfEmpty(newlines):
+          if not p.consumeLineIfEmpty(p.newlines):
             indentation = p.lexer.getColNumber(p.lexer.bufpos)
             initDocValues()
             yield startDocEvent()
@@ -1345,14 +1348,14 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           flowdepth.inc()
           p.lexer.bufpos.inc()
         of '}':
-          assert(level.kind == fplUnknown)
-          level = ancestry.pop()
-          case level.kind
+          assert(p.level.kind == fplUnknown)
+          p.level = p.ancestry.pop()
+          case p.level.kind
           of fplMapValue:
             yieldEmptyScalar()
-            level.kind = fplMapKey
+            p.level.kind = fplMapKey
           of fplMapKey:
-            if tag != yTagQuestionMark or anchor != yAnchorNone or
+            if p.tag != yTagQuestionMark or p.anchor != yAnchorNone or
                 explicitFlowKey:
               yieldEmptyScalar()
               yield scalarEvent("", yTagQuestionMark, yAnchorNone)
@@ -1366,17 +1369,17 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           p.lexer.bufpos.inc()
           leaveFlowLevel()
         of ']':
-          assert(level.kind == fplUnknown)
-          level = ancestry.pop()
-          case level.kind
+          assert(p.level.kind == fplUnknown)
+          p.level = p.ancestry.pop()
+          case p.level.kind
           of fplSequence:
-            if tag != yTagQuestionMark or anchor != yAnchorNone:
+            if p.tag != yTagQuestionMark or p.anchor != yAnchorNone:
               yieldEmptyScalar()
           of fplSinglePairValue:
             yieldEmptyScalar()
-            level = ancestry.pop()
+            p.level = p.ancestry.pop()
             yield endMapEvent()
-            assert(level.kind == fplSequence)
+            assert(p.level.kind == fplSequence)
           of fplMapKey, fplMapValue:
             p.startToken()
             raise p.generateError("Unexpected token (expected '}')")
@@ -1384,13 +1387,13 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           p.lexer.bufpos.inc()
           leaveFlowLevel()
         of ',':
-          assert(level.kind == fplUnknown)
-          level = ancestry.pop()
-          case level.kind
+          assert(p.level.kind == fplUnknown)
+          p.level = p.ancestry.pop()
+          case p.level.kind
           of fplSequence: yieldEmptyScalar()
           of fplMapValue:
             yieldEmptyScalar()
-            level.kind = fplMapKey
+            p.level.kind = fplMapKey
             explicitFlowKey = false
           of fplMapKey:
             yieldEmptyScalar
@@ -1398,67 +1401,67 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             explicitFlowKey = false
           of fplSinglePairValue:
             yieldEmptyScalar()
-            level = ancestry.pop()
+            p.level = p.ancestry.pop()
             yield endMapEvent()
-            assert(level.kind == fplSequence)
+            assert(p.level.kind == fplSequence)
           of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail()
-          ancestry.add(level)
-          level = initLevel(fplUnknown)
+          p.ancestry.add(p.level)
+          p.level = initLevel(fplUnknown)
           p.lexer.bufpos.inc()
         of ':':
           if p.lexer.isPlainSafe(p.lexer.bufpos + 1, cFlow):
             handleFlowItemStart()
             handleFlowPlainScalar()
           else:
-            level = ancestry.pop()
-            case level.kind
+            p.level = p.ancestry.pop()
+            case p.level.kind
             of fplSequence:
-              yield startMapEvent(tag, anchor)
+              yield startMapEvent(p.tag, p.anchor)
               debug("started single-pair map at " &
-                  (if level.indentation == UnknownIndentation:
-                   $indentation else: $level.indentation))
-              tag = yTagQuestionMark
-              anchor = yAnchorNone
-              if level.indentation == UnknownIndentation:
-                level.indentation = indentation
-              ancestry.add(level)
-              level = initLevel(fplSinglePairValue)
+                  (if p.level.indentation == UnknownIndentation:
+                   $indentation else: $p.level.indentation))
+              p.tag = yTagQuestionMark
+              p.anchor = yAnchorNone
+              if p.level.indentation == UnknownIndentation:
+                p.level.indentation = indentation
+              p.ancestry.add(p.level)
+              p.level = initLevel(fplSinglePairValue)
               yield scalarEvent("")
             of fplMapValue, fplSinglePairValue:
               p.startToken()
               raise p.generateError("Unexpected token (expected ',')")
             of fplMapKey:
               yieldEmptyScalar()
-              level.kind = fplMapValue
+              p.level.kind = fplMapValue
             of fplSinglePairKey:
               yieldEmptyScalar()
-              level.kind = fplSinglePairValue
+              p.level.kind = fplSinglePairValue
             of fplUnknown, fplScalar, fplDocument: debugFail()
-            ancestry.add(level)
-            level = initLevel(fplUnknown)
+            p.ancestry.add(p.level)
+            p.level = initLevel(fplUnknown)
             p.lexer.bufpos.inc()
         of '\'':
           handleFlowItemStart()
-          content.reset()
+          p.content.reset()
           p.startToken()
-          p.singleQuotedScalar(content, after)
-          if tag == yTagQuestionMark: tag = yTagExclamationMark
-          yieldShallowScalar(content)
+          p.singleQuotedScalar()
+          if p.tag == yTagQuestionMark: p.tag = yTagExclamationMark
+          yieldShallowScalar(p.content)
           handleObjectEnd(fpFlowAfterObject)
         of '"':
           handleFlowItemStart()
-          content.reset()
+          p.content.reset()
           p.startToken()
-          p.doubleQuotedScalar(content, after)
-          if tag == yTagQuestionMark: tag = yTagExclamationMark
-          yieldShallowScalar(content)
+          p.doubleQuotedScalar()
+          if p.tag == yTagQuestionMark: p.tag = yTagExclamationMark
+          yieldShallowScalar(p.content)
           handleObjectEnd(fpFlowAfterObject)
         of '!':
           handleFlowItemStart()
-          handleTagHandle()
+          p.handleTagHandle()
         of '&':
           handleFlowItemStart()
-          handleAnchor()
+          p.handleAnchor()
         of '*':
           handleAlias()
           state = fpFlowAfterObject
@@ -1469,8 +1472,8 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           elif explicitFlowKey:
             p.startToken()
             raise p.generateError("Duplicate '?' in flow mapping")
-          elif level.kind == fplUnknown:
-            case ancestry[ancestry.high].kind
+          elif p.level.kind == fplUnknown:
+            case p.ancestry[p.ancestry.high].kind
             of fplMapKey, fplMapValue, fplDocument: discard
             of fplSequence: handleObjectStart(yamlStartMap, true)
             else:
@@ -1489,20 +1492,20 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
         p.lexer.skipWhitespaceCommentsAndNewlines()
         case p.lexer.buf[p.lexer.bufpos]
         of ']':
-          case level.kind
+          case p.level.kind
           of fplSequence: discard
           of fplMapKey, fplMapValue:
             p.startToken()
             raise p.generateError("Unexpected token (expected '}')")
           of fplSinglePairValue:
-            level = ancestry.pop()
-            assert(level.kind == fplSequence)
+            p.level = p.ancestry.pop()
+            assert(p.level.kind == fplSequence)
             yield endMapEvent()
           of fplScalar, fplUnknown, fplSinglePairKey, fplDocument: debugFail()
           p.lexer.bufpos.inc()
           leaveFlowLevel()
         of '}':
-          case level.kind
+          case p.level.kind
           of fplMapKey, fplMapValue: discard
           of fplSequence, fplSinglePairValue:
             p.startToken()
@@ -1511,31 +1514,31 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           p.lexer.bufpos.inc()
           leaveFlowLevel()
         of ',':
-          case level.kind
+          case p.level.kind
           of fplSequence: discard
           of fplMapValue:
             yield scalarEvent("", yTagQuestionMark, yAnchorNone)
-            level.kind = fplMapKey
+            p.level.kind = fplMapKey
             explicitFlowKey = false
           of fplSinglePairValue:
-            level = ancestry.pop()
-            assert(level.kind == fplSequence)
+            p.level = p.ancestry.pop()
+            assert(p.level.kind == fplSequence)
             yield endMapEvent()
           of fplMapKey: explicitFlowKey = false
           of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail()
-          ancestry.add(level)
-          level = initLevel(fplUnknown)
+          p.ancestry.add(p.level)
+          p.level = initLevel(fplUnknown)
           state = fpFlow
           p.lexer.bufpos.inc()
         of ':':
-          case level.kind
+          case p.level.kind
           of fplSequence, fplMapKey:
             p.startToken()
             raise p.generateError("Unexpected token (expected ',')")
           of fplMapValue, fplSinglePairValue: discard
           of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail()
-          ancestry.add(level)
-          level = initLevel(fplUnknown)
+          p.ancestry.add(p.level)
+          p.level = initLevel(fplUnknown)
           state = fpFlow
           p.lexer.bufpos.inc()
         of '#':
diff --git a/yaml.nim b/yaml.nim
index 323e91e..36b0462 100644
--- a/yaml.nim
+++ b/yaml.nim
@@ -144,6 +144,14 @@ type
     ##   ``1.2``.
     ## - If there is an unknown directive encountered.
     
+  FastParseLevelKind = enum
+    fplUnknown, fplSequence, fplMapKey, fplMapValue, fplSinglePairKey,
+    fplSinglePairValue, fplScalar, fplDocument
+  
+  FastParseLevel = object
+    kind: FastParseLevelKind
+    indentation: int
+  
   YamlParser* = ref object
     ## A parser object. Retains its ``TagLibrary`` across calls to
     ## `parse <#parse,YamlParser,Stream>`_. Can be used
@@ -151,10 +159,19 @@ type
     ## only until the document goes out of scope (i.e. until
     ## ``yamlEndDocument`` is yielded).
     tagLib: TagLibrary
-    anchors: OrderedTable[string, AnchorId]
     callback: WarningCallback
     lexer: BaseLexer
     tokenstart: int
+    content, after: string
+    ancestry: seq[FastParseLevel]
+    level: FastParseLevel
+    tagUri: string
+    tag: TagId
+    anchor: AnchorId
+    shorthands: Table[string, string]
+    anchors: Table[string, AnchorId]
+    nextAnchorId: AnchorId
+    newlines: int
         
   PresentationStyle* = enum
     ## Different styles for YAML character stream output.

From f5431af0fdc36d0142b09ea196b011e85519056e Mon Sep 17 00:00:00 2001
From: Felix Krause <contact@flyx.org>
Date: Mon, 23 May 2016 21:10:28 +0200
Subject: [PATCH 4/4] fastparse refactoring, part III

 * Changed some more templates to procs
---
 private/fastparse.nim | 187 +++++++++++++++++++++---------------------
 yaml.nim              |   1 +
 2 files changed, 93 insertions(+), 95 deletions(-)

diff --git a/private/fastparse.nim b/private/fastparse.nim
index 765bce2..12dd460 100644
--- a/private/fastparse.nim
+++ b/private/fastparse.nim
@@ -86,23 +86,23 @@ proc reset(buffer: var string) {.raises: [], inline.} = buffer.setLen(0)
 proc initLevel(k: FastParseLevelKind): FastParseLevel {.raises: [], inline.} =
   FastParseLevel(kind: k, indentation: UnknownIndentation)
 
-template yieldEmptyScalar() {.dirty.} =
-  yield scalarEvent("", p.tag, p.anchor)
+proc emptyScalar(p: YamlParser): YamlStreamEvent {.raises: [], inline.} =
+  result = scalarEvent("", p.tag, p.anchor)
   p.tag = yTagQuestionMark
   p.anchor = yAnchorNone
 
-template yieldShallowScalar(content: string) {.dirty.} =
-  var e = YamlStreamEvent(kind: yamlScalar, scalarTag: p.tag,
-                          scalarAnchor: p.anchor)
-  shallowCopy(e.scalarContent, content)
-  yield e
+proc currentScalar(p: YamlParser): YamlStreamEvent {.raises: [], inline.} =
+  result = YamlStreamEvent(kind: yamlScalar, scalarTag: p.tag,
+                           scalarAnchor: p.anchor, scalarContent: p.content)
+  p.tag = yTagQuestionMark
+  p.anchor = yAnchorNone
 
 template yieldLevelEnd() {.dirty.} =
   case p.level.kind
   of fplSequence: yield endSeqEvent()
   of fplMapKey: yield endMapEvent()
   of fplMapValue, fplSinglePairValue:
-    yieldEmptyScalar()
+    yield emptyScalar(p)
     yield endMapEvent()
   of fplScalar:
     if scalarType != stFlow:
@@ -113,11 +113,12 @@ template yieldLevelEnd() {.dirty.} =
       of ctClip:
         if p.content.len != 0: p.content.add('\l')
       of ctStrip: discard
-    yieldShallowScalar(p.content)
+    yield currentScalar(p)
     p.tag = yTagQuestionMark
     p.anchor = yAnchorNone
   of fplUnknown:
-    if p.ancestry.len > 1: yieldEmptyScalar() # don't yield scalar for empty doc
+    if p.ancestry.len > 1:
+      yield emptyScalar(p) # don't yield scalar for empty doc
   of fplSinglePairKey, fplDocument: debugFail()
 
 template handleLineEnd(insideDocument: bool) {.dirty.} =
@@ -136,8 +137,6 @@ template handleObjectEnd(nextState: FastParseState) {.dirty.} =
     yield endMapEvent()
     p.level = p.ancestry.pop()
   state = if p.level.kind == fplDocument: fpExpectDocEnd else: nextState
-  p.tag = yTagQuestionMark
-  p.anchor = yAnchorNone
   case p.level.kind
   of fplMapKey: p.level.kind = fplMapValue
   of fplSinglePairKey: p.level.kind = fplSinglePairValue
@@ -145,53 +144,53 @@ template handleObjectEnd(nextState: FastParseState) {.dirty.} =
   of fplSequence, fplDocument: discard
   of fplUnknown, fplScalar, fplSinglePairValue: debugFail()
 
-template handleObjectStart(k: YamlStreamEventKind, single: bool = false)
-    {.dirty.} =
+proc objectStart(p: YamlParser, k: static[YamlStreamEventKind],
+                 single: bool = false): YamlStreamEvent {.raises: [].} =
   assert(p.level.kind == fplUnknown)
   when k == yamlStartMap:
-    yield startMapEvent(p.tag, p.anchor)
+    result = startMapEvent(p.tag, p.anchor)
     if single:
       debug("started single-pair map at " &
-          (if p.level.indentation == UnknownIndentation: $indentation else:
+          (if p.level.indentation == UnknownIndentation: $p.indentation else:
            $p.level.indentation))
       p.level.kind = fplSinglePairKey
     else:
       debug("started map at " &
-          (if p.level.indentation == UnknownIndentation: $indentation else:
+          (if p.level.indentation == UnknownIndentation: $p.indentation else:
            $p.level.indentation))
       p.level.kind = fplMapKey
   else:
-    yield startSeqEvent(p.tag, p.anchor)
+    result = startSeqEvent(p.tag, p.anchor)
     debug("started sequence at " &
-        (if p.level.indentation == UnknownIndentation: $indentation else:
+        (if p.level.indentation == UnknownIndentation: $p.indentation else:
          $p.level.indentation))
     p.level.kind = fplSequence
   p.tag = yTagQuestionMark
   p.anchor = yAnchorNone
   if p.level.indentation == UnknownIndentation:
-    p.level.indentation = indentation
+    p.level.indentation = p.indentation
   p.ancestry.add(p.level)
   p.level = initLevel(fplUnknown)
   
 template closeMoreIndentedLevels(atSequenceItem: bool = false) {.dirty.} =
   while p.level.kind != fplDocument:
     let parent = p.ancestry[p.ancestry.high]
-    if parent.indentation >= indentation:
+    if parent.indentation >= p.indentation:
       when atSequenceItem:
-        if (indentation == p.level.indentation and
+        if (p.indentation == p.level.indentation and
             p.level.kind == fplSequence) or
-           (indentation == parent.indentation and p.level.kind == fplUnknown and
-            parent.kind != fplSequence):
+           (p.indentation == parent.indentation and
+            p.level.kind == fplUnknown and parent.kind != fplSequence):
           break
       debug("Closing because parent.indentation (" & $parent.indentation &
-            ") >= indentation(" & $indentation & ")")
+            ") >= indentation(" & $p.indentation & ")")
       yieldLevelEnd()
       handleObjectEnd(state)
     else: break
   if p.level.kind == fplDocument: state = fpExpectDocEnd
 
 template closeEverything() {.dirty.} =
-  indentation = 0
+  p.indentation = 0
   closeMoreIndentedLevels()
   case p.level.kind
   of fplUnknown: discard p.ancestry.pop()
@@ -204,30 +203,30 @@ template closeEverything() {.dirty.} =
 template handleBlockSequenceIndicator() {.dirty.} =
   p.startToken()
   case p.level.kind
-  of fplUnknown: handleObjectStart(yamlStartSeq)
+  of fplUnknown: yield p.objectStart(yamlStartSeq)
   of fplSequence:
-    if p.level.indentation != indentation:
-      raise p.generateError("Invalid indentation of block sequence indicator")
+    if p.level.indentation != p.indentation:
+      raise p.generateError("Invalid p.indentation of block sequence indicator")
     p.ancestry.add(p.level)
     p.level = initLevel(fplUnknown)
   else: raise p.generateError("Illegal sequence item in map")
   p.lexer.skipWhitespace()
-  indentation = p.lexer.getColNumber(p.lexer.bufpos)
+  p.indentation = p.lexer.getColNumber(p.lexer.bufpos)
 
 template handleMapKeyIndicator() {.dirty.} =
   p.startToken()
   case p.level.kind
-  of fplUnknown: handleObjectStart(yamlStartMap)
+  of fplUnknown: yield p.objectStart(yamlStartMap)
   of fplMapValue:
-    if p.level.indentation != indentation:
-      raise p.generateError("Invalid indentation of map key indicator")
+    if p.level.indentation != p.indentation:
+      raise p.generateError("Invalid p.indentation of map key indicator")
     yield scalarEvent("", yTagQuestionMark, yAnchorNone)
     p.level.kind = fplMapKey
     p.ancestry.add(p.level)
     p.level = initLevel(fplUnknown)
   of fplMapKey:
-    if p.level.indentation != indentation:
-      raise p.generateError("Invalid indentation of map key indicator")
+    if p.level.indentation != p.indentation:
+      raise p.generateError("Invalid p.indentation of map key indicator")
     p.ancestry.add(p.level)
     p.level = initLevel(fplUnknown)
   of fplSequence:
@@ -237,27 +236,27 @@ template handleMapKeyIndicator() {.dirty.} =
         "Unexpected map key indicator (expected multiline scalar end)")
   of fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail()
   p.lexer.skipWhitespace()
-  indentation = p.lexer.getColNumber(p.lexer.bufpos)
+  p.indentation = p.lexer.getColNumber(p.lexer.bufpos)
 
 template handleMapValueIndicator() {.dirty.} =
   p.startToken()
   case p.level.kind
   of fplUnknown:
     if p.level.indentation == UnknownIndentation:
-      handleObjectStart(yamlStartMap)
+      yield p.objectStart(yamlStartMap)
       yield scalarEvent("", yTagQuestionMark, yAnchorNone)
-    else: yieldEmptyScalar()
+    else: yield emptyScalar(p)
     p.ancestry[p.ancestry.high].kind = fplMapValue
   of fplMapKey:
-    if p.level.indentation != indentation:
-      raise p.generateError("Invalid indentation of map key indicator")
+    if p.level.indentation != p.indentation:
+      raise p.generateError("Invalid p.indentation of map key indicator")
     yield scalarEvent("", yTagQuestionMark, yAnchorNone)
     p.level.kind = fplMapValue
     p.ancestry.add(p.level)
     p.level = initLevel(fplUnknown)
   of fplMapValue:
-    if p.level.indentation != indentation:
-      raise p.generateError("Invalid indentation of map key indicator")
+    if p.level.indentation != p.indentation:
+      raise p.generateError("Invalid p.indentation of map key indicator")
     p.ancestry.add(p.level)
     p.level = initLevel(fplUnknown)
   of fplSequence:
@@ -267,9 +266,9 @@ template handleMapValueIndicator() {.dirty.} =
         "Unexpected map value indicator (expected multiline scalar end)")
   of fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail()
   p.lexer.skipWhitespace()
-  indentation = p.lexer.getColNumber(p.lexer.bufpos)
+  p.indentation = p.lexer.getColNumber(p.lexer.bufpos)
 
-template initDocValues() {.dirty.} =
+proc initDocValues(p: YamlParser) {.raises: [].} =
   p.shorthands = initTable[string, string]()
   p.anchors = initTable[string, AnchorId]()
   p.shorthands["!"] = "!"
@@ -336,7 +335,7 @@ template handlePossibleMapStart(flow: bool = false,
       of ':':
         if flowDepth == 0 and (p.lexer.buf[pos + 1] in spaceOrLineEnd or
             recentJsonStyle):
-          handleObjectStart(yamlStartMap, single)
+          yield p.objectStart(yamlStartMap, single)
           break
       of lineEnd: break
       of '[', '{': flowDepth.inc()
@@ -368,7 +367,7 @@ template handlePossibleMapStart(flow: bool = false,
         recentJsonStyle = p.lexer.buf[pos] in {']', '}', '\'', '"'}
       pos.inc()
     if p.level.indentation == UnknownIndentation:
-      p.level.indentation = indentation
+      p.level.indentation = p.indentation
 
 template handleBlockItemStart() {.dirty.} =
   case p.level.kind
@@ -378,12 +377,12 @@ template handleBlockItemStart() {.dirty.} =
         "Unexpected token (expected block sequence indicator)")
   of fplMapKey:
     p.ancestry.add(p.level)
-    p.level = FastParseLevel(kind: fplUnknown, indentation: indentation)
+    p.level = FastParseLevel(kind: fplUnknown, indentation: p.indentation)
   of fplMapValue:
-    yieldEmptyScalar()
+    yield emptyScalar(p)
     p.level.kind = fplMapKey
     p.ancestry.add(p.level)
-    p.level = FastParseLevel(kind: fplUnknown, indentation: indentation)
+    p.level = FastParseLevel(kind: fplUnknown, indentation: p.indentation)
   of fplScalar, fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail()
 
 template handleFlowItemStart() {.dirty.} =
@@ -793,7 +792,7 @@ template handleFlowPlainScalar() {.dirty.} =
           p.content.addMultiple(' ', p.newlines - 1)
           p.newlines = 0
         p.plainScalar(cFlow)
-  yieldShallowScalar(p.content)
+  yield currentScalar(p)
   handleObjectEnd(fpFlowAfterObject)
 
 proc tagHandle(p: YamlParser, shorthandEnd: var int)
@@ -909,7 +908,7 @@ template blockScalarHeader() {.dirty.} =
       chomp = ctStrip
     of '1'..'9':
       if p.level.indentation != UnknownIndentation:
-        raise p.lexer.generateError("Only one indentation indicator is allowed")
+        raise p.lexer.generateError("Only one p.indentation indicator is allowed")
       p.level.indentation = p.ancestry[p.ancestry.high].indentation +
           ord(p.lexer.buf[p.lexer.bufpos]) - ord('\x30')
     of spaceOrLineEnd: break
@@ -925,22 +924,22 @@ template blockScalarHeader() {.dirty.} =
 
 template blockScalarLine() {.dirty.} =
   debug("lex: blockScalarLine")
-  if indentation < p.level.indentation:
+  if p.indentation < p.level.indentation:
     if p.lexer.buf[p.lexer.bufpos] == '#':
       # skip all following comment lines
-      while indentation > p.ancestry[p.ancestry.high].indentation:
+      while p.indentation > p.ancestry[p.ancestry.high].indentation:
         p.lineEnding()
         handleLineEnd(true)
         p.newlines.inc(-1)
         p.lexer.skipIndentation()
-        indentation = p.lexer.getColNumber(p.lexer.bufpos)
-      if indentation > p.ancestry[p.ancestry.high].indentation:
+        p.indentation = p.lexer.getColNumber(p.lexer.bufpos)
+      if p.indentation > p.ancestry[p.ancestry.high].indentation:
         raise p.lexer.generateError(
             "Invalid content in block scalar after comments")
       closeMoreIndentedLevels()
     else:
       raise p.lexer.generateError(
-          "Invalid indentation (expected indentation of at least " &
+          "Invalid p.indentation (expected p.indentation of at least " &
           $p.level.indentation & " spaces)")    
   else:
     if p.level.indentation == UnknownIndentation:
@@ -948,13 +947,13 @@ template blockScalarLine() {.dirty.} =
         handleLineEnd(true)
         continue
       else:
-        p.level.indentation = indentation
+        p.level.indentation = p.indentation
         p.content.addMultiple('\l', p.newlines)
-    elif indentation > p.level.indentation or
+    elif p.indentation > p.level.indentation or
         p.lexer.buf[p.lexer.bufpos] == '\t':
       p.content.addMultiple('\l', p.newlines)
       recentWasMoreIndented = true
-      p.content.addMultiple(' ', indentation - p.level.indentation)
+      p.content.addMultiple(' ', p.indentation - p.level.indentation)
     elif scalarType == stFolded:
       if recentWasMoreIndented:
         recentWasMoreIndented = false
@@ -977,7 +976,6 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
   var backend = iterator(): YamlStreamEvent =
     var
       state = fpInitial
-      indentation: int
       flowdepth: int = 0
       explicitFlowKey: bool
       scalarType: ScalarType
@@ -985,7 +983,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
       chomp: ChompType
     
     p.lexer.open(s)
-    initDocValues()
+    p.initDocValues()
     
     while true:
       case state
@@ -1026,7 +1024,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             handleLineEnd(false)
         of ' ', '\t':
           if not p.consumeLineIfEmpty(p.newlines):
-            indentation = p.lexer.getColNumber(p.lexer.bufpos)
+            p.indentation = p.lexer.getColNumber(p.lexer.bufpos)
             yield startDocEvent()
             state = fpBlockObjectStart
         of '\l': p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos)
@@ -1067,7 +1065,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           p.newlines.inc()
           state = fpBlockObjectStart
         else:
-          yieldShallowScalar(p.content)
+          yield currentScalar(p)
           handleObjectEnd(fpBlockAfterObject)
       of fpBlockAfterObject:
         debug("state: blockAfterObject")
@@ -1084,8 +1082,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos)
         of ':':
           case p.level.kind
-          of fplUnknown:
-            handleObjectStart(yamlStartMap)
+          of fplUnknown: yield p.objectStart(yamlStartMap)
           of fplMapKey:
             yield scalarEvent("", yTagQuestionMark, yAnchorNone)
             p.level.kind = fplMapValue
@@ -1105,7 +1102,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           of fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail()
           p.lexer.bufpos.inc()
           p.lexer.skipWhitespace()
-          indentation = p.lexer.getColNumber(p.lexer.bufpos)
+          p.indentation = p.lexer.getColNumber(p.lexer.bufpos)
           state = fpBlockObjectStart
         of '#':
           p.lineEnding()
@@ -1118,14 +1115,14 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
       of fpBlockObjectStart:
         debug("state: blockObjectStart")
         p.lexer.skipIndentation()
-        indentation = p.lexer.getColNumber(p.lexer.bufpos)
-        if indentation == 0:
+        p.indentation = p.lexer.getColNumber(p.lexer.bufpos)
+        if p.indentation == 0:
           var success: bool
           p.lexer.directivesEndMarker(success)
           if success:
             p.lexer.bufpos.inc(3)
             closeEverything()
-            initDocValues()
+            p.initDocValues()
             yield startDocEvent()
             continue
           p.lexer.documentEndMarker(success)
@@ -1136,7 +1133,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             handleLineEnd(false)
             state = fpAfterDocument
             continue
-        if indentation <= p.ancestry[p.ancestry.high].indentation:
+        if p.indentation <= p.ancestry[p.ancestry.high].indentation:
           if p.lexer.buf[p.lexer.bufpos] in lineEnd:
             handleLineEnd(true)
             continue
@@ -1148,7 +1145,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
               p.lexer.isPlainSafe(p.lexer.bufpos + 1, cBlock):
             closeMoreIndentedLevels(true)
           else: closeMoreIndentedLevels()
-        elif indentation <= p.level.indentation and
+        elif p.indentation <= p.level.indentation and
             p.lexer.buf[p.lexer.bufpos] in lineEnd:
           handleLineEnd(true)
           continue
@@ -1180,7 +1177,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           p.startToken()
           p.singleQuotedScalar()
           if p.tag == yTagQuestionMark: p.tag = yTagExclamationMark
-          yieldShallowScalar(p.content)
+          yield currentScalar(p)
           handleObjectEnd(fpBlockAfterObject)
         of '"':
           handleBlockItemStart()
@@ -1188,7 +1185,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           p.startToken()
           p.doubleQuotedScalar()
           if p.tag == yTagQuestionMark: p.tag = yTagExclamationMark
-          yieldShallowScalar(p.content)
+          yield currentScalar(p)
           handleObjectEnd(fpBlockAfterObject)
         of '|', '>':
           blockScalarHeader()
@@ -1276,7 +1273,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             p.lexer.bufpos.inc(3)
             yield endDocEvent()
             discard p.ancestry.pop()
-            initDocValues()
+            p.initDocValues()
             yield startDocEvent()
             state = fpBlockObjectStart
           else:
@@ -1316,7 +1313,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
             p.lineEnding()
             handleLineEnd(false)
           else:
-            initDocValues()
+            p.initDocValues()
             yield startDocEvent()
             state = fpBlockObjectStart
         of '#':
@@ -1324,13 +1321,13 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           handleLineEnd(false)
         of '\t', ' ':
           if not p.consumeLineIfEmpty(p.newlines):
-            indentation = p.lexer.getColNumber(p.lexer.bufpos)
-            initDocValues()
+            p.indentation = p.lexer.getColNumber(p.lexer.bufpos)
+            p.initDocValues()
             yield startDocEvent()
             state = fpBlockObjectStart
         of EndOfFile: break
         else:
-          initDocValues()
+          p.initDocValues()
           state = fpInitial
       of fpFlow:
         debug("state: flow")
@@ -1338,13 +1335,13 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
         case p.lexer.buf[p.lexer.bufpos]
         of '{':
           handleFlowItemStart()
-          handleObjectStart(yamlStartMap)
+          yield p.objectStart(yamlStartMap)
           flowdepth.inc()
           p.lexer.bufpos.inc()
           explicitFlowKey = false
         of '[':
           handleFlowItemStart()
-          handleObjectStart(yamlStartSeq)
+          yield p.objectStart(yamlStartSeq)
           flowdepth.inc()
           p.lexer.bufpos.inc()
         of '}':
@@ -1352,12 +1349,12 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           p.level = p.ancestry.pop()
           case p.level.kind
           of fplMapValue:
-            yieldEmptyScalar()
+            yield emptyScalar(p)
             p.level.kind = fplMapKey
           of fplMapKey:
             if p.tag != yTagQuestionMark or p.anchor != yAnchorNone or
                 explicitFlowKey:
-              yieldEmptyScalar()
+              yield emptyScalar(p)
               yield scalarEvent("", yTagQuestionMark, yAnchorNone)
           of fplSequence:
             p.startToken()
@@ -1374,9 +1371,9 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           case p.level.kind
           of fplSequence:
             if p.tag != yTagQuestionMark or p.anchor != yAnchorNone:
-              yieldEmptyScalar()
+              yield emptyScalar(p)
           of fplSinglePairValue:
-            yieldEmptyScalar()
+            yield emptyScalar(p)
             p.level = p.ancestry.pop()
             yield endMapEvent()
             assert(p.level.kind == fplSequence)
@@ -1390,17 +1387,17 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           assert(p.level.kind == fplUnknown)
           p.level = p.ancestry.pop()
           case p.level.kind
-          of fplSequence: yieldEmptyScalar()
+          of fplSequence: yield emptyScalar(p)
           of fplMapValue:
-            yieldEmptyScalar()
+            yield emptyScalar(p)
             p.level.kind = fplMapKey
             explicitFlowKey = false
           of fplMapKey:
-            yieldEmptyScalar
+            yield emptyScalar(p)
             yield scalarEvent("", yTagQuestionMark, yAnchorNone)
             explicitFlowKey = false
           of fplSinglePairValue:
-            yieldEmptyScalar()
+            yield emptyScalar(p)
             p.level = p.ancestry.pop()
             yield endMapEvent()
             assert(p.level.kind == fplSequence)
@@ -1419,11 +1416,11 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
               yield startMapEvent(p.tag, p.anchor)
               debug("started single-pair map at " &
                   (if p.level.indentation == UnknownIndentation:
-                   $indentation else: $p.level.indentation))
+                   $p.indentation else: $p.level.indentation))
               p.tag = yTagQuestionMark
               p.anchor = yAnchorNone
               if p.level.indentation == UnknownIndentation:
-                p.level.indentation = indentation
+                p.level.indentation = p.indentation
               p.ancestry.add(p.level)
               p.level = initLevel(fplSinglePairValue)
               yield scalarEvent("")
@@ -1431,10 +1428,10 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
               p.startToken()
               raise p.generateError("Unexpected token (expected ',')")
             of fplMapKey:
-              yieldEmptyScalar()
+              yield emptyScalar(p)
               p.level.kind = fplMapValue
             of fplSinglePairKey:
-              yieldEmptyScalar()
+              yield emptyScalar(p)
               p.level.kind = fplSinglePairValue
             of fplUnknown, fplScalar, fplDocument: debugFail()
             p.ancestry.add(p.level)
@@ -1446,7 +1443,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           p.startToken()
           p.singleQuotedScalar()
           if p.tag == yTagQuestionMark: p.tag = yTagExclamationMark
-          yieldShallowScalar(p.content)
+          yield currentScalar(p)
           handleObjectEnd(fpFlowAfterObject)
         of '"':
           handleFlowItemStart()
@@ -1454,7 +1451,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           p.startToken()
           p.doubleQuotedScalar()
           if p.tag == yTagQuestionMark: p.tag = yTagExclamationMark
-          yieldShallowScalar(p.content)
+          yield currentScalar(p)
           handleObjectEnd(fpFlowAfterObject)
         of '!':
           handleFlowItemStart()
@@ -1475,7 +1472,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
           elif p.level.kind == fplUnknown:
             case p.ancestry[p.ancestry.high].kind
             of fplMapKey, fplMapValue, fplDocument: discard
-            of fplSequence: handleObjectStart(yamlStartMap, true)
+            of fplSequence: yield p.objectStart(yamlStartMap, true)
             else:
               p.startToken()
               raise p.generateError("Unexpected token")
diff --git a/yaml.nim b/yaml.nim
index 36b0462..c9e093a 100644
--- a/yaml.nim
+++ b/yaml.nim
@@ -172,6 +172,7 @@ type
     anchors: Table[string, AnchorId]
     nextAnchorId: AnchorId
     newlines: int
+    indentation: int
         
   PresentationStyle* = enum
     ## Different styles for YAML character stream output.