Made token output conform to yaml test suite

* Added -d:yamlScalarRepInd switch
 * Modified `$` on stream items to use yaml test suite format

config.nims

@@ -72,6 +72,7 @@ task clean, "Remove all generated files":
 
 task server, "Compile server daemon":
   --d:release
+  --d:yamlScalarRepInd
   setCommand "c", "server/server"
 
 task testSuiteEvents, "Compile the testSuiteEvents tool":

server/server.nim

@@ -32,10 +32,11 @@ routes:
       of "block": style = psBlockOnly
       of "tokens":
         var
-          output = ""
+          output = "+STR\n"
           parser = newYamlParser()
           events = parser.parse(newStringStream(@"input"))
         for event in events: output.add($event & "\n")
+        output &= "-STR"
         resultNode["code"] = %0
         resultNode["output"] = %output
         msg = resultNode.pretty

yaml/parser.nim

@@ -143,7 +143,10 @@ proc initLevel(k: FastParseLevelKind): FastParseLevel {.raises: [], inline.} =
   FastParseLevel(kind: k, indentation: UnknownIndentation)
 
 proc emptyScalar(c: ParserContext): YamlStreamEvent {.raises: [], inline.} =
-  result = scalarEvent("", c.tag, c.anchor)
+  when defined(yamlScalarRepInd):
+    result = scalarEvent("", c.tag, c.anchor, srPlain)
+  else:
+    result = scalarEvent("", c.tag, c.anchor)
   c.tag = yTagQuestionMark
   c.anchor = yAnchorNone
 
@@ -244,6 +247,12 @@ proc handlePossibleMapStart(c: ParserContext, e: var YamlStreamEvent,
       result = true
     c.level.indentation = c.lex.indentation
 
+template implicitScalar(): YamlStreamEvent =
+  when defined(yamlScalarRepInd):
+    scalarEvent("", yTagQuestionMark, yAnchorNone, srPlain)
+  else:
+    scalarEvent("", yTagQuestionMark, yAnchorNone)
+
 proc handleMapKeyIndicator(c: ParserContext, e: var YamlStreamEvent): bool =
   result = false
   case c.level.kind
@@ -255,7 +264,7 @@ proc handleMapKeyIndicator(c: ParserContext, e: var YamlStreamEvent): bool =
       raise c.generateError("Invalid p.indentation of map key indicator " &
           "(expected" & $c.level.indentation & ", got " & $c.lex.indentation &
           ")")
-    e = scalarEvent("", yTagQuestionMark, yAnchorNone)
+    e = implicitScalar()
     result = true
     c.level.kind = fplMapKey
     c.ancestry.add(c.level)
@@ -456,7 +465,7 @@ proc handleMapValueIndicator(c: ParserContext, e: var YamlStreamEvent): bool =
   of fplMapKey:
     if c.level.indentation != c.lex.indentation:
       raise c.generateError("Invalid p.indentation of map key indicator")
-    e = scalarEvent("", yTagQuestionMark, yAnchorNone)
+    e = implicitScalar()
     result = true
     c.level.kind = fplMapValue
     c.ancestry.add(c.level)
@@ -640,6 +649,12 @@ parserState blockObjectStart:
 parserState scalarEnd:
   if c.tag == yTagQuestionMark: c.tag = yTagExclamationMark
   c.currentScalar(e)
+  when defined(yamlScalarRepInd):
+    case c.lex.scalarKind
+    of skSingleQuoted: e.scalarRep = srSingleQuoted
+    of skDoubleQuoted: e.scalarRep = srDoubleQuoted
+    of skLiteral: e.scalarRep = srLiteral
+    of skFolded: e.scalarRep = srFolded
   result = true
   state = objectEnd
   stored = blockAfterObject
@@ -671,7 +686,7 @@ parserState blockAfterObject:
       e = c.objectStart(yamlStartMap)
       result = true
     of fplMapKey:
-      e = scalarEvent("", yTagQuestionMark, yAnchorNone)
+      e = implicitScalar()
       result = true
       c.level.kind = fplMapValue
       c.ancestry.add(c.level)
@@ -771,7 +786,7 @@ parserState closeMoreIndentedLevels:
     state = stored
 
 parserState emitEmptyScalar:
-  e = scalarEvent("", yTagQuestionMark, yAnchorNone)
+  e = implicitScalar()
   result = true
   state = stored
 
@@ -989,7 +1004,7 @@ parserState flowAfterObject:
     case c.level.kind
     of fplSequence: discard
     of fplMapValue:
-      e = scalarEvent("", yTagQuestionMark, yAnchorNone)
+      e = implicitScalar()
       result = true
       c.level.kind = fplMapKey
       c.explicitFlowKey = false

yaml/private/lex.nim

@@ -9,6 +9,10 @@ when defined(yamlDebug):
   import terminal
   export terminal
 
+when defined(yamlScalarRepInd):
+  type ScalarKind* = enum
+    skSingleQuoted, skDoubleQuoted, skLiteral, skFolded
+
 type
   StringSource* = object
     src: string
@@ -30,6 +34,9 @@ type
     indentation*: int
     # ltTagHandle
     shorthandEnd*: int
+    when defined(yamlScalarRepInd):
+      # ltQuotedScalar, ltBlockScalarHeader
+      scalarKind*: ScalarKind
 
     # may be modified from outside; will be consumed at plain scalar starts
     newlines*: int
@@ -585,6 +592,7 @@ proc processQuotedWhitespace[T](lex: YamlLexer, newlines: var int) =
 proc singleQuotedScalar[T](lex: YamlLexer) =
   debug("lex: singleQuotedScalar")
   startToken[T](lex)
+  when defined(yamlScalarRepInd): lex.scalarKind = skSingleQuoted
   lex.advance(T)
   while true:
     case lex.c
@@ -627,6 +635,7 @@ proc unicodeSequence[T](lex: YamlLexer, length: int) =
 proc doubleQuotedScalar[T](lex: YamlLexer) =
   debug("lex: doubleQuotedScalar")
   startToken[T](lex)
+  when defined(yamlScalarRepInd): lex.scalarKind = skDoubleQuoted
   lex.advance(T)
   while true:
     case lex.c
@@ -778,6 +787,8 @@ proc blockScalarHeader[T](lex: YamlLexer): bool =
   lex.chomp = ctClip
   lex.blockScalarIndent = UnknownIndentation
   lex.folded = lex.c == '>'
+  when defined(yamlScalarRepInd):
+    lex.scalarKind = if lex.folded: skFolded else: skLiteral
   startToken[T](lex)
   while true:
     lex.advance(T)

yaml/stream.nim

@@ -15,6 +15,10 @@
 import hashes
 import private/internal, taglib
 
+when defined(yamlScalarRepInd):
+  type ScalarRepresentationIndicator* = enum
+    srPlain, srSingleQuoted, srDoubleQuoted, srLiteral, srFolded
+
 type
   AnchorId* = distinct int ## \
     ## An ``AnchorId`` identifies an anchor in the current document. It
@@ -52,6 +56,8 @@ type
       scalarAnchor* : AnchorId
       scalarTag*    : TagId
       scalarContent*: string # may not be nil (but empty)
+      when defined(yamlScalarRepInd):
+        scalarRep*  : ScalarRepresentationIndicator
     of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard
     of yamlAlias:
       aliasTarget* : AnchorId
@@ -97,8 +103,7 @@ proc noLastContext(s: YamlStream, line, column: var int,
 
 proc basicInit*(s: YamlStream, lastTokenContextImpl:
     proc(s: YamlStream, line, column: var int, lineContent: var string): bool
-    {.raises: [].} = noLastContext)
-    {.raises: [].} =
+    {.raises: [].} = noLastContext) {.raises: [].} =
   ## initialize basic values of the YamlStream. Call this in your constructor
   ## if you subclass YamlStream.
   s.peeked = false
@@ -240,25 +245,46 @@ proc `==`*(left: YamlStreamEvent, right: YamlStreamEvent): bool {.raises: [].} =
              left.scalarContent == right.scalarContent
   of yamlAlias: result = left.aliasTarget == right.aliasTarget
 
+proc renderAttrs(tag: TagId, anchor: AnchorId): string =
+  result = ""
+  case tag
+  of yTagQuestionmark: discard
+  of yTagExclamationmark: result &= " !"
+  else: result &= " !<" & $tag & ">"
+  if anchor != yAnchorNone: result &= " &" & $anchor
+
+proc yamlEscape(s: string): string =
+  result = ""
+  for c in s:
+    case c
+    of '\l': result.add("\\n")
+    of '\c': result.add("\\c")
+    of '\\': result.add("\\\\")
+    else: result.add(c)
+
 proc `$`*(event: YamlStreamEvent): string {.raises: [].} =
-  ## outputs a human-readable string describing the given event
+  ## outputs a human-readable string describing the given event.
+  ## This string is compatible to the format used in the yaml test suite.
   result = $event.kind & '('
   case event.kind
-  of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard
-  of yamlStartMap:
-    result &= "tag=" & $event.mapTag
-    if event.mapAnchor != yAnchorNone: result &= ", anchor=" & $event.mapAnchor
-  of yamlStartSeq:
-    result &= "tag=" & $event.seqTag
-    if event.seqAnchor != yAnchorNone: result &= ", anchor=" & $event.seqAnchor
+  of yamlEndMap: result = "-MAP"
+  of yamlEndSeq: result = "-SEQ"
+  of yamlStartDoc: result = "+DOC"
+  of yamlEndDoc: result = "-DOC"
+  of yamlStartMap: result = "+MAP" & renderAttrs(event.mapTag, event.mapAnchor)
+  of yamlStartSeq: result = "+SEQ" & renderAttrs(event.seqTag, event.seqAnchor)
   of yamlScalar:
-    result &= "tag=" & $event.scalarTag
-    if event.scalarAnchor != yAnchorNone:
-      result &= ", anchor=" & $event.scalarAnchor
-    result &= ", content=\"" & event.scalarContent & '\"'
-  of yamlAlias:
-    result &= "aliasTarget=" & $event.aliasTarget
-  result &= ")"
+    result = "=VAL" & renderAttrs(event.scalarTag, event.scalarAnchor)
+    when defined(yamlScalarRepInd):
+      case event.scalarRep
+      of srPlain: result &= " :"
+      of srSingleQuoted: result &= " \'"
+      of srDoubleQuoted: result &= " \""
+      of srLiteral: result &= " |"
+      of srFolded: result &= " >"
+    else: result &= " :"
+    result &= yamlEscape(event.scalarContent)
+  of yamlAlias: result = "=ALI *" & $event.aliasTarget
 
 proc tag*(event: YamlStreamEvent): TagId {.raises: [FieldError].} =
   ## returns the tag of the given event
@@ -294,11 +320,21 @@ proc endSeqEvent*(): YamlStreamEvent {.inline, raises: [].} =
   ## creates a new event that marks the end of a YAML sequence
   result = YamlStreamEvent(kind: yamlEndSeq)
 
-proc scalarEvent*(content: string = "", tag: TagId = yTagQuestionMark,
-    anchor: AnchorId = yAnchorNone): YamlStreamEvent {.inline, raises: [].} =
-  ## creates a new event that represents a YAML scalar
-  result = YamlStreamEvent(kind: yamlScalar, scalarTag: tag,
-                           scalarAnchor: anchor, scalarContent: content)
+when defined(yamlScalarRepInd):
+  proc scalarEvent*(content: string = "", tag: TagId = yTagQuestionMark,
+      anchor: AnchorId = yAnchorNone,
+      scalarRep: ScalarRepresentationIndicator = srPlain):
+      YamlStreamEvent {.inline, raises: [].} =
+    ## creates a new event that represents a YAML scalar
+    result = YamlStreamEvent(kind: yamlScalar, scalarTag: tag,
+                            scalarAnchor: anchor, scalarContent: content,
+                            scalarRep: scalarRep)
+else:
+  proc scalarEvent*(content: string = "", tag: TagId = yTagQuestionMark,
+      anchor: AnchorId = yAnchorNone): YamlStreamEvent {.inline, raises: [].} =
+    ## creates a new event that represents a YAML scalar
+    result = YamlStreamEvent(kind: yamlScalar, scalarTag: tag,
+                            scalarAnchor: anchor, scalarContent: content)
 
 proc aliasEvent*(anchor: AnchorId): YamlStreamEvent {.inline, raises: [].} =
   ## creates a new event that represents a YAML alias