Support additional boolean scalars

* on, off, yes, no, literals with leading or complete capital letters
 * added documentation

src/private/json.nim

@@ -14,13 +14,20 @@ proc jsonFromScalar(content: string, tag: TagId,
     of tagExclamationMark, tagString:
         mappedType = yTypeString
     of tagBoolean:
-        mappedType = yTypeBoolean
+        case typeHint
+        of yTypeBoolTrue:
+            mappedType = yTypeBoolTrue
+        of yTypeBoolFalse:
+            mappedType = yTypeBoolFalse
+        else:
+            raise newException(ValueError, "Invalid boolean value: " & content)
     of tagInteger:
         mappedType = yTypeInteger
     of tagNull:
         mappedType = yTypeNull
     of tagFloat:
         mappedType = yTypeFloat
+        ## TODO: NaN, inf
     else:
         mappedType = yTypeUnknown
     
@@ -31,9 +38,12 @@ proc jsonFromScalar(content: string, tag: TagId,
     of yTypeFloat:
         result.kind = JFloat
         result.fnum = parseFloat(content)
-    of yTypeBoolean:
+    of yTypeBoolTrue:
         result.kind = JBool
-        result.bval = parseBool(content)
+        result.bval = true
+    of yTypeBoolFalse:
+        result.kind = JBool
+        result.bval = false
     of yTypeNull:
         result.kind = JNull
     else:

src/private/lexer.nim

@@ -58,9 +58,28 @@ type
         ylAnchor, ylAlias
     
     YamlTypeHintState = enum
-        ythInitial, ythN, ythNU, ythNUL, ythNULL, ythF, ythFA, ythFAL, ythFALS,
+        ythInitial,
-        ythFALSE, ythT, ythTR, ythTRU, ythTRUE, ythMinus, yth0, ythInt, 
+        ythF, ythFA, ythFAL, ythFALS, ythFALSE,
-        ythDecimal, ythNumE, ythNumEPlusMinus, ythExponent, ythNone
+        ythN, ythNU, ythNUL, ythNULL,
+              ythNO,
+        ythO, ythON,
+              ythOF, ythOFF,
+        ythT, ythTR, ythTRU, ythTRUE,
+        ythY, ythYE, ythYES,
+        
+        ythPoint, ythPointI, ythPointIN, ythPointINF,
+                  ythPointN, ythPointNA, ythPointNAN,
+        
+        ythLowerFA, ythLowerFAL, ythLowerFALS,
+        ythLowerNU, ythLowerNUL,
+        ythLowerOF,
+        ythLowerTR, ythLowerTRU,
+        ythLowerYE,
+        
+        ythPointLowerIN, ythPointLowerN, ythPointLowerNA,
+        
+        ythMinus, yth0, ythInt, ythDecimal, ythNumE, ythNumEPlusMinus,
+        ythExponent, ythNone
     
     YamlLexer = object of BaseLexer
         indentations: seq[int]
@@ -157,12 +176,18 @@ template yieldScalarPart() {.dirty.} =
     case typeHintState
     of ythNULL:
         my.typeHint = yTypeNull
-    of ythTRUE, ythFALSE:
+    of ythTRUE, ythON, ythYES, ythY:
-        my.typeHint = yTypeBoolean
+        my.typeHint = yTypeBoolTrue
+    of ythFALSE, ythOFF, ythNO, ythN:
+        my.typeHint = yTypeBoolFalse
     of ythInt, yth0:
         my.typeHint = yTypeInteger
     of ythDecimal, ythExponent:
         my.typeHint = yTypeFloat
+    of ythPointINF:
+        my.typeHint = yTypeFloatInf
+    of ythPointNAN:
+        my.typeHint = yTypeFloatNaN
     else:
         my.typeHint = yTypeUnknown
     
@@ -194,136 +219,132 @@ template handleLF() {.dirty.} =
 template `or`(r: Rune, i: int): Rune =
     cast[Rune](cast[int](r) or i)
 
+macro typeHintStateMachine(c: untyped, content: untyped): stmt =
+    assert content.kind == nnkStmtList
+    result = newNimNode(nnkCaseStmt, content).add(copyNimNode(c))
+    for branch in content.children:
+        assert branch.kind == nnkOfBranch
+        var 
+            charBranch = newNimNode(nnkOfBranch, branch)
+            i = 0
+            stateBranches = newNimNode(nnkCaseStmt, branch).add(
+                    newIdentNode("typeHintState"))
+        while branch[i].kind != nnkStmtList:
+            charBranch.add(copyNimTree(branch[i]))
+            inc(i)
+        for rule in branch[i].children:
+            assert rule.kind == nnkInfix
+            assert ($rule[0].ident == "=>")
+            var stateBranch = newNimNode(nnkOfBranch, rule)
+            case rule[1].kind
+            of nnkBracket:
+                for item in rule[1].children:
+                    stateBranch.add(item)
+            of nnkIdent:
+                stateBranch.add(rule[1])
+            else:
+                assert false
+            if rule[2].kind == nnkNilLit:
+                stateBranch.add(newStmtList(newNimNode(nnkDiscardStmt).add(
+                        newEmptyNode())))
+            else:
+                stateBranch.add(newStmtList(newAssignment(
+                        newIdentNode("typeHintState"), copyNimTree(rule[2]))))
+            stateBranches.add(stateBranch)
+        stateBranches.add(newNimNode(nnkElse).add(newStmtList(newAssignment(
+                newIdentNode("typeHintState"), newIdentNode("ythNone")),
+                newAssignment(newIdentNode("state"),
+                newIdentNode("ylPlainScalarNone")))))
+        charBranch.add(newStmtList(stateBranches))
+        result.add(charBranch)
+    result.add(newNimNode(nnkElse).add(newStmtList(newAssignment(
+                newIdentNode("typeHintState"), newIdentNode("ythNone")),
+                newAssignment(newIdentNode("state"),
+                newIdentNode("ylPlainScalarNone")))))
+
 template advanceTypeHint(ch: char) {.dirty.} =
-    case ch
+    typeHintStateMachine ch:
     of '.':
-        case typeHintState
+        [yth0, ythInt]         => ythDecimal
-        of yth0, ythInt:
+        [ythInitial, ythMinus] => ythPoint
-            typeHintState = ythDecimal
+    of '+': ythNumE => ythNumEPlusMinus
-        else:
-            typeHintState = ythNone
-            state = ylPlainScalarNone
-    of '+':
-        case typeHintState
-        of ythNumE:
-            typeHintState = ythNumEPlusMinus
-        else:
-            typeHintState = ythNone
-            state = ylPlainScalarNone
     of '-':
-        case typeHintState
+        ythInitial => ythMinus
-        of ythInitial:
+        ythNumE    => ythNumEPlusMinus
-            typeHintState = ythMinus
-        of ythNumE:
-            typeHintState = ythNumEPlusMinus
-        else:
-            typeHintState = ythNone
-            state = ylPlainScalarNone
     of '0':
-        case typeHintState
+        [ythInitial, ythMinus]      => yth0
-        of ythInitial, ythMinus:
+        [ythNumE, ythNumEPlusMinus] => ythExponent
-            typeHintState = yth0
-        of ythNumE, ythNumEPlusMinus:
-            typeHintState = ythExponent
-        of ythInt, ythDecimal, ythExponent:
-            discard
-        else:
-            typeHintState = ythNone
-            state = ylPlainScalarNone
     of '1'..'9':
-        case typeHintState
+        [ythInitial, ythMinus]            => ythInt
-        of ythInitial, ythMinus:
+        [ythNumE, ythNumEPlusMinus]       => ythExponent
-            typeHintState = ythInt
+        [ythInt, ythDecimal, ythExponent] => nil
-        of ythNumE, ythNumEPlusMinus:
-            typeHintState = ythExponent
-        of ythInt, ythDecimal, ythExponent:
-            discard
-        else:
-            typeHintState = ythNone
-            state = ylPlainScalarNone
     of 'a':
-        case typeHintState
+        ythF           => ythLowerFA
-        of ythF:
+        ythPointN      => ythPointNA
-            typeHintState = ythFA
+        ythPointLowerN => ythPointLowerNA
-        else:
+    of 'A':
-            typeHintState = ythNone
+        ythF      => ythFA
-            state = ylPlainScalarNone
+        ythPointN => ythPointNA
     of 'e':
-        case typeHintState
+        [yth0, ythInt, ythDecimal] => ythNumE
-        of yth0, ythInt, ythDecimal:
+        ythLowerFALS => ythFALSE
-            typeHintState = ythNumE
+        ythLowerTRU  => ythTRUE
-        of ythTRU:
+        ythY         => ythLowerYE
-            typeHintState = ythTRUE
-        of ythFALS:
-            typeHintState = ythFALSE
-        else:
-            typeHintState = ythNone
-            state = ylPlainScalarNone
     of 'E':
-        case typeHintState
+        [yth0, ythInt, ythDecimal] => ythNumE
-        of yth0, ythInt, ythDecimal:
+        ythFALS => ythFALSE
-            typeHintState = ythNumE
+        ythTRU  => ythTRUE
-        else:
+        ythY    => ythYE
-            typeHintState = ythNone
-            state = ylPlainScalarNone
     of 'f':
-        case typeHintState
+        ythInitial      => ythF
-        of ythInitial:
+        ythO            => ythLowerOF
-            typeHintState = ythF
+        ythLowerOF      => ythOFF
-        else:
+        ythPointLowerIN => ythPointINF
-            typeHintState = ythNone
+    of 'F':
-            state = ylPlainScalarNone
+        ythInitial => ythF
+        ythO       => ythOF
+        ythOF      => ythOFF
+        ythPointIN => ythPointINF
+    of 'i', 'I': ythPoint => ythPointI
     of 'l':
-        case typeHintState
+        ythLowerNU  => ythLowerNUL
-        of ythNU:
+        ythLowerNUL => ythNULL
-            typeHintState = ythNUL
+        ythLowerFA  => ythLowerFAL
-        of ythNUL:
+    of 'L':
-            typeHintState = ythNULL
+        ythNU  => ythNUL
-        of ythFA:
+        ythNUL => ythNULL
-            typeHintState = ythFAL
+        ythFA  => ythFAL
-        else:
-            typeHintState = ythNone
-            state = ylPlainScalarNone
     of 'n':
-        case typeHintState
+        ythInitial      => ythN
-        of ythInitial:
+        ythO            => ythON
-            typeHintState = ythN
+        ythPoint        => ythPointLowerN
-        else:
+        ythPointI       => ythPointLowerIN
-            typeHintState = ythNone
+        ythPointLowerNA => ythPointNAN
-            state = ylPlainScalarNone
+    of 'N':
-    of 'r':
+        ythInitial => ythN
-        case typeHintState
+        ythO       => ythON
-        of ythT:
+        ythPoint   => ythPointN
-            typeHintState = ythTR
+        ythPointI  => ythPointIN
-        else:
+        ythPointNA => ythPointNAN
-            typeHintState = ythNone
+    of 'o', 'O':
-            state = ylPlainScalarNone
+        ythInitial => ythO
+        ythN       => ythNO
+    of 'r': ythT => ythLowerTR
+    of 'R': ythT => ythTR
     of 's':
-        case typeHintState
+        ythLowerFAL => ythLowerFALS
-        of ythFAL:
+        ythLowerYE  => ythYES
-            typeHintState = ythFALS
+    of 'S':
-        else:
+        ythFAL => ythFALS
-            typeHintState = ythNone
+        ythYE  => ythYES
-            state = ylPlainScalarNone
+    of 't', 'T': ythInitial => ythT
-    of 't':
-        case typeHintState
-        of ythInitial:
-            typeHintState = ythT
-        else:
-            typeHintState = ythNone
-            state = ylPlainScalarNone
     of 'u':
-        case typeHintState
+        ythN       => ythLowerNU
-        of ythN:
+        ythLowerTR => ythLowerTRU
-            typeHintState = ythNU
+    of 'U':
-        of ythTR:
+        ythN  => ythNU
-            typeHintState = ythTRU
+        ythTR => ythTRU
-        else:
+    of 'y', 'Y': ythInitial => ythY
-            typeHintState = ythNone
-            state = ylPlainScalarNone
-    else:
-        typeHintState = ythNone
-        state = ylPlainScalarNone
 
 iterator tokens(my: var YamlLexer): YamlLexerToken {.closure.} =
     var

src/yaml.nim

@@ -1,9 +1,41 @@
-import streams, unicode, lexbase, tables, strutils, json, hashes, queues
+## This module provides facilities to generate and interpret
+## `YAML <http://yaml.org>`_ character streams. All primitive operations on
+## data objects use a ``YamlStream`` either as source or as output. Because this
+## stream is implemented as iterator, it is possible to process YAML input and
+## output sequentially, i.e. without loading the processed data structure
+## completely into RAM. This supports processing of large data structures.
+##
+## As YAML is a strict superset of `JSON <http://json.org>`_, JSON input is
+## automatically supported. Additionally, there is functionality available to
+## convert any YAML stream into JSON. While JSON is less readable than YAML,
+## this enhances interoperability with other languages.
+
+import streams, unicode, lexbase, tables, strutils, json, hashes, queues, macros
 
 type
     YamlTypeHint* = enum
-        yTypeInteger, yTypeFloat, yTypeBoolean, yTypeNull, yTypeString,
+        ## A type hint is a friendly message from the YAML lexer, telling you
-        yTypeUnknown
+        ## it thinks a scalar string probably is of a certain type. You are not
+        ## required to adhere to this information. The first matching RegEx will
+        ## be the type hint of a scalar string.
+        ##
+        ## ================== =========================
+        ## Name               RegEx
+        ## ================== =========================
+        ## ``yTypeInteger``   ``0 | -? [1-9] [0-9]*``
+        ## ``yTypeFloat``     ``-? [1-9] ( \. [0-9]* [1-9] )? ( e [-+] [1-9] [0-9]* )?``
+        ## ``yTypeFloatInf``  ``-? \. (inf | Inf | INF)``
+        ## ``yTypeFloatNaN``  ``-? \. (nan | NaN | NAN)``
+        ## ``yTypeBoolTrue``  ``y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON``
+        ## ``yTypeBoolFalse`` ``n|N|no|No|NO|false|False|FALSE|off|Off|OFF``
+        ## ``yTypeNull``      ``~ | null | Null | NULL``
+        ## ``yTypeString``    *none*
+        ## ``yTypeUnknown``   ``*``
+        ##
+        ## The value `yTypeString` is not returned based on RegExes, but for
+        ## scalars that are quoted within the YAML input character stream.
+        yTypeInteger, yTypeFloat, yTypeFloatInf, yTypeFloatNaN, yTypeBoolTrue,
+        yTypeBoolFalse, yTypeNull, yTypeString, yTypeUnknown
     
     YamlStreamEventKind* = enum
         yamlStartDocument, yamlEndDocument, yamlStartMap, yamlEndMap,

test/lexing.nim

@@ -1,11 +1,11 @@
-import streams, unicode, lexbase
+import streams, unicode, lexbase, macros
 
 import unittest
 
 type
     YamlTypeHint* = enum
-        yTypeInteger, yTypeFloat, yTypeBoolean, yTypeNull, yTypeString,
+        yTypeInteger, yTypeFloat, yTypeFloatInf, yTypeFloatNaN, yTypeBoolTrue,
-        yTypeUnknown
+        yTypeBoolFalse, yTypeNull, yTypeString, yTypeUnknown
 include "../src/private/lexer"
 
 type BasicLexerToken = tuple[kind: YamlLexerToken, content: string,
@@ -210,13 +210,15 @@ foo:
                   t(tVerbatimTag, "tag:http://example.com/str"),
                   t(tScalarPart, "tagged"), t(tStreamEnd, nil)])
     test "Lexing: Type hints":
-        ensure("false\nnull\nunknown\n\"string\"\n-13\n42.25\n-4e+3\n5.42e78",
+        ensure("False\nnull\nYES\nunknown\n\"string\"\n-13\n42.25\n-4e+3\n5.42e78\n-.NaN",
-               [t(tLineStart, ""), t(tScalarPart, "false", yTypeBoolean),
+               [t(tLineStart, ""), t(tScalarPart, "False", yTypeBoolFalse),
                 t(tLineStart, ""), t(tScalarPart, "null", yTypeNull),
+                t(tLineStart, ""), t(tScalarPart, "YES", yTypeBoolTrue),
                 t(tLineStart, ""), t(tScalarPart, "unknown", yTypeUnknown),
                 t(tLineStart, ""), t(tScalar, "string", yTypeString),
                 t(tLineStart, ""), t(tScalarPart, "-13", yTypeInteger),
                 t(tLineStart, ""), t(tScalarPart, "42.25", yTypeFloat),
                 t(tLineStart, ""), t(tScalarPart, "-4e+3", yTypeFloat),
                 t(tLineStart, ""), t(tScalarPart, "5.42e78", yTypeFloat),
+                t(tLineStart, ""), t(tScalarPart, "-.NaN", yTypeFloatNaN),
                 t(tStreamEnd, nil)])

test/parsing.nim

@@ -128,16 +128,16 @@ suite "Parsing":
     test "Parsing: Simple Scalar":
         ensure("Scalar", startDoc(), scalar("Scalar"), endDoc())
     test "Parsing: Simple Sequence":
-        ensure("- false", startDoc(), startSequence(),
+        ensure("- off", startDoc(), startSequence(),
-               scalar("false", yTypeBoolean), endSequence(), endDoc())
+               scalar("off", yTypeBoolFalse), endSequence(), endDoc())
     test "Parsing: Simple Map":
         ensure("42: value\nkey2: -7.5", startDoc(), startMap(),
                scalar("42", yTypeInteger), scalar("value"), scalar("key2"),
                scalar("-7.5", yTypeFloat), endMap(), endDoc())
     test "Parsing: Explicit Map":
-        ensure("? null\n: value\n? true\n: value2", startDoc(), startMap(),
+        ensure("? null\n: value\n? ON\n: value2", startDoc(), startMap(),
                scalar("null", yTypeNull), scalar("value"),
-               scalar("true", yTypeBoolean), scalar("value2"),
+               scalar("ON", yTypeBoolTrue), scalar("value2"),
                endMap(), endDoc())
     test "Parsing: Mixed Map (explicit to implicit)":
         ensure("? a\n: 13\n1.5: d", startDoc(), startMap(), scalar("a"),
@@ -176,8 +176,8 @@ suite "Parsing":
         ensure("[2, b]", startDoc(), startSequence(), scalar("2", yTypeInteger),
                scalar("b"), endSequence(), endDoc())
     test "Parsing: Flow Map":
-        ensure("{a: true, 1.337: d}", startDoc(), startMap(), scalar("a"),
+        ensure("{a: Y, 1.337: d}", startDoc(), startMap(), scalar("a"),
-               scalar("true", yTypeBoolean), scalar("1.337", yTypeFloat),
+               scalar("Y", yTypeBoolTrue), scalar("1.337", yTypeFloat),
                scalar("d"), endMap(), endDoc())
     test "Parsing: Flow Sequence in Flow Sequence":
         ensure("[a, [b, c]]", startDoc(), startSequence(), scalar("a"),