Support serializing tuples. Multiple bugfixes.

* Parser: Properly support explicit keys in flow style.
 * Parser: Properly support object tags in flow style.
 * Serializer: Basic support for tuples.
 * Serializer: Properly parse int and bool types when given as
   string with explicit tag.

README.md

@@ -213,18 +213,25 @@ Output:
    - Add type hints for more scalar types
  * Parser:
    - Properly handle leading spaces in block scalars
+   - Raise exceptions instead of yielding yamlError tokens, which are a pain to
+     handle for the caller.
+   - Add an optional warning callback instead of yielding yamlWarning tokens,
+     which are a pain to handle for the caller.
+   - Oh and did I mention `yamlError` and `yamlWarning` are evil and need to
+     die.
  * Serialization:
-   - Generate local tags and use them
-   - Support for tuples
    - Support for more standard library types
    - Support for ref and ptr types
-   - Support for anchors and aliases
+   - Support for anchors and aliases (requires ref and ptr types)
-   - Support polymorphism
+   - Support polymorphism (requires ref and ptr types)
    - Support variant objects
+   - Support transient fields (i.e. fields that will not be (de-)serialized on
+     objects and tuples)
    - Make it possible for user to define a tag URI for custom types
    - Use `concept` type class `Serializable` or something
  * General:
    - Proper error handling (do not use `ValueError` for everything)
+   - Proper error handling, seriously
    - Document exceptions with `raises` pragmas in code
 
 ## License

private/lexer.nim

@@ -25,7 +25,7 @@ type
         # from here on tokens only in content
         tLineStart,
         # control characters
-        tColon, tDash, tQuestionmark, tComma, tOpeningBrace,
+        tColon, tDash, tQuestionMark, tComma, tOpeningBrace,
         tOpeningBracket, tClosingBrace, tClosingBracket, tPipe, tGreater,
         # block scalar header
         tBlockIndentationIndicator, tPlus,
@@ -887,6 +887,7 @@ iterator tokens(my: var YamlLexer): YamlLexerToken {.closure.} =
                my.content = "!"
                yield(tTagHandle)
                my.content = suffix
+               my.content.add(c)
                state = ylTagSuffix
             of ' ', '\t', EndOfFile, '\r', '\x0A':
                 let suffix = my.content[1..^1]

private/sequential.nim

@@ -11,7 +11,8 @@ type
         ypBlockAfterAlias, ypBlockAfterColon, ypBlockMultilineScalar,
         ypBlockLineEnd, ypBlockScalarHeader, ypBlockScalar, ypFlow,
         ypFlowAfterObject, ypFlowAfterTag, ypFlowAfterAnchor,
-        ypFlowAfterAnchorAndTag, ypExpectingDocumentEnd, ypAfterDirectivesEnd
+        ypFlowAfterQuestionMark, ypFlowAfterAnchorAndTag,
+        ypExpectingDocumentEnd, ypAfterDirectivesEnd
     
     DocumentLevelMode = enum
         mBlockSequenceItem, mFlowSequenceItem, mExplicitBlockMapKey,
@@ -112,7 +113,8 @@ template yieldScalar(content: string, typeHint: YamlTypeHint,
              "scalar[\"", content, "\", type=", typeHint, "]"
     if objectTag.len > 0:
         if tag.len > 0:
-            yieldError("Duplicate tag for scalar")
+            yieldError("Duplicate tag for scalar (tag=" & tag & ", objectTag=" &
+                       objectTag)
         tag = objectTag
         objectTag = ""
     yield YamlStreamEvent(kind: yamlScalar,
@@ -809,6 +811,8 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
                                           indentationColumn: -1)
                 else:
                     yieldUnexpectedToken("scalar, comma or map end")
+            of tQuestionMark:
+                state = ypFlowAfterQuestionMark
             of tComma:
                 yieldScalar("", yTypeUnknown)
                 level = ancestry.pop()
@@ -826,6 +830,10 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
                 if level.mode != mUnknown:
                     yieldUnexpectedToken()
                 level.mode = mFlowMapKey
+                if objectTag.len > 0:
+                    assert tag.len == 0
+                    tag = objectTag
+                    objectTag = ""
                 yieldStart(yamlStartMap)
                 ancestry.add(level)
                 level = DocumentLevel(mode: mUnknown, indicatorColumn: -1,
@@ -834,6 +842,10 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
                 if level.mode != mUnknown:
                     yieldUnexpectedToken()
                 level.mode = mFlowSequenceItem
+                if objectTag.len > 0:
+                    assert tag.len == 0
+                    tag = objectTag
+                    objectTag = ""
                 yieldStart(yamlStartSequence)
                 ancestry.add(level)
                 level = DocumentLevel(mode: mUnknown, indicatorColumn: -1,
@@ -884,6 +896,15 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
                 level = ancestry.pop()
             else:
                 yieldUnexpectedToken()
+        of ypFlowAfterQuestionMark:
+            case token
+            of tScalar, tScalarPart, tColon, tComma, tOpeningBrace, 
+               tOpeningBracket, tClosingBrace, tClosingBracket, tTagHandle,
+               tAnchor, tAlias:
+               state = ypFlow
+               continue
+            else:
+                yieldUnexpectedToken()
         of ypFlowAfterTag:
             case token
             of tTagHandle:

private/tagLibrary.nim

@@ -13,6 +13,7 @@ proc registerUri*(tagLib: var YamlTagLibrary, uri: string): TagId =
     tagLib.tags[uri] = tagLib.nextCustomTagId
     result = tagLib.nextCustomTagId
     tagLib.nextCustomTagId = cast[TagId](cast[int](tagLib.nextCustomTagId) + 1)
+    echo "registered ", uri, " as: ", result
     
 proc uri*(tagLib: YamlTagLibrary, id: TagId): string =
     for iUri, iId in tagLib.tags.pairs:

yaml/serialization.nim

@@ -1,5 +1,5 @@
 import "../yaml"
-import macros, strutils, streams, tables, json, hashes
+import macros, strutils, streams, tables, json, hashes, re
 export yaml, streams, tables, json
 
 type
@@ -33,11 +33,13 @@ var
 
 static:
     iterator objectFields(n: NimNode): tuple[name: NimNode, t: NimNode] =
-        assert n.kind == nnkRecList
+        assert n.kind in [nnkRecList, nnkTupleTy]
         for identDefs in n.children:
             let numFields = identDefs.len - 2
             for i in 0..numFields - 1:
                 yield (name: identDefs[i], t: identDefs[^2])
+    
+    var existingTuples = newSeq[NimNode]()
 
 template presentTag(t: typedesc, tagStyle: YamlTagStyle): TagId =
      if tagStyle == ytsNone: yTagQuestionMark else: yamlTag(t)
@@ -56,22 +58,56 @@ macro make_serializable*(types: stmt): stmt =
         let
             tName = $typedef[0].symbol
             tIdent = newIdentNode(tName)
-            tUri = "!nim:" & tName
+        var
+            tUri: NimNode
+            recList: NimNode
         assert typedef[1].kind == nnkEmpty
         let objectTy = typedef[2]
-        assert objectTy.kind == nnkObjectTy
+        case objectTy.kind
-        assert objectTy[0].kind == nnkEmpty
+        of nnkObjectTy:
-        assert objectTy[1].kind == nnkEmpty
+            assert objectTy[0].kind == nnkEmpty
-        let recList = objectTy[2]
+            assert objectTy[1].kind == nnkEmpty
-        assert recList.kind == nnkRecList
+            tUri = newStrLitNode("!nim:" & tName)
-        
+            recList = objectTy[2]
+        of nnkTupleTy:
+            if objectTy in existingTuples:
+                continue
+            existingTuples.add(objectTy)
+            
+            recList = objectTy
+            tUri = newStmtList()
+            var
+                first = true
+                curStrLit = "!nim:tuple("
+                curInfix = tUri
+            for field in objectFields(recList):
+                if first:
+                    first = false
+                else:
+                    curStrLit &= ","
+                curStrLit &= $field.name & "="
+                var tmp = newNimNode(nnkInfix).add(newIdentNode("&"),
+                        newStrLitNode(curStrLit))
+                curInfix.add(tmp)
+                curInfix = tmp
+                tmp = newNimNode(nnkInfix).add(newIdentNode("&"),
+                        newCall("safeTagUri", newCall("yamlTag",
+                            newCall("type", field.t))))
+                curInfix.add(tmp)
+                curInfix = tmp
+                curStrLit = ""
+            curInfix.add(newStrLitNode(curStrLit & ")"))
+            tUri = tUri[0]
+        else:
+            assert false
+                
         # yamlTag()
         
         var yamlTagProc = newProc(newIdentNode("yamlTag"), [
                 newIdentNode("TagId"),
                 newIdentDefs(newIdentNode("T"), newNimNode(nnkBracketExpr).add(
                              newIdentNode("typedesc"), tIdent))])
-        var impl = newStmtList(newCall("lazyLoadTag", newStrLitNode(tUri)))
+        var impl = newStmtList(newCall("lazyLoadTag", tUri))
         yamlTagProc[6] = impl
         result.add(yamlTagProc)
         
@@ -93,6 +129,7 @@ macro make_serializable*(types: stmt): stmt =
             if finished(s):
                 raise newException(ValueError, "Construction error!")
             while event.kind != yamlEndMap:
+                if event.kind == yamlError: echo event.description
                 assert event.kind == yamlScalar
                 assert event.scalarTag in [yTagQuestionMark, yTagString]
                 case hash(event.scalarContent)
@@ -102,7 +139,7 @@ macro make_serializable*(types: stmt): stmt =
                 event = s()
                 if finished(s):
                     raise newException(ValueError, "Construction error!")
-        var keyCase = impl[5][1][2]
+        var keyCase = impl[5][1][3]
         assert keyCase.kind == nnkCaseStmt
         for field in objectFields(recList):
             let nameHash = hash($field.name.ident)
@@ -178,6 +215,7 @@ macro make_serializable*(types: stmt): stmt =
                 nnkIteratorDef)))
         serializeProc[6] = impl
         result.add(serializeProc)
+    echo result.repr
 
 proc prepend*(event: YamlStreamEvent, s: YamlStream): YamlStream =
     result = iterator(): YamlStreamEvent =
@@ -187,6 +225,13 @@ proc prepend*(event: YamlStreamEvent, s: YamlStream): YamlStream =
 
 proc yamlTag*(T: typedesc[string]): TagId {.inline.} = yTagString
 
+proc safeTagUri*(id: TagId): string =
+    let uri = serializationTagLibrary.uri(id)
+    if uri.len > 0 and uri[0] == '!':
+        return uri[1..uri.len - 1]
+    else:
+        return uri
+
 proc construct*(s: YamlStream, result: var string) =
     let item = s()
     if finished(s) or item.kind != yamlScalar:
@@ -208,8 +253,8 @@ proc construct*(s: YamlStream, result: var int) =
     let item = s()
     if finished(s) or item.kind != yamlScalar:
         raise newException(ValueError, "Construction error!")
-    if item.scalarTag notin [yTagQuestionMark, yTagInteger] or
+    if item.scalarTag != yTagInteger and not (
-        item.scalarType != yTypeInteger:
+       item.scalarTag == yTagQuestionMark and item.scalarType == yTypeInteger):
         raise newException(ValueError, "Wrong scalar type for int.")
     result = parseInt(item.scalarContent)
 
@@ -226,8 +271,8 @@ proc contruct*(s: YamlStream, result: var int64) =
     let item = s()
     if finished(s) or item.kind != yamlScalar:
         raise newException(ValueError, "Construction error!")
-    if item.scalarTag notin [yTagQuestionMark, yTagInteger] or
+    if item.scalarTag != yTagInteger and not (
-        item.scalarType != yTypeInteger:
+       item.scalarTag == yTagQuestionMark and item.scalarType == yTypeInteger):
         raise newException(ValueError, "Wrong scalar type for int64.")
     result = parseBiggestInt(item.scalarContent)
 
@@ -244,7 +289,8 @@ proc construct*(s: YamlStream, result: var float) =
     let item = s()
     if finished(s) or item.kind != yamlScalar:
         raise newException(ValueError, "Construction error!")
-    if item.scalarTag notin [yTagQuestionMark, yTagFloat]:
+    if item.scalarTag != yTagFloat and not (
+       item.scalarTag == yTagQuestionMark and item.scalarType == yTypeFloat):
         raise newException(ValueError, "Wrong scalar type for float.")
     case item.scalarType
     of yTypeFloat:
@@ -278,16 +324,27 @@ proc construct*(s: YamlStream, result: var bool) =
     let item = s()
     if finished(s) or item.kind != yamlScalar:
         raise newException(ValueError, "Construction error!")
-    if item.scalarTag notin [yTagQuestionMark, yTagBoolean]:
+    case item.scalarTag
-        raise newException(ValueError, "Wrong scalar type for bool.")
+    of yTagQuestionMark:
-    case item.scalarType
+        case item.scalarType
-    of yTypeBoolTrue:
+        of yTypeBoolTrue:
-        result = true
+            result = true
-    of yTypeBoolFalse:
+        of yTypeBoolFalse:
-        result = false
+            result = false
+        else:
+            raise newException(ValueError, "Wrong scalar type for bool.")
+    of yTagBoolean:
+        if item.scalarContent.match(
+                re"y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON"):
+            result = true
+        elif item.scalarContent.match(
+                re"n|N|no|No|NO|false|False|FALSE|off|Off|OFF"):
+            result = false
+        else:
+            raise newException(ValueError, "Wrong content for bool.")
     else:
-        raise newException(ValueError, "Wrong scalar type for bool.")
+        raise newException(ValueError, "Wrong scalar type for bool")
-
+        
 proc serialize*(value: bool,
                 tagStyle: YamlTagStyle = ytsNone): YamlStream =
     result = iterator(): YamlStreamEvent =
@@ -297,11 +354,7 @@ proc serialize*(value: bool,
                               if value: "y" else: "n")
 
 proc yamlTag*[I](T: typedesc[seq[I]]): TagId {.inline.} =
-    let
+    let uri = "!nim:seq(" & safeTagUri(yamlTag(I)) & ")"
-        childUri = serializationTagLibrary.uri(yamlTag(I))
-        childId  = if childUri[0] == '!':
-                childUri[1..childUri.len-1] else: childUri
-        uri      = "!nim:seq(" & childId & ")"
     result = lazyLoadTag(uri)
 
 proc construct*[T](s: YamlStream, result: var seq[T]) =
@@ -389,7 +442,7 @@ proc serialize*[K, V](value: Table[K, V],
 
 proc load*[K](input: Stream, target: var K) =
     var
-        parser = newParser(coreTagLibrary())
+        parser = newParser(serializationTagLibrary)
         events = parser.parse(input)
     assert events().kind == yamlStartDocument
     construct(events, target)