WIP Deserialization of the JsonString literal fields

json_serialization/lexer.nim

@@ -41,7 +41,7 @@ type
     errNonPortableInt       = "number is outside the range of portable values"
 
   JsonLexer* = object
-    stream: InputStream
+    stream*: InputStream
     mode*: JsonMode
 
     line*: int
@@ -61,6 +61,23 @@ const
                  1e20, 1e21, 1e22] # TODO: this table should be much larger
                                    # The largest JSON number value is 1E308
 
+proc renderTok*(s: JsonLexer): string =
+  case s.tok
+  of tkError, tkEof: ""
+  of tkString: ""
+  of tkInt: ""
+  of tkNegativeInt: ""
+  of tkFloat: ""
+  of tkTrue: "true"
+  of tkFalse: "false"
+  of tkNull: "null"
+  of tkCurlyLe: "{"
+  of tkCurlyRi: "}"
+  of tkBracketLe: "["
+  of tkBracketRi: "]"
+  of tkColon: ":"
+  of tkComma: ","
+
 template peek(s: InputStream): char =
   char inputs.peek(s)
 

json_serialization/reader.nim

@@ -11,7 +11,7 @@ export
 type
   JsonReader* = object
     lexer*: JsonLexer
-    forwardCompatible: bool
+    allowUnknownFields: bool
 
   JsonReaderError* = object of JsonError
     line*, col*: int
@@ -120,8 +120,8 @@ proc handleReadException*(r: JsonReader,
 proc init*(T: type JsonReader,
            stream: InputStream,
            mode = defaultJsonMode,
-           forwardCompatible = false): T =
+           allowUnknownFields = false): T =
-  result.forwardCompatible = forwardCompatible
+  result.allowUnknownFields = allowUnknownFields
   result.lexer = JsonLexer.init(stream, mode)
   result.lexer.next()
 
@@ -177,13 +177,24 @@ proc skipSingleJsValue(r: var JsonReader) =
   of tkString, tkInt, tkNegativeInt, tkFloat, tkTrue, tkFalse, tkNull:
     r.lexer.next()
 
+proc captureSingleJsValue(r: var JsonReader, value: var string) =
+  value = $r.lexer.tok
+  let pos = r.lexer.stream.createRewindPoint()
+  echo "skipiing ", pos
+  r.skipSingleJsValue()
+  let finalPos = r.lexer.stream.pos
+  echo "done ", finalPos
+  echo "Rewiding ", finalPos - pos
+  r.lexer.stream.rewindTo pos
+  value.setLen(finalPos - pos)
+  doAssert r.lexer.stream.readInto(value)
+
 proc allocPtr[T](p: var ptr T) =
   p = create(T)
 
 proc allocPtr[T](p: var ref T) =
   p = new(T)
 
-
 iterator readArray*(r: var JsonReader, ElemType: typedesc): ElemType =
   mixin readValue
 
@@ -240,12 +251,14 @@ proc readValue*[T](r: var JsonReader, value: var T)
     r.requireToken tkString
     value = r.lexer.strVal
     r.lexer.next()
+  
   elif value is seq[char]:
     r.requireToken tkString
     value.setLen(r.lexer.strVal.len)
     for i in 0..<r.lexer.strVal.len:
       value[i] = r.lexer.strVal[i]
     r.lexer.next()
+  
   elif isCharArray(value):
     r.requireToken tkString
     if r.lexer.strVal.len != value.len:
@@ -254,6 +267,7 @@ proc readValue*[T](r: var JsonReader, value: var T)
     for i in 0..<value.len:
       value[i] = r.lexer.strVal[i]
     r.lexer.next()
+  
   elif value is bool:
     case tok
     of tkTrue: value = true
@@ -342,6 +356,9 @@ proc readValue*[T](r: var JsonReader, value: var T)
     readValue(r, value[high(value)])
     r.skipToken tkBracketRi
 
+  elif value is JsonString:
+    r.captureSingleJsValue(string value)
+
   elif value is (object or tuple):
     type T = type(value)
     r.skipToken tkCurlyLe
@@ -359,7 +376,7 @@ proc readValue*[T](r: var JsonReader, value: var T)
         r.skipToken tkColon
         if reader != nil:
           reader(value, r)
-        elif r.forwardCompatible:
+        elif r.allowUnknownFields:
           r.skipSingleJsValue()
         else:
           const typeName = typetraits.name(T)

json_serialization/types.nim

@@ -1,4 +1,5 @@
 import
+  tables,
   serialization/errors
 
 export
@@ -18,3 +19,6 @@ const
   minPortableInt* = -9007199254740991 # -2**53 + 1
   maxPortableInt* =  9007199254740991 # +2**53 - 1
 
+template `==`*(lhs, rhs: JsonString): bool =
+  string(lhs) == string(rhs)
+

tests/test_serialization.nim

@@ -36,6 +36,11 @@ type
   MyUseCaseObject = object
     field: MyCaseObject
 
+  HasJsonString = object
+    name: string
+    data: JsonString
+    id: int
+
 # TODO `borrowSerialization` still doesn't work
 # properly when it's placed in another module:
 Meter.borrowSerialization int
@@ -111,7 +116,7 @@ suite "toJson tests":
         }
       """
 
-    let decoded = Json.decode(json, Simple, forwardCompatible = true)
+    let decoded = Json.decode(json, Simple, allowUnknownFields = true)
 
     check:
       decoded.x == -20
@@ -184,3 +189,51 @@ suite "toJson tests":
 
     expect JsonReaderError: # too long
       discard Json.decode(Json.encode(@['a', 'b']), array[1, char])
+
+  test "Holders of JsonString":
+    let
+      data1 = dedent"""
+        {
+          "name": "Data 1",
+          "data": [1, 2, 3, 4],
+          "id": 101
+        }
+      """
+    let
+      data2 = dedent"""
+        {
+          "name": "Data 2",
+          "data": "some string",
+          "id": 1002
+        }
+      """
+    let
+      data3 = dedent"""
+        {
+          "name": "Data 3",
+          "data": {"field1": 10, "field2": [1, 2, 3], "field3": "test"},
+          "id": 10003
+        }
+      """
+
+    try:
+      let
+        d1 = Json.decode(data1, HasJsonString)
+        d2 = Json.decode(data2, HasJsonString)
+        d3 = Json.decode(data3, HasJsonString)
+
+      check:
+        d1.name == "Data 1"
+        d1.data == JsonString "[1, 2, 3, 4]"
+        d1.id == 101
+
+        d2.name == "Data 2"
+        d2.data == JsonString "\"some string\""
+        d2.id == 1002
+
+        d3.name == "Data 3"
+        d3.data == JsonString """{"field1": 10, "field2": [1, 2, 3], "field3": "test"}"""
+        d3.id == 10003
+    except SerializationError as e:
+      echo e.formatMsg("<>")
+