Alternative implementation that doesn't require rewind points

json_serialization/lexer.nim

@@ -1,5 +1,5 @@
 import
-  unicode,
+  unicode, json,
   faststreams/inputs,
   types
 
@@ -61,22 +61,39 @@ const
                  1e20, 1e21, 1e22] # TODO: this table should be much larger
                                    # The largest JSON number value is 1E308
 
-proc renderTok*(s: JsonLexer): string =
-  case s.tok
-  of tkError, tkEof: ""
-  of tkString: ""
-  of tkInt: ""
-  of tkNegativeInt: ""
-  of tkFloat: ""
-  of tkTrue: "true"
-  of tkFalse: "false"
-  of tkNull: "null"
-  of tkCurlyLe: "{"
-  of tkCurlyRi: "}"
-  of tkBracketLe: "["
-  of tkBracketRi: "]"
-  of tkColon: ":"
-  of tkComma: ","
+proc renderTok*(lexer: JsonLexer, output: var string) =
+  case lexer.tok
+  of tkError, tkEof:
+    discard
+  of tkString:
+    output.add '"'
+    lexer.strVal.escapeJsonUnquoted output
+    output.add '"'
+  of tkInt:
+    output.add $lexer.absIntVal
+  of tkNegativeInt:
+    output.add '-'
+    output.add $lexer.absIntVal
+  of tkFloat:
+    output.add $lexer.floatVal
+  of tkTrue:
+    output.add "true"
+  of tkFalse:
+    output.add "false"
+  of tkNull:
+    output.add "null"
+  of tkCurlyLe:
+    output.add '{'
+  of tkCurlyRi:
+    output.add '}'
+  of tkBracketLe:
+    output.add '['
+  of tkBracketRi:
+    output.add ']'
+  of tkColon:
+    output.add ':'
+  of tkComma:
+    output.add ','
 
 template peek(s: InputStream): char =
   char inputs.peek(s)

json_serialization/reader.nim

@@ -177,17 +177,48 @@ proc skipSingleJsValue(r: var JsonReader) =
   of tkString, tkInt, tkNegativeInt, tkFloat, tkTrue, tkFalse, tkNull:
     r.lexer.next()
 
-proc captureSingleJsValue(r: var JsonReader, value: var string) =
-  value = $r.lexer.tok
-  let pos = r.lexer.stream.createRewindPoint()
-  echo "skipiing ", pos
-  r.skipSingleJsValue()
-  let finalPos = r.lexer.stream.pos
-  echo "done ", finalPos
-  echo "Rewiding ", finalPos - pos
-  r.lexer.stream.rewindTo pos
-  value.setLen(finalPos - pos)
-  doAssert r.lexer.stream.readInto(value)
+proc captureSingleJsValue(r: var JsonReader, output: var string) =
+  r.lexer.renderTok output
+  case r.lexer.tok
+  of tkCurlyLe:
+    r.lexer.next()
+    if r.lexer.tok != tkCurlyRi:
+      while true:
+        r.lexer.renderTok output
+        r.skipToken tkString
+        r.lexer.renderTok output
+        r.skipToken tkColon
+        r.captureSingleJsValue(output)
+        r.lexer.renderTok output
+        if r.lexer.tok == tkCurlyRi:
+          break
+        else:
+          r.skipToken tkComma
+    else:
+      output.add '}'
+    # Skip over the last tkCurlyRi
+    r.lexer.next()
+
+  of tkBracketLe:
+    r.lexer.next()
+    if r.lexer.tok != tkBracketRi:
+      while true:
+        r.captureSingleJsValue(output)
+        r.lexer.renderTok output
+        if r.lexer.tok == tkBracketRi:
+          break
+        else:
+          r.skipToken tkComma
+    else:
+      output.add ']'
+    # Skip over the last tkBracketRi
+    r.lexer.next()
+
+  of tkColon, tkComma, tkEof, tkError, tkBracketRi, tkCurlyRi:
+    r.raiseUnexpectedToken etValue
+
+  of tkString, tkInt, tkNegativeInt, tkFloat, tkTrue, tkFalse, tkNull:
+    r.lexer.next()
 
 proc allocPtr[T](p: var ptr T) =
   p = create(T)
@@ -251,14 +282,14 @@ proc readValue*[T](r: var JsonReader, value: var T)
     r.requireToken tkString
     value = r.lexer.strVal
     r.lexer.next()
-  
+
   elif value is seq[char]:
     r.requireToken tkString
     value.setLen(r.lexer.strVal.len)
     for i in 0..<r.lexer.strVal.len:
       value[i] = r.lexer.strVal[i]
     r.lexer.next()
-  
+
   elif isCharArray(value):
     r.requireToken tkString
     if r.lexer.strVal.len != value.len:
@@ -267,7 +298,7 @@ proc readValue*[T](r: var JsonReader, value: var T)
     for i in 0..<value.len:
       value[i] = r.lexer.strVal[i]
     r.lexer.next()
-  
+
   elif value is bool:
     case tok
     of tkTrue: value = true

tests/test_serialization.nim

@@ -224,7 +224,7 @@ suite "toJson tests":
 
       check:
         d1.name == "Data 1"
-        d1.data == JsonString "[1, 2, 3, 4]"
+        d1.data == JsonString "[1,2,3,4]"
         d1.id == 101
 
         d2.name == "Data 2"
@@ -232,7 +232,7 @@ suite "toJson tests":
         d2.id == 1002
 
         d3.name == "Data 3"
-        d3.data == JsonString """{"field1": 10, "field2": [1, 2, 3], "field3": "test"}"""
+        d3.data == JsonString """{"field1":10,"field2":[1,2,3],"field3":"test"}"""
         d3.id == 10003
     except SerializationError as e:
       echo e.formatMsg("<>")