Handle int overflows better; Support Nim 1.2

json_serialization/lexer.nim

@@ -12,6 +12,7 @@ type
     tkEof,
     tkString,
     tkInt,
+    tkNegativeInt,
     tkFloat,
     tkTrue,
     tkFalse,
@@ -50,7 +51,7 @@ type
     tok*: TokKind
     err*: JsonErrorKind
 
-    intVal*: int64
+    absIntVal*: uint64 # BEWARE: negative integers will have tok == tkNegativeInt
     floatVal*: float
     strVal*: string
 
@@ -84,7 +85,7 @@ proc init*(T: type JsonLexer, stream: ref AsciiStream, mode = defaultJsonMode):
     tokenStart: -1,
     tok: tkError,
     err: errNone,
-    intVal: int64 0,
+    absIntVal: uint64 0,
     floatVal: 0'f,
     strVal: "")
 
@@ -271,15 +272,16 @@ proc scanNumber(lexer: var JsonLexer) =
     lexer.tok = tkFloat
     c = lexer.stream[].peek()
   elif c.isDigit:
-    lexer.tok = tkInt
+    lexer.tok = if sign > 0: tkInt
+                else: tkNegativeInt
     let scannedValue = lexer.scanInt()
     checkForNonPortableInt scannedValue
-    lexer.intVal = int64(scannedValue) * sign
+    lexer.absIntVal = scannedValue
     if lexer.stream[].eof: return
     c = lexer.stream[].peek()
     if c == '.':
       lexer.tok = tkFloat
-      lexer.floatVal = float(lexer.intVal)
+      lexer.floatVal = float(lexer.absIntVal) * float(sign)
       c = eatDigitAndPeek()
   else:
     error errNumberExpected

json_serialization/reader.nim

@@ -37,6 +37,15 @@ type
     encountedToken*: TokKind
     expectedToken*: ExpectedTokenCategory
 
+  IntOverflowError* = object of JsonReaderError
+    isNegative: bool
+    absIntVal: uint64
+
+func valueStr(err: ref IntOverflowError): string =
+  if err.isNegative:
+    result.add '-'
+  result.add($err.absIntVal)
+
 method formatMsg*(err: ref JsonReaderError, filename: string): string =
   fmt"{filename}({err.line}, {err.col}) Error while reading json file"
 
@@ -49,6 +58,9 @@ method formatMsg*(err: ref UnexpectedToken, filename: string): string =
 method formatMsg*(err: ref GenericJsonReaderError, filename: string): string =
   fmt"{filename}({err.line}, {err.col}) Exception encountered while deserializing '{err.deserializedField}': [{err.innerException.name}] {err.innerException.msg}"
 
+method formatMsg*(err: ref IntOverflowError, filename: string): string =
+  fmt"{filename}({err.line}, {err.col}) The value '{err.valueStr}' is outside of the allowed range"
+
 template init*(T: type JsonReader, stream: ByteStreamVar, mode = defaultJsonMode): auto =
   init JsonReader, AsciiStreamVar(stream), mode
 
@@ -56,14 +68,21 @@ proc assignLineNumber*(ex: ref JsonReaderError, r: JsonReader) =
   ex.line = r.lexer.line
   ex.col = r.lexer.tokenStartCol
 
-proc raiseUnexpectedToken*(r: JsonReader, expected: ExpectedTokenCategory) =
+proc raiseUnexpectedToken*(r: JsonReader, expected: ExpectedTokenCategory) {.noreturn.} =
   var ex = new UnexpectedToken
   ex.assignLineNumber(r)
   ex.encountedToken = r.lexer.tok
   ex.expectedToken = expected
   raise ex
 
-proc raiseUnexpectedField*(r: JsonReader, fieldName, deserializedType: cstring) =
+proc raiseIntOverflow*(r: JsonReader, absIntVal: uint64, isNegative: bool) {.noreturn.} =
+  var ex = new IntOverflowError
+  ex.assignLineNumber(r)
+  ex.absIntVal = absIntVal
+  ex.isNegative = isNegative
+  raise ex
+
+proc raiseUnexpectedField*(r: JsonReader, fieldName, deserializedType: cstring) {.noreturn.} =
   var ex = new UnexpectedField
   ex.assignLineNumber(r)
   ex.encounteredField = fieldName
@@ -92,7 +111,7 @@ proc requireToken*(r: JsonReader, tk: TokKind) =
   if r.lexer.tok != tk:
     r.raiseUnexpectedToken case tk
       of tkString: etString
-      of tkInt: etInt
+      of tkInt, tkNegativeInt: etInt
       of tkComma: etComma
       of tkBracketRi: etBracketRi
       of tkBracketLe: etBracketLe
@@ -141,6 +160,13 @@ iterator readObject*(r: var JsonReader, KeyType: typedesc, ValueType: typedesc):
       r.lexer.next()
   r.skipToken tkCurlyRi
 
+func maxAbsValue(T: type[SomeInteger]): uint64 {.compileTime.} =
+  when T is int8 : 128'u64
+  elif T is int16: 32768'u64
+  elif T is int32: 2147483648'u64
+  elif T is int64: 9223372036854775808'u64
+  else: uint64(high(T))
+
 proc readValue*(r: var JsonReader, value: var auto) =
   mixin readValue
 
@@ -173,20 +199,39 @@ proc readValue*(r: var JsonReader, value: var auto) =
       value.setParsed(r.lexer.strVal)
     of tkInt:
       # TODO: validate that the value is in range
-      value = type(value)(r.lexer.intVal)
+      value = type(value)(r.lexer.absIntVal)
     else:
       r.raiseUnexpectedToken etEnum
     r.lexer.next()
 
   elif value is SomeInteger:
     type TargetType = type(value)
-    r.requireToken tkInt
+    const maxValidValue = maxAbsValue(TargetType)
-    value = TargetType(r.lexer.intVal)
+
+    if r.lexer.absIntVal > maxValidValue:
+      r.raiseIntOverflow r.lexer.absIntVal, tok == tkNegativeInt
+
+    case tok
+    of tkInt:
+      value = TargetType(r.lexer.absIntVal)
+    of tkNegativeInt:
+      when value is SomeSignedInt:
+        if r.lexer.absIntVal == maxValidValue:
+          # We must handle this as a special case because it would be illegal
+          # to convert a value like 128 to int8 before negating it. The max
+          # int8 value is 127 (while the minimum is -128).
+          value = low(TargetType)
+        else:
+          value = -TargetType(r.lexer.absIntVal)
+      else:
+        r.raiseIntOverflow r.lexer.absIntVal, true
+    else:
+      r.raiseUnexpectedToken etInt
     r.lexer.next()
 
   elif value is SomeFloat:
     case tok
-    of tkInt: value = float(r.lexer.intVal)
+    of tkInt: value = float(r.lexer.absIntVal)
     of tkFloat: value = r.lexer.floatVal
     else:
       r.raiseUnexpectedToken etNumber