Add serde options to the json util, along with tests

next step is to:
1. change back any ethers var names that were changed for serialization purposes, eg `from` and `type`
2. move the json util to its own lib

ethers/providers/jsonrpc/json.nim

@@ -1,12 +1,14 @@
 
-import std/json except `%`, `%*`
+import std/json as stdjson except `%`, `%*`
 import std/macros
 import std/options
+import std/sequtils
+import std/sets
 import std/strutils
 # import std/strformat
 import std/tables
 import std/typetraits
-import pkg/chronicles
+import pkg/chronicles except toJson
 import pkg/contractabi
 import pkg/stew/byteutils
 import pkg/stint
@@ -15,21 +17,52 @@ import pkg/questionable/results
 
 import ../../basics
 
-export json except `%`, `%*`
+export stdjson except `%`, `%*`, parseJson
+export chronicles except toJson
+export sets
 
 {.push raises: [].}
 
 logScope:
-  topics = "json serialization"
+  topics = "json de/serialization"
 
 type
-  SerializationError = object of EthersError
-  UnexpectedKindError = object of SerializationError
+  SerdeError* = object of EthersError
+  UnexpectedKindError* = object of SerdeError
+  DeserializeMode* = enum
+    Default,  ## objects can have more or less fields than json
+    OptIn,    ## json must have fields marked with {.serialize.}
+    Strict    ## object fields and json fields must match exactly
 
-template serialize* {.pragma.}
+# template serializeAll* {.pragma.}
+template serialize*(key = "", ignore = false) {.pragma.}
+template deserialize*(key = "", mode = DeserializeMode.Default) {.pragma.}
 
-proc mapErrTo[T, E1: CatchableError, E2: CatchableError](r: Result[T, E1], _: type E2): ?!T =
-  r.mapErr(proc (e: E1): E2 = E2(msg: e.msg))
+template expectEmptyPragma(value, pragma, msg) =
+  static:
+    when value.hasCustomPragma(pragma):
+      const params = value.getCustomPragmaVal(pragma)
+      for param in params.fields:
+        if param != typeof(param).default:
+          raiseAssert(msg)
+
+template expectMissingPragmaParam(value, pragma, name, msg) =
+  static:
+    when value.hasCustomPragma(pragma):
+      const params = value.getCustomPragmaVal(pragma)
+      for paramName, paramValue in params.fieldPairs:
+        if paramName == name and paramValue != typeof(paramValue).default:
+          raiseAssert(msg)
+
+proc mapErrTo[E1: ref CatchableError, E2: SerdeError](
+  e1: E1,
+  _: type E2,
+  msg: string = e1.msg): ref E2 =
+
+  return newException(E2, msg, e1)
+
+proc newSerdeError(msg: string): ref SerdeError =
+  newException(SerdeError, msg)
 
 proc newUnexpectedKindError(
   expectedType: type,
@@ -40,7 +73,7 @@ proc newUnexpectedKindError(
              else: $json.kind
   newException(UnexpectedKindError,
     "deserialization to " & $expectedType & " failed: expected " &
-    expectedKinds & "but got " & $kind)
+    expectedKinds & " but got " & $kind)
 
 proc newUnexpectedKindError(
   expectedType: type,
@@ -71,24 +104,34 @@ template expectJsonKind*(
 ) =
   expectJsonKind(expectedType, {expectedKind}, json)
 
+proc fieldKeys[T](obj: T): seq[string] =
+  for name, _ in fieldPairs(when type(T) is ref: obj[] else: obj):
+    result.add name
+
+func keysNotIn[T](json: JsonNode, obj: T): HashSet[string] =
+  let jsonKeys = json.keys.toSeq.toHashSet
+  let objKeys = obj.fieldKeys.toHashSet
+  difference(jsonKeys, objKeys)
+
 proc fromJson*(
   T: type enum,
   json: JsonNode
 ): ?!T =
   expectJsonKind(string, JString, json)
-  catch parseEnum[T](json.str)
+  without val =? parseEnum[T](json.str).catch, error:
+    return failure error.mapErrTo(SerdeError)
+  return success val
 
 proc fromJson*(
   _: type string,
   json: JsonNode
 ): ?!string =
   if json.isNil:
-    let err = newException(ValueError, "'json' expected, but was nil")
-    return failure(err)
+    return failure newSerdeError("'json' expected, but was nil")
   elif json.kind == JNull:
     return success("null")
   elif json.isNil or json.kind != JString:
-    return failure(newUnexpectedKindError(string, JString, json))
+    return failure newUnexpectedKindError(string, JString, json)
   catch json.getStr
 
 proc fromJson*(
@@ -113,7 +156,8 @@ proc fromJson*[T: SomeInteger](
     expectJsonKind(T, {JInt, JString}, json)
     case json.kind
     of JString:
-      let x = parseBiggestUInt(json.str)
+      without x =? parseBiggestUInt(json.str).catch, error:
+        return failure newSerdeError(error.msg)
       return success cast[T](x)
     else:
       return success T(json.num)
@@ -201,41 +245,82 @@ proc fromJson*[T](
     arr.add(? T.fromJson(elem))
   success arr
 
+template getDeserializationKey(fieldName, fieldValue): string =
+  when fieldValue.hasCustomPragma(deserialize):
+    fieldValue.expectMissingPragmaParam(deserialize, "mode",
+                                    "Cannot set 'mode' on field defintion.")
+    let (key, mode) = fieldValue.getCustomPragmaVal(deserialize)
+    if key != "": key
+    else: fieldName
+  else: fieldName
+
+template getDeserializationMode(T): DeserializeMode =
+  when T.hasCustomPragma(deserialize):
+    T.expectMissingPragmaParam(deserialize, "key",
+                               "Cannot set 'key' on object definition.")
+    T.getCustomPragmaVal(deserialize)[1] # mode = second pragma param
+  else:
+    DeserializeMode.Default
+
 proc fromJson*[T: ref object or object](
   _: type T,
   json: JsonNode
 ): ?!T =
+
   when T is JsonNode:
     return success T(json)
 
   expectJsonKind(T, JObject, json)
   var res = when type(T) is ref: T.new() else: T.default
+  let mode = T.getDeserializationMode()
 
-  # Leave this in, it's good for debugging:
-  trace "deserializing object", to = $T, json
   for name, value in fieldPairs(when type(T) is ref: res[] else: res):
-
     logScope:
       field = $T & "." & name
+      mode
 
-    if name in json and
-       jsonVal =? json{name}.catch and
-       not jsonVal.isNil:
+    let key = getDeserializationKey(name, value)
+    var skip = false # workaround for 'continue' not supported in a 'fields' loop
+
+    if mode == Strict and key notin json:
+      return failure newSerdeError("object field missing in json: " & key)
+
+    if mode == OptIn:
+      if not value.hasCustomPragma(deserialize):
+        debug "object field not marked as 'deserialize', skipping", name = name
+        # use skip as workaround for 'continue' not supported in a 'fields' loop
+        skip = true
+      elif key notin json:
+        return failure newSerdeError("object field missing in json: " & key)
+
+    if key in json and
+       jsonVal =? json{key}.catch and
+       not jsonVal.isNil and
+       not skip:
 
       without parsed =? type(value).fromJson(jsonVal), e:
-        error "error deserializing field",
+        warn "failed to deserialize field",
+          `type` = $typeof(value),
           json = jsonVal,
           error = e.msg
         return failure(e)
       value = parsed
 
-    else:
-      debug "object field does not exist in json, skipping", json
+    elif mode == DeserializeMode.Default:
+      debug "object field missing in json, skipping", key, json
+
+    # ensure there's no extra fields in json
+    if mode == DeserializeMode.Strict:
+      let extraFields = json.keysNotIn(res)
+      if extraFields.len > 0:
+        return failure newSerdeError("json field(s) missing in object: " & $extraFields)
+
   success(res)
 
-proc parse*(json: string): ?!JsonNode =
+proc parseJson*(json: string): ?!JsonNode =
+  ## fix for nim raising Exception
   try:
-    return parseJson(json).catch
+    return stdjson.parseJson(json).catch
   except Exception as e:
     return err newException(CatchableError, e.msg)
 
@@ -254,10 +339,10 @@ proc fromJson*(
 
 proc fromJson*[T: ref object or object](
   _: type T,
-  json: string
+  jsn: string
 ): ?!T =
-  let json = ? parse(json)
-  T.fromJson(json)
+  let jsn = ? json.parseJson(jsn) # full qualification required in-module only
+  T.fromJson(jsn)
 
 func `%`*(s: string): JsonNode = newJString(s)
 
@@ -301,18 +386,34 @@ func `%`*[T](table: Table[string, T]|OrderedTable[string, T]): JsonNode =
 func `%`*[T](opt: Option[T]): JsonNode =
   if opt.isSome: %(opt.get) else: newJNull()
 
-func `%`*[T: object](obj: T): JsonNode =
-  let jsonObj = newJObject()
-  for name, value in obj.fieldPairs:
-    when value.hasCustomPragma(serialize):
-      jsonObj[name] = %value
-  jsonObj
 
-func `%`*[T: ref object](obj: T): JsonNode =
+func `%`*[T: object or ref object](obj: T): JsonNode =
+
+  # T.expectMissingPragma(serialize, "Invalid pragma on object definition.")
+
   let jsonObj = newJObject()
-  for name, value in obj[].fieldPairs:
-    when value.hasCustomPragma(serialize):
-      jsonObj[name] = %(value)
+  let o = when T is ref object: obj[]
+          else: obj
+
+  T.expectEmptyPragma(serialize, "Cannot specify 'key' or 'ignore' on object defition")
+
+  const serializeAllFields = T.hasCustomPragma(serialize)
+
+  for name, value in o.fieldPairs:
+    # TODO: move to %
+    # value.expectMissingPragma(deserializeMode, "Invalid pragma on field definition.")
+    # static:
+    const serializeField = value.hasCustomPragma(serialize)
+    when serializeField:
+      let (keyOverride, ignore) = value.getCustomPragmaVal(serialize)
+      if not ignore:
+        let key = if keyOverride != "": keyOverride
+                  else: name
+        jsonObj[key] = %value
+
+    elif serializeAllFields:
+      jsonObj[name] = %value
+
   jsonObj
 
 proc `%`*(o: enum): JsonNode = % $o

testmodule/providers/jsonrpc/testjson.nim

@@ -0,0 +1,385 @@
+import std/math
+import std/options
+import std/strformat
+import std/strutils
+import std/unittest
+import pkg/stew/byteutils
+import pkg/stint
+import pkg/ethers/providers/jsonrpc/json as utilsjson
+import pkg/questionable
+import pkg/questionable/results
+
+
+func flatten(s: string): string =
+  s.replace(" ")
+    .replace("\n")
+
+suite "json serialization - serialize":
+
+  test "serializes UInt256 to non-hex string representation":
+    check (% 100000.u256) == newJString("100000")
+
+  test "serializes sequence to an array":
+    let json = % @[1, 2, 3]
+    let expected = "[1,2,3]"
+    check $json == expected
+
+  test "serializes Option[T] when has a value":
+    let obj = %(some 1)
+    let expected = "1"
+    check $obj == expected
+
+  test "serializes Option[T] when doesn't have a value":
+    let obj = %(none int)
+    let expected = "null"
+    check $obj == expected
+
+  test "serializes uints int.high or smaller":
+    let largeUInt: uint = uint(int.high)
+    check %largeUInt == newJInt(BiggestInt(largeUInt))
+
+  test "serializes large uints":
+    let largeUInt: uint = uint(int.high) + 1'u
+    check %largeUInt == newJString($largeUInt)
+
+  test "serializes Inf float":
+    check %Inf == newJString("inf")
+
+  test "serializes -Inf float":
+    check %(-Inf) == newJString("-inf")
+
+  test "can construct json objects with %*":
+    type MyObj = object
+      mystring {.serialize.}: string
+      myint {.serialize.}: int
+      myoption {.serialize.}: ?bool
+
+    let myobj = MyObj(mystring: "abc", myint: 123, myoption: some true)
+    let mystuint = 100000.u256
+
+    let json = %*{
+      "myobj": myobj,
+      "mystuint": mystuint
+    }
+
+    let expected = """{
+                        "myobj": {
+                          "mystring": "abc",
+                          "myint": 123,
+                          "myoption": true
+                        },
+                        "mystuint": "100000"
+                      }""".flatten
+
+    check $json == expected
+
+  test "only serializes marked fields":
+    type MyObj = object
+      mystring {.serialize.}: string
+      myint {.serialize.}: int
+      mybool: bool
+
+    let obj = % MyObj(mystring: "abc", myint: 1, mybool: true)
+
+    let expected = """{
+                        "mystring": "abc",
+                        "myint": 1
+                      }""".flatten
+
+    check $obj == expected
+
+  test "serializes ref objects":
+    type MyRef = ref object
+      mystring {.serialize.}: string
+      myint {.serialize.}: int
+
+    let obj = % MyRef(mystring: "abc", myint: 1)
+
+    let expected = """{
+                        "mystring": "abc",
+                        "myint": 1
+                      }""".flatten
+
+    check $obj == expected
+
+suite "json serialization - deserialize":
+
+  test "deserializes NaN float":
+    check %NaN == newJString("nan")
+
+  test "deserialize enum":
+    type MyEnum = enum
+      First,
+      Second
+    let json = newJString("Second")
+    check !MyEnum.fromJson(json) == Second
+
+  test "deserializes UInt256 from non-hex string representation":
+    let json = newJString("100000")
+    check !UInt256.fromJson(json) == 100000.u256
+
+  test "deserializes Option[T] when has a value":
+    let json = newJInt(1)
+    check (!fromJson(?int, json) == some 1)
+
+  test "deserializes Option[T] when doesn't have a value":
+    let json = newJNull()
+    check !fromJson(?int, json) == none int
+
+  test "deserializes float":
+    let json = newJFloat(1.234)
+    check !float.fromJson(json) == 1.234
+
+  test "deserializes Inf float":
+    let json = newJString("inf")
+    check !float.fromJson(json) == Inf
+
+  test "deserializes -Inf float":
+    let json = newJString("-inf")
+    check !float.fromJson(json) == -Inf
+
+  test "deserializes NaN float":
+    let json = newJString("nan")
+    check (!float.fromJson(json)).isNaN
+
+  test "deserializes array to sequence":
+    let expected = @[1, 2, 3]
+    let json = !"[1,2,3]".parseJson
+    check !seq[int].fromJson(json) == expected
+
+  test "deserializes uints int.high or smaller":
+    let largeUInt: uint = uint(int.high)
+    let json = newJInt(BiggestInt(largeUInt))
+    check !uint.fromJson(json) == largeUInt
+
+  test "deserializes large uints":
+    let largeUInt: uint = uint(int.high) + 1'u
+    let json = newJString($BiggestUInt(largeUInt))
+    check !uint.fromJson(json) == largeUInt
+
+  test "can deserialize json objects":
+    type MyObj = object
+      mystring: string
+      myint: int
+      myoption: ?bool
+
+    let expected = MyObj(mystring: "abc", myint: 123, myoption: some true)
+
+    let json = !parseJson("""{
+                              "mystring": "abc",
+                              "myint": 123,
+                              "myoption": true
+                            }""")
+    check !MyObj.fromJson(json) == expected
+
+  test "ignores serialize pragma when deserializing":
+    type MyObj = object
+      mystring {.serialize.}: string
+      mybool: bool
+
+    let expected = MyObj(mystring: "abc", mybool: true)
+
+    let json = !parseJson("""{
+                              "mystring": "abc",
+                              "mybool": true
+                            }""")
+
+    check !MyObj.fromJson(json) == expected
+
+  test "deserializes objects with extra fields":
+    type MyObj = object
+      mystring: string
+      mybool: bool
+
+    let expected = MyObj(mystring: "abc", mybool: true)
+
+    let json = !"""{
+                    "mystring": "abc",
+                    "mybool": true,
+                    "extra": "extra"
+                  }""".parseJson
+    check !MyObj.fromJson(json) == expected
+
+  test "deserializes objects with less fields":
+    type MyObj = object
+      mystring: string
+      mybool: bool
+
+    let expected = MyObj(mystring: "abc", mybool: false)
+
+    let json = !"""{
+                    "mystring": "abc"
+                  }""".parseJson
+    check !MyObj.fromJson(json) == expected
+
+  test "deserializes ref objects":
+    type MyRef = ref object
+      mystring: string
+      myint: int
+
+    let expected = MyRef(mystring: "abc", myint: 1)
+
+    let json = !"""{
+                    "mystring": "abc",
+                    "myint": 1
+                  }""".parseJson
+
+    let deserialized = !MyRef.fromJson(json)
+    check deserialized.mystring == expected.mystring
+    check deserialized.myint == expected.myint
+
+suite "json serialization pragmas":
+
+  test "fails to compile when object marked with 'serialize' specifies options":
+    type
+      MyObj {.serialize(key="test", ignore=true).} = object
+
+    check not compiles(%MyObj())
+
+  test "compiles when object marked with 'serialize' only":
+    type
+      MyObj {.serialize.} = object
+
+    check compiles(%MyObj())
+
+  test "fails to compile when field marked with 'deserialize' specifies mode":
+    type
+      MyObj = object
+       field {.deserialize(mode=OptIn).}: bool
+
+    check not compiles(MyObj.fromJson("""{"field":true}"""))
+
+  test "compiles when object marked with 'deserialize' specifies mode":
+    type
+      MyObj {.deserialize(mode=OptIn).} = object
+       field: bool
+
+    check compiles(MyObj.fromJson("""{"field":true}"""))
+
+  test "fails to compile when object marked with 'deserialize' specifies key":
+    type
+      MyObj {.deserialize("test").} = object
+       field: bool
+
+    check not compiles(MyObj.fromJson("""{"field":true}"""))
+
+  test "compiles when field marked with 'deserialize' specifies key":
+    type
+      MyObj = object
+       field {.deserialize("test").}: bool
+
+    check compiles(MyObj.fromJson("""{"field":true}"""))
+
+  test "compiles when field marked with empty 'deserialize'":
+    type
+      MyObj = object
+       field {.deserialize.}: bool
+
+    check compiles(MyObj.fromJson("""{"field":true}"""))
+
+  test "compiles when field marked with 'serialize'":
+    type
+      MyObj = object
+        field {.serialize.}: bool
+
+    check compiles(%MyObj())
+
+  test "serializes field with key when specified":
+    type MyObj = object
+      field {.serialize("test").}: bool
+
+    let obj = MyObj(field: true)
+    check obj.toJson == """{"test":true}"""
+
+  test "does not serialize ignored field":
+    type MyObj = object
+      field1 {.serialize.}: bool
+      field2 {.serialize(ignore=true).}: bool
+
+    let obj = MyObj(field1: true, field2: true)
+    check obj.toJson == """{"field1":true}"""
+
+  test "serialize on object definition serializes all fields":
+    type MyObj {.serialize.} = object
+      field1: bool
+      field2: bool
+
+    let obj = MyObj(field1: true, field2: true)
+    check obj.toJson == """{"field1":true,"field2":true}"""
+
+  test "ignores field when object has serialize":
+    type MyObj {.serialize.} = object
+      field1 {.serialize(ignore=true).}: bool
+      field2: bool
+
+    let obj = MyObj(field1: true, field2: true)
+    check obj.toJson == """{"field2":true}"""
+
+  test "serializes field with key when object has serialize":
+    type MyObj {.serialize.} = object
+      field1 {.serialize("test").}: bool
+      field2: bool
+
+    let obj = MyObj(field1: true, field2: true)
+    check obj.toJson == """{"test":true,"field2":true}"""
+
+  test "deserializes matching object and json fields when mode is Strict":
+    type MyObj {.deserialize(mode=Strict).} = object
+      field1: bool
+      field2: bool
+
+    let val = !MyObj.fromJson("""{"field1":true,"field2":true}""")
+    check val == MyObj(field1: true, field2: true)
+
+  test "fails to deserialize with missing json field when mode is Strict":
+    type MyObj {.deserialize(mode=Strict).} = object
+      field1: bool
+      field2: bool
+
+    let r = MyObj.fromJson("""{"field2":true}""")
+    check r.isFailure
+    check r.error of SerdeError
+    check r.error.msg == "object field missing in json: field1"
+
+  test "fails to deserialize with missing object field when mode is Strict":
+    type MyObj {.deserialize(mode=Strict).} = object
+      field2: bool
+
+    let r = MyObj.fromJson("""{"field1":true,"field2":true}""")
+    check r.isFailure
+    check r.error of SerdeError
+    check r.error.msg == "json field(s) missing in object: {\"field1\"}"
+
+  test "deserializes only fields marked as deserialize when mode is OptIn":
+    type MyObj {.deserialize(mode=OptIn).} = object
+      field1: int
+      field2 {.deserialize.}: bool
+
+    let val = !MyObj.fromJson("""{"field1":true,"field2":true}""")
+    check val == MyObj(field1: 0, field2: true)
+
+  test "can deserialize object in default mode when not marked with deserialize":
+    type MyObj = object
+      field1: bool
+      field2: bool
+
+    let val = !MyObj.fromJson("""{"field1":true,"field3":true}""")
+    check val == MyObj(field1: true, field2: false)
+
+  test "deserializes object field with marked json key":
+    type MyObj = object
+      field1 {.deserialize("test").}: bool
+      field2: bool
+
+    let val = !MyObj.fromJson("""{"test":true,"field2":true}""")
+    check val == MyObj(field1: true, field2: true)
+
+  test "fails to deserialize object field with wrong type":
+    type MyObj = object
+      field1: int
+      field2: bool
+
+    let r = MyObj.fromJson("""{"field1":true,"field2":true}""")
+    check r.isFailure
+    check r.error of UnexpectedKindError
+    check r.error.msg == "deserialization to int failed: expected {JInt} but got JBool"