refactor: use serde for json de/serialization instead of utils/json (#704)

* json > nim-serde bump

Should wait until serde is integrated into nim-ethers before making these changes as there will be less import exceptions required.

* bump nim-serde

* change func to proc due to chronicles side effects

* import serde into utils/json, use as proxy

import nim-serde into utils/json and use utils/json as a proxy for serde functions, including overloading `%` and `fromJson` for application types.

* update tests to use serde

* bump serde to latest

* remove testjson -- no longer needed

* bump serde in nimble

* updates to reconcile rebase with master

.gitmodules

@@ -209,3 +209,6 @@
 	url = https://github.com/codex-storage/codex-storage-proofs-circuits.git
 	ignore = untracked
 	branch = master
+[submodule "vendor/nim-serde"]
+	path = vendor/nim-serde
+	url = https://github.com/codex-storage/nim-serde.git

codex.nimble

@@ -22,6 +22,7 @@ requires "presto"
 requires "protobuf_serialization >= 0.2.0 & < 0.3.0"
 requires "questionable >= 0.10.13 & < 0.11.0"
 requires "secp256k1"
+requires "serde >= 1.0.0 & < 2.0.0"
 requires "stew"
 requires "upraises >= 0.1.0 & < 0.2.0"
 requires "toml_serialization"

codex/logutils.nim

@@ -17,16 +17,16 @@
 ## module, and specifying `formatIt`. If textlines log output and json log output
 ## need to be different, overload `formatIt` and specify a `LogFormat`. If json
 ## serialization is needed, it can be declared with a `%` proc. `logutils`
-## imports and exports `utils/json` which handles the de/serialization, examples
+## imports and exports `nim-serde` which handles the de/serialization, examples
 ## below. **Only `codex/logutils` needs to be imported.**
 ##
 ## Using `logutils` in the Codex codebase:
 ## - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils`
 ##     - most of `chronicles` is exported by `logutils`
-## - Instead of importing `std/json`, import `pkg/codex/utils/json`
+## - Instead of importing `std/json`, import `pkg/serde/json`
-##     - `std/json` is exported by `utils/json` which is exported by `logutils`
+##     - `std/json` is exported by `serde` which is exported by `logutils`
 ## - Instead of importing `pkg/nim-json-serialization`, import
-##   `pkg/codex/utils/json`
+##   `pkg/serde/json` or use codex-specific overloads by importing `utils/json`
 ##     - one of the goals is to remove the use of `nim-json-serialization`
 ##
 ## ```nim
@@ -54,7 +54,7 @@
 ## # chronicles json output
 ## {"lvl":"TRC","msg":"test","tid":14397405,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}}
 ## ```
-## In this case, `BlockAddress` is just an object, so `utils/json` can handle
+## In this case, `BlockAddress` is just an object, so `nim-serde` can handle
 ## serializing it without issue (only fields annotated with `{.serialize.}` will
 ## serialize (aka opt-in serialization)).
 ##
@@ -95,20 +95,19 @@ import pkg/chronicles except toJson, `%`
 from pkg/libp2p import Cid, MultiAddress, `$`
 import pkg/questionable
 import pkg/questionable/results
+import ./utils/json except formatIt # TODO: remove exception?
 import pkg/stew/byteutils
 import pkg/stint
 import pkg/upraises
 
-import ./utils/json
-
 export byteutils
 export chronicles except toJson, formatIt, `%`
 export questionable
 export sequtils
+export json except formatIt
 export strutils
 export sugar
 export upraises
-export json
 export results
 
 func shortLog*(long: string, ellipses = "*", start = 3, stop = 6): string =

codex/rest/json.nim

@@ -104,11 +104,11 @@ proc init*(_: type RestNodeId, id: NodeId): RestNodeId =
     id: id
   )
 
-func `%`*(obj: StorageRequest | Slot): JsonNode =
+proc `%`*(obj: StorageRequest | Slot): JsonNode =
   let jsonObj = newJObject()
   for k, v in obj.fieldPairs: jsonObj[k] = %v
   jsonObj["id"] = %(obj.id)
 
   return jsonObj
 
-func `%`*(obj: RestNodeId): JsonNode = % $obj.id
+proc `%`*(obj: RestNodeId): JsonNode = % $obj.id

codex/slots/builder/builder.nim

@@ -15,11 +15,11 @@ import std/sugar
 
 import pkg/libp2p
 import pkg/chronos
-import pkg/chronicles
 import pkg/questionable
 import pkg/questionable/results
 import pkg/constantine/math/io/io_fields
 
+import ../../logutils
 import ../../utils
 import ../../stores
 import ../../manifest

codex/slots/sampler/sampler.nim

@@ -9,12 +9,12 @@
 
 import std/sugar
 
-import pkg/chronicles
 import pkg/chronos
 import pkg/questionable
 import pkg/questionable/results
 import pkg/stew/arrayops
 
+import ../../logutils
 import ../../market
 import ../../blocktype as bt
 import ../../merkletree

codex/utils/json.nim

@@ -1,181 +1,16 @@
 
-import std/json except `%`, `%*`
-import std/macros
 import std/options
-import std/strutils
-import std/strformat
-import std/tables
 import std/typetraits
 from pkg/ethers import Address
 from pkg/libp2p import Cid, PeerId, SignedPeerRecord, MultiAddress, AddressInfo, init, `$`
 import pkg/contractabi
 import pkg/codexdht/discv5/node as dn
-import pkg/stew/byteutils
+import pkg/serde/json
-import pkg/stint
 import pkg/questionable/results
 import ../errors
 
-export json except `%`, `%*`
+export json
 
-type
-  SerializationError = object of CodexError
-  UnexpectedKindError = object of SerializationError
-
-template serialize* {.pragma.}
-
-proc newUnexpectedKindError(
-  expectedType: type,
-  expectedKinds: string,
-  json: JsonNode
-): ref UnexpectedKindError =
-  let kind = if json.isNil: "nil"
-             else: $json.kind
-  newException(UnexpectedKindError,
-    &"deserialization to {$expectedType} failed: expected {expectedKinds} " &
-    &"but got {kind}")
-
-proc newUnexpectedKindError(
-  expectedType: type,
-  expectedKinds: set[JsonNodeKind],
-  json: JsonNode
-): ref UnexpectedKindError =
-  newUnexpectedKindError(expectedType, $expectedKinds, json)
-
-proc newUnexpectedKindError(
-  expectedType: type,
-  expectedKind: JsonNodeKind,
-  json: JsonNode
-): ref UnexpectedKindError =
-  newUnexpectedKindError(expectedType, {expectedKind}, json)
-
-template expectJsonKind(
-  expectedType: type,
-  expectedKinds: set[JsonNodeKind],
-  json: JsonNode
-) =
-  if json.isNil or json.kind notin expectedKinds:
-    return failure(newUnexpectedKindError(expectedType, expectedKinds, json))
-
-template expectJsonKind(
-  expectedType: type,
-  expectedKind: JsonNodeKind,
-  json: JsonNode
-) =
-  expectJsonKind(expectedType, {expectedKind}, json)
-
-proc fromJson*(
-  T: type enum,
-  json: JsonNode
-): ?!T =
-  expectJsonKind(string, JString, json)
-  catch parseEnum[T](json.str)
-
-proc fromJson*(
-  _: type string,
-  json: JsonNode
-): ?!string =
-  if json.isNil:
-    let err = newException(ValueError, "'json' expected, but was nil")
-    return failure(err)
-  elif json.kind == JNull:
-    return success("null")
-  elif json.isNil or json.kind != JString:
-    return failure(newUnexpectedKindError(string, JString, json))
-  catch json.getStr
-
-proc fromJson*(
-  _: type bool,
-  json: JsonNode
-): ?!bool =
-  expectJsonKind(bool, JBool, json)
-  catch json.getBool
-
-proc fromJson*(
-  _: type int,
-  json: JsonNode
-): ?!int =
-  expectJsonKind(int, JInt, json)
-  catch json.getInt
-
-proc fromJson*[T: SomeInteger](
-  _: type T,
-  json: JsonNode
-): ?!T =
-  when T is uint|uint64 or (not defined(js) and int.sizeof == 4):
-    expectJsonKind(T, {JInt, JString}, json)
-    case json.kind
-    of JString:
-      let x = parseBiggestUInt(json.str)
-      return success cast[T](x)
-    else:
-      return success T(json.num)
-  else:
-    expectJsonKind(T, {JInt}, json)
-    return success cast[T](json.num)
-
-proc fromJson*[T: SomeFloat](
-  _: type T,
-  json: JsonNode
-): ?!T =
-  expectJsonKind(T, {JInt, JFloat, JString}, json)
-  if json.kind == JString:
-    case json.str
-    of "nan":
-      let b = NaN
-      return success T(b)
-      # dst = NaN # would fail some tests because range conversions would cause CT error
-      # in some cases; but this is not a hot-spot inside this branch and backend can optimize this.
-    of "inf":
-      let b = Inf
-      return success T(b)
-    of "-inf":
-      let b = -Inf
-      return success T(b)
-    else:
-      let err = newUnexpectedKindError(T, "'nan|inf|-inf'", json)
-      return failure(err)
-  else:
-    if json.kind == JFloat:
-      return success T(json.fnum)
-    else:
-      return success T(json.num)
-
-proc fromJson*(
-  _: type seq[byte],
-  json: JsonNode
-): ?!seq[byte] =
-  expectJsonKind(seq[byte], JString, json)
-  hexToSeqByte(json.getStr).catch
-
-proc fromJson*[N: static[int], T: array[N, byte]](
-  _: type T,
-  json: JsonNode
-): ?!T =
-  expectJsonKind(T, JString, json)
-  T.fromHex(json.getStr).catch
-
-proc fromJson*[T: distinct](
-  _: type T,
-  json: JsonNode
-): ?!T =
-  success T(? T.distinctBase.fromJson(json))
-
-proc fromJson*[N: static[int], T: StUint[N]](
-  _: type T,
-  json: JsonNode
-): ?!T =
-  expectJsonKind(T, JString, json)
-  catch parse(json.getStr, T)
-
-proc fromJson*[T](
-  _: type Option[T],
-  json: JsonNode
-): ?! Option[T] =
-  if json.isNil or json.kind == JNull:
-    return success(none T)
-  without val =? T.fromJson(json), error:
-    return failure(error)
-  success(val.some)
 
 proc fromJson*(
   _: type Cid,
@@ -184,123 +19,6 @@ proc fromJson*(
   expectJsonKind(Cid, JString, json)
   Cid.init(json.str).mapFailure
 
-proc fromJson*[T](
-  _: type seq[T],
-  json: JsonNode
-): ?! seq[T] =
-  expectJsonKind(seq[T], JArray, json)
-  var arr: seq[T] = @[]
-  for elem in json.elems:
-    arr.add(? T.fromJson(elem))
-  success arr
-
-proc fromJson*[T: ref object or object](
-  _: type T,
-  json: JsonNode
-): ?!T =
-  expectJsonKind(T, JObject, json)
-  var res = when type(T) is ref: T.new() else: T.default
-
-  # Leave this in, it's good for debugging:
-  # trace "deserializing object", to = $T, json
-  for name, value in fieldPairs(when type(T) is ref: res[] else: res):
-    if json{name} != nil:
-      without parsed =? type(value).fromJson(json{name}), e:
-        error "error deserializing field",
-          field = $T & "." & name,
-          json = json{name},
-          error = e.msg
-        return failure(e)
-      value = parsed
-  success(res)
-
-proc fromJson*[T: object](
-  _: type T,
-  bytes: seq[byte]
-): ?!T =
-  let json = ?catch parseJson(string.fromBytes(bytes))
-  T.fromJson(json)
-
-proc fromJson*[T: ref object](
-  _: type T,
-  bytes: seq[byte]
-): ?!T =
-  let json = ?catch parseJson(string.fromBytes(bytes))
-  T.fromJson(json)
-
-func `%`*(s: string): JsonNode = newJString(s)
-
-func `%`*(n: uint): JsonNode =
-  if n > cast[uint](int.high):
-    newJString($n)
-  else:
-    newJInt(BiggestInt(n))
-
-func `%`*(n: int): JsonNode = newJInt(n)
-
-func `%`*(n: BiggestUInt): JsonNode =
-  if n > cast[BiggestUInt](BiggestInt.high):
-    newJString($n)
-  else:
-    newJInt(BiggestInt(n))
-
-func `%`*(n: BiggestInt): JsonNode = newJInt(n)
-
-func `%`*(n: float): JsonNode =
-  if n != n: newJString("nan")
-  elif n == Inf: newJString("inf")
-  elif n == -Inf: newJString("-inf")
-  else: newJFloat(n)
-
-func `%`*(b: bool): JsonNode = newJBool(b)
-
-func `%`*(keyVals: openArray[tuple[key: string, val: JsonNode]]): JsonNode =
-  if keyVals.len == 0: return newJArray()
-  let jObj = newJObject()
-  for key, val in items(keyVals): jObj.fields[key] = val
-  jObj
-
-template `%`*(j: JsonNode): JsonNode = j
-
-func `%`*[T](table: Table[string, T]|OrderedTable[string, T]): JsonNode =
-  let jObj = newJObject()
-  for k, v in table: jObj[k] = ? %v
-  jObj
-
-func `%`*[T](opt: Option[T]): JsonNode =
-  if opt.isSome: %(opt.get) else: newJNull()
-
-func `%`*[T: object](obj: T): JsonNode =
-  let jsonObj = newJObject()
-  for name, value in obj.fieldPairs:
-    when value.hasCustomPragma(serialize):
-      jsonObj[name] = %value
-  jsonObj
-
-func `%`*[T: ref object](obj: T): JsonNode =
-  let jsonObj = newJObject()
-  for name, value in obj[].fieldPairs:
-    when value.hasCustomPragma(serialize):
-      jsonObj[name] = %(value)
-  jsonObj
-
-proc `%`*(o: enum): JsonNode = % $o
-
-func `%`*(stint: StInt|StUint): JsonNode = %stint.toString
-
-func `%`*(cstr: cstring): JsonNode = % $cstr
-
-func `%`*(arr: openArray[byte]): JsonNode = % arr.to0xHex
-
-func `%`*[T](elements: openArray[T]): JsonNode =
-  let jObj = newJArray()
-  for elem in elements: jObj.add(%elem)
-  jObj
-
-func `%`*[T: distinct](id: T): JsonNode =
-  type baseType = T.distinctBase
-  % baseType(id)
-
 func `%`*(cid: Cid): JsonNode = % $cid
 
 func `%`*(obj: PeerId): JsonNode = % $obj
@@ -314,36 +32,3 @@ func `%`*(obj: AddressInfo): JsonNode = % $obj.address
 func `%`*(obj: MultiAddress): JsonNode = % $obj
 
 func `%`*(address: ethers.Address): JsonNode = % $address
-
-func toJson*[T](item: T): string = $(%item)
-
-proc toJsnImpl(x: NimNode): NimNode =
-  case x.kind
-  of nnkBracket: # array
-    if x.len == 0: return newCall(bindSym"newJArray")
-    result = newNimNode(nnkBracket)
-    for i in 0 ..< x.len:
-      result.add(toJsnImpl(x[i]))
-    result = newCall(bindSym("%", brOpen), result)
-  of nnkTableConstr: # object
-    if x.len == 0: return newCall(bindSym"newJObject")
-    result = newNimNode(nnkTableConstr)
-    for i in 0 ..< x.len:
-      x[i].expectKind nnkExprColonExpr
-      result.add newTree(nnkExprColonExpr, x[i][0], toJsnImpl(x[i][1]))
-    result = newCall(bindSym("%", brOpen), result)
-  of nnkCurly: # empty object
-    x.expectLen(0)
-    result = newCall(bindSym"newJObject")
-  of nnkNilLit:
-    result = newCall(bindSym"newJNull")
-  of nnkPar:
-    if x.len == 1: result = toJsnImpl(x[0])
-    else: result = newCall(bindSym("%", brOpen), x)
-  else:
-    result = newCall(bindSym("%", brOpen), x)
-
-macro `%*`*(x: untyped): JsonNode =
-  ## Convert an expression to a JsonNode directly, without having to specify
-  ## `%` for every element.
-  result = toJsnImpl(x)

tests/codex/sales/testslotqueue.nim

@@ -1,5 +1,4 @@
 import std/sequtils
-import pkg/chronicles
 import pkg/chronos
 import pkg/datastore
 import pkg/questionable

tests/codex/slots/backends/testcircomcompat.nim

@@ -8,11 +8,11 @@ import ../../../asynctest
 import pkg/chronos
 import pkg/poseidon2
 import pkg/datastore
+import pkg/serde/json
 
 import pkg/codex/slots {.all.}
 import pkg/codex/slots/types {.all.}
 import pkg/codex/merkletree
-import pkg/codex/utils/json
 import pkg/codex/codextypes
 import pkg/codex/manifest
 import pkg/codex/stores
@@ -33,7 +33,7 @@ suite "Test Circom Compat Backend - control inputs":
   setup:
     let
       inputData = readFile("tests/circuits/fixtures/input.json")
-      inputJson = parseJson(inputData)
+      inputJson = !JsonNode.parse(inputData)
 
     proofInputs = Poseidon2Hash.jsonToProofInput(inputJson)
     circom = CircomCompat.init(r1cs, wasm, zkey)

tests/codex/slots/sampler/testsampler.nim

@@ -35,7 +35,7 @@ suite "Test Sampler - control samples":
 
   setup:
     inputData = readFile("tests/circuits/fixtures/input.json")
-    inputJson = parseJson(inputData)
+    inputJson = !JsonNode.parse(inputData)
     proofInput = Poseidon2Hash.jsonToProofInput(inputJson)
 
   test "Should verify control samples":

tests/codex/slots/sampler/testutils.nim

@@ -18,9 +18,9 @@ import pkg/codex/contracts/requests
 import pkg/codex/contracts
 import pkg/codex/merkletree
 import pkg/codex/stores/cachestore
-
 import pkg/codex/slots/types
 import pkg/codex/slots/sampler/utils
+import pkg/codex/utils/json
 
 import ../backends/helpers
 import ../../helpers
@@ -38,7 +38,7 @@ asyncchecksuite "Test proof sampler utils":
 
   setup:
     inputData = readFile("tests/circuits/fixtures/input.json")
-    inputJson = parseJson(inputData)
+    inputJson = !JsonNode.parse(inputData)
     proofInput = Poseidon2Hash.jsonToProofInput(inputJson)
 
   test "Extract low bits":

tests/codex/testlogutils.nim

@@ -7,6 +7,7 @@ import pkg/codex/contracts/requests
 import pkg/codex/logutils
 import pkg/codex/purchasing/purchaseid
 import pkg/codex/units
+import pkg/codex/utils/json
 import pkg/libp2p/cid
 import pkg/libp2p/multiaddress
 import pkg/questionable
@@ -61,8 +62,8 @@ checksuite "Test logging output":
     outputLines.contains(toFind)
 
   template loggedJson(prop, expected): auto =
-    let json = $ parseJson(outputJson){prop}
+    let jsonVal = !JsonNode.parse(outputJson)
-    json == expected
+    $ jsonVal{prop} == expected
 
   template log(val) =
     testlines.trace "test", val

tests/codex/testutils.nim

@@ -1,4 +1,3 @@
-import ./utils/testjson
 import ./utils/testoptionalcast
 import ./utils/testkeyutils
 import ./utils/testasyncstatemachine

tests/codex/utils/testjson.nim

@@ -1,345 +0,0 @@
-import std/math
-import std/options
-import std/strformat
-import std/strutils
-import std/unittest
-import pkg/stew/byteutils
-import pkg/stint
-import pkg/codex/contracts/requests
-from pkg/codex/rest/json import RestPurchase
-import pkg/codex/logutils
-import pkg/codex/utils/json as utilsjson
-import pkg/questionable
-import pkg/questionable/results
-import pkg/libp2p
-import ../helpers
-
-checksuite "json serialization":
-  var request: StorageRequest
-  var requestJson: JsonNode
-
-  func flatten(s: string): string =
-    s.replace(" ")
-     .replace("\n")
-
-  setup:
-    request = StorageRequest(
-      client: Address.init("0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174").get(),
-      ask: StorageAsk(
-        slots: 4,
-        slotSize: (1 * 1024 * 1024 * 1024).u256, # 1 Gigabyte
-        duration: (10 * 60 * 60).u256, # 10 hours
-        collateral: 200.u256,
-        proofProbability: 4.u256, # require a proof roughly once every 4 periods
-        reward: 84.u256,
-        maxSlotLoss: 2 # 2 slots can be freed without data considered to be lost
-      ),
-      content: StorageContent(
-        cid: "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob",
-        merkleRoot: array[32, byte].fromHex("0xc066dd7e405de5a795ce1e765209cfaa6de6c74829c607d1a2fe53107107a298")
-      ),
-      expiry: 1691545330.u256,
-      nonce: Nonce array[32, byte].fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1")
-    )
-    requestJson = """{
-                      "client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174",
-                      "ask": {
-                        "slots": 4,
-                        "slotSize": "1073741824",
-                        "duration": "36000",
-                        "proofProbability": "4",
-                        "reward": "84",
-                        "collateral": "200",
-                        "maxSlotLoss": 2
-                      },
-                      "content": {
-                        "cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob",
-                        "merkleRoot": "0xc066dd7e405de5a795ce1e765209cfaa6de6c74829c607d1a2fe53107107a298"
-                      },
-                      "expiry": "1691545330",
-                      "nonce": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"
-                    }""".parseJson
-
-  test "serializes UInt256 to non-hex string representation":
-    check (% 100000.u256) == newJString("100000")
-
-  test "serializes sequence to an array":
-    let json = % @[1, 2, 3]
-    let expected = "[1,2,3]"
-    check $json == expected
-
-  test "serializes Option[T] when has a value":
-    let obj = %(some 1)
-    let expected = "1"
-    check $obj == expected
-
-  test "serializes Option[T] when doesn't have a value":
-    let obj = %(none int)
-    let expected = "null"
-    check $obj == expected
-
-  test "serializes uints int.high or smaller":
-    let largeUInt: uint = uint(int.high)
-    check %largeUInt == newJInt(BiggestInt(largeUInt))
-
-  test "serializes large uints":
-    let largeUInt: uint = uint(int.high) + 1'u
-    check %largeUInt == newJString($largeUInt)
-
-
-  test "serializes Inf float":
-    check %Inf == newJString("inf")
-
-  test "serializes -Inf float":
-    check %(-Inf) == newJString("-inf")
-
-  test "deserializes NaN float":
-    check %NaN == newJString("nan")
-
-  test "can construct json objects with %*":
-    type MyObj = object
-      mystring {.serialize.}: string
-      myint {.serialize.}: int
-      myoption {.serialize.}: ?bool
-
-    let myobj = MyObj(mystring: "abc", myint: 123, myoption: some true)
-    let mystuint = 100000.u256
-
-    let json = %*{
-      "myobj": myobj,
-      "mystuint": mystuint
-    }
-
-    let expected = """{
-                        "myobj": {
-                          "mystring": "abc",
-                          "myint": 123,
-                          "myoption": true
-                        },
-                        "mystuint": "100000"
-                      }""".flatten
-
-    check $json == expected
-
-  test "only serializes marked fields":
-    type MyObj = object
-      mystring {.serialize.}: string
-      myint {.serialize.}: int
-      mybool: bool
-
-    let obj = % MyObj(mystring: "abc", myint: 1, mybool: true)
-
-    let expected = """{
-                        "mystring": "abc",
-                        "myint": 1
-                      }""".flatten
-
-    check $obj == expected
-
-  test "serializes ref objects":
-    type MyRef = ref object
-      mystring {.serialize.}: string
-      myint {.serialize.}: int
-
-    let obj = % MyRef(mystring: "abc", myint: 1)
-
-    let expected = """{
-                        "mystring": "abc",
-                        "myint": 1
-                      }""".flatten
-
-    check $obj == expected
-
-  test "serializes RestPurchase":
-    let request = % RestPurchase(
-      request: some request,
-      requestId: RequestId.fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"),
-      error: some "error",
-      state: "state"
-    )
-    let expected = """{
-                        "requestId": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1",
-                        "request": {
-                          "client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174",
-                          "ask": {
-                            "slots": 4,
-                            "slotSize": "1073741824",
-                            "duration": "36000",
-                            "proofProbability": "4",
-                            "reward": "84",
-                            "collateral": "200",
-                            "maxSlotLoss": 2
-                          },
-                          "content": {
-                            "cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob"
-                          },
-                          "expiry": "1691545330"
-                        },
-                        "state": "state",
-                        "error": "error"
-                      }""".flatten
-    check $request == expected
-
-  test "serializes StorageRequest":
-    let expected = """{
-                        "client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174",
-                        "ask": {
-                          "slots": 4,
-                          "slotSize": "1073741824",
-                          "duration": "36000",
-                          "proofProbability": "4",
-                          "reward": "84",
-                          "collateral": "200",
-                          "maxSlotLoss": 2
-                        },
-                        "content": {
-                          "cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob"
-                        },
-                        "expiry": "1691545330"
-                      }""".flatten
-    check request.toJson == expected
-
-  test "deserialize enum":
-    let json = newJString($CidVersion.CIDv1)
-    check !CidVersion.fromJson(json) == CIDv1
-
-  test "deserializes UInt256 from non-hex string representation":
-    let json = newJString("100000")
-    check !UInt256.fromJson(json) == 100000.u256
-
-  test "deserializes Option[T] when has a value":
-    let json = newJInt(1)
-    check (!fromJson(?int, json) == some 1)
-
-  test "deserializes Option[T] when doesn't have a value":
-    let json = newJNull()
-    check !fromJson(?int, json) == none int
-
-  test "deserializes float":
-    let json = newJFloat(1.234)
-    check !float.fromJson(json) == 1.234
-
-  test "deserializes Inf float":
-    let json = newJString("inf")
-    check !float.fromJson(json) == Inf
-
-  test "deserializes -Inf float":
-    let json = newJString("-inf")
-    check !float.fromJson(json) == -Inf
-
-  test "deserializes NaN float":
-    let json = newJString("nan")
-    check float.fromJson(json).get.isNaN
-
-  test "deserializes array to sequence":
-    let expected = @[1, 2, 3]
-    let json = "[1,2,3]".parseJson
-    check !seq[int].fromJson(json) == expected
-
-  test "deserializes uints int.high or smaller":
-    let largeUInt: uint = uint(int.high)
-    let json = newJInt(BiggestInt(largeUInt))
-    check !uint.fromJson(json) == largeUInt
-
-  test "deserializes large uints":
-    let largeUInt: uint = uint(int.high) + 1'u
-    let json = newJString($BiggestUInt(largeUInt))
-    check !uint.fromJson(json) == largeUInt
-
-  test "can deserialize json objects":
-    type MyObj = object
-      mystring: string
-      myint: int
-      myoption: ?bool
-
-    let expected = MyObj(mystring: "abc", myint: 123, myoption: some true)
-
-    let json = parseJson("""{
-                              "mystring": "abc",
-                              "myint": 123,
-                              "myoption": true
-                            }""")
-    check !MyObj.fromJson(json) == expected
-
-  test "ignores serialize pragma when deserializing":
-    type MyObj = object
-      mystring {.serialize.}: string
-      mybool: bool
-
-    let expected = MyObj(mystring: "abc", mybool: true)
-
-    let json = parseJson("""{
-                              "mystring": "abc",
-                              "mybool": true
-                            }""")
-
-    check !MyObj.fromJson(json) == expected
-
-  test "deserializes objects with extra fields":
-    type MyObj = object
-      mystring: string
-      mybool: bool
-
-    let expected = MyObj(mystring: "abc", mybool: true)
-
-    let json = """{
-                    "mystring": "abc",
-                    "mybool": true,
-                    "extra": "extra"
-                  }""".parseJson
-    check !MyObj.fromJson(json) == expected
-
-  test "deserializes objects with less fields":
-    type MyObj = object
-      mystring: string
-      mybool: bool
-
-    let expected = MyObj(mystring: "abc", mybool: false)
-
-    let json = """{
-                    "mystring": "abc"
-                  }""".parseJson
-    check !MyObj.fromJson(json) == expected
-
-  test "deserializes ref objects":
-    type MyRef = ref object
-      mystring: string
-      myint: int
-
-    let expected = MyRef(mystring: "abc", myint: 1)
-
-    let json = """{
-                    "mystring": "abc",
-                    "myint": 1
-                  }""".parseJson
-
-    let deserialized = !MyRef.fromJson(json)
-    check deserialized.mystring == expected.mystring
-    check deserialized.myint == expected.myint
-
-  test "deserializes Cid":
-    let
-      jCid = newJString("zdj7Wakya26ggQWkvMdHYFcPgZ7Qh2HdMooQDDFDHkk4uHS14")
-      cid = "zdj7Wakya26ggQWkvMdHYFcPgZ7Qh2HdMooQDDFDHkk4uHS14"
-
-    check:
-      !Cid.fromJson(jCid) == !Cid.init(cid).mapFailure
-
-  test "deserializes StorageRequest":
-    check !StorageRequest.fromJson(requestJson) == request
-
-  test "deserializes RestPurchase":
-    let json = """{
-                    "requestId": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1",
-                    "state": "state",
-                    "error": "error"
-                  }""".parseJson
-    json["request"] = requestJson
-
-    let expected = RestPurchase(
-      requestId: RequestId.fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"),
-      state: "state",
-      error: some "error",
-      request: some request
-    )
-    check !RestPurchase.fromJson(json) == expected

tests/integration/codexclient.nim

@@ -19,9 +19,9 @@ type CodexClient* = ref object
 proc new*(_: type CodexClient, baseurl: string): CodexClient =
   CodexClient(http: newHttpClient(), baseurl: baseurl)
 
-proc info*(client: CodexClient): JsonNode =
+proc info*(client: CodexClient): ?!JsonNode =
   let url = client.baseurl & "/debug/info"
-  client.http.getContent(url).parseJson()
+  JsonNode.parse( client.http.getContent(url) )
 
 proc setLogLevel*(client: CodexClient, level: string) =
   let url = client.baseurl & "/debug/chronicles/loglevel?level=" & level
@@ -52,8 +52,7 @@ proc list*(client: CodexClient): ?!seq[RestContent] =
   if response.status != "200 OK":
     return failure(response.status)
 
-  let json = ? parseJson(response.body).catch
+  seq[RestContent].fromJson(response.body)
-  seq[RestContent].fromJson(json)
 
 proc space*(client: CodexClient): ?!RestRepoStore =
   let url = client.baseurl & "/space"
@@ -62,8 +61,7 @@ proc space*(client: CodexClient): ?!RestRepoStore =
   if response.status != "200 OK":
     return failure(response.status)
 
-  let json = ? parseJson(response.body).catch
+  RestRepoStore.fromJson(response.body)
-  RestRepoStore.fromJson(json)
 
 proc requestStorageRaw*(
     client: CodexClient,
@@ -116,8 +114,7 @@ proc getPurchase*(client: CodexClient, purchaseId: PurchaseId): ?!RestPurchase =
   let url = client.baseurl & "/storage/purchases/" & purchaseId.toHex
   try:
     let body = client.http.getContent(url)
-    let json = ? parseJson(body).catch
+    return RestPurchase.fromJson(body)
-    return RestPurchase.fromJson(json)
   except CatchableError as e:
     return failure e.msg
 
@@ -125,16 +122,14 @@ proc getSalesAgent*(client: CodexClient, slotId: SlotId): ?!RestSalesAgent =
   let url = client.baseurl & "/sales/slots/" & slotId.toHex
   try:
     let body = client.http.getContent(url)
-    let json = ? parseJson(body).catch
+    return RestSalesAgent.fromJson(body)
-    return RestSalesAgent.fromJson(json)
   except CatchableError as e:
     return failure e.msg
 
 proc getSlots*(client: CodexClient): ?!seq[Slot] =
   let url = client.baseurl & "/sales/slots"
   let body = client.http.getContent(url)
-  let json = ? parseJson(body).catch
+  seq[Slot].fromJson(body)
-  seq[Slot].fromJson(json)
 
 proc postAvailability*(
     client: CodexClient,
@@ -151,13 +146,13 @@ proc postAvailability*(
   }
   let response = client.http.post(url, $json)
   doAssert response.status == "200 OK", "expected 200 OK, got " & response.status & ", body: " & response.body
-  Availability.fromJson(response.body.parseJson)
+  Availability.fromJson(response.body)
 
 proc getAvailabilities*(client: CodexClient): ?!seq[Availability] =
   ## Call sales availability REST endpoint
   let url = client.baseurl & "/sales/availability"
   let body = client.http.getContent(url)
-  seq[Availability].fromJson(parseJson(body))
+  seq[Availability].fromJson(body)
 
 proc close*(client: CodexClient) =
   client.http.close()

tests/integration/multinodes.nim

@@ -288,7 +288,11 @@ template multinodesuite*(name: string, body: untyped) =
                           node: node
                         )
             if clients().len == 1:
-              bootstrap = CodexProcess(node).client.info()["spr"].getStr()
+              without ninfo =? CodexProcess(node).client.info():
+                # raise CatchableError instead of Defect (with .get or !) so we
+                # can gracefully shutdown and prevent zombies
+                raiseMultiNodeSuiteError "Failed to get node info"
+              bootstrap = ninfo["spr"].getStr()
 
       if var providers =? nodeConfigs.providers:
         failAndTeardownOnError "failed to start provider nodes":

tests/integration/testIntegration.nim

@@ -26,7 +26,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
     await ethProvider.advanceTime(1.u256)
 
   test "nodes can print their peer information":
-    check client1.info() != client2.info()
+    check !client1.info() != !client2.info()
 
   test "nodes can set chronicles log level":
     client1.setLogLevel("DEBUG;TRACE:codex")

tests/integration/twonodes.nim

@@ -52,7 +52,7 @@ template twonodessuite*(name: string, debug1, debug2: string, body) =
       node1 = startNode(node1Args, debug = debug1)
       node1.waitUntilStarted()
 
-      let bootstrap = client1.info()["spr"].getStr()
+      let bootstrap = (!client1.info()["spr"]).getStr()
 
       var node2Args = @[
         "--api-port=8081",

tests/testTaiko.nim

@@ -29,7 +29,7 @@ suite "Taiko L2 Integration Tests":
     ])
     node1.waitUntilStarted()
 
-    let bootstrap = node1.client.info()["spr"].getStr()
+    let bootstrap = (!node1.client.info())["spr"].getStr()
 
     node2 = startNode([
       "--data-dir=" & createTempDir("", ""),

vendor/nim-serde

@@ -0,0 +1 @@
+Subproject commit a261c3d214b87d9de0338d6cfd5d134da03d2dc3