Improve integration testing client (CodexClient) and json serialization (#514)

* Improve integration testing client (CodexClient) and json serialization

The current client used for integration testing against the REST endpoints for Codex accepts and passes primitive types. This caused a hard to diagnose bug where a `uint` was not being deserialized correctly.

In addition, the json de/serializing done between the CodexClient and REST client was not easy to read and was not tested.

These changes bring non-primitive types to most of the CodexClient functions, allowing us to lean on the compiler to ensure we're providing correct typings. More importantly, a json de/serialization util was created as a drop-in replacement for the std/json lib, with the main two differences being that field serialization is opt-in (instead of opt-out as in the case of json_serialization) and serialization errors are captured and logged, making debugging serialization issues much easier.

* Update integration test to use nodes=2 and tolerance=1

* clean up

codex/contracts/requests.nim

@@ -1,4 +1,5 @@
 import std/hashes
+import std/typetraits
 import pkg/contractabi
 import pkg/nimcrypto
 import pkg/ethers/fields
@@ -6,26 +7,27 @@ import pkg/questionable/results
 import pkg/stew/byteutils
 import pkg/json_serialization
 import pkg/upraises
+import ../utils/json
 
 export contractabi
 
 type
   StorageRequest* = object
-    client*: Address
+    client* {.serialize.}: Address
-    ask*: StorageAsk
+    ask* {.serialize.}: StorageAsk
-    content*: StorageContent
+    content* {.serialize.}: StorageContent
-    expiry*: UInt256
+    expiry* {.serialize.}: UInt256
     nonce*: Nonce
   StorageAsk* = object
-    slots*: uint64
+    slots* {.serialize.}: uint64
-    slotSize*: UInt256
+    slotSize* {.serialize.}: UInt256
-    duration*: UInt256
+    duration* {.serialize.}: UInt256
-    proofProbability*: UInt256
+    proofProbability* {.serialize.}: UInt256
-    reward*: UInt256
+    reward* {.serialize.}: UInt256
-    collateral*: UInt256
+    collateral* {.serialize.}: UInt256
-    maxSlotLoss*: uint64
+    maxSlotLoss* {.serialize.}: uint64
   StorageContent* = object
-    cid*: string
+    cid* {.serialize.}: string
     erasure*: StorageErasure
     por*: StoragePoR
   StorageErasure* = object
@@ -35,8 +37,8 @@ type
     publicKey*: seq[byte]
     name*: seq[byte]
   Slot* = object
-    request*: StorageRequest
+    request* {.serialize.}: StorageRequest
-    slotIndex*: UInt256
+    slotIndex* {.serialize.}: UInt256
   SlotId* = distinct array[32, byte]
   RequestId* = distinct array[32, byte]
   Nonce* = distinct array[32, byte]
@@ -75,6 +77,10 @@ proc fromHex*(T: type SlotId, hex: string): T =
 proc fromHex*(T: type Nonce, hex: string): T =
   T array[32, byte].fromHex(hex)
 
+proc fromHex*[T: distinct](_: type T, hex: string): T =
+  type baseType = T.distinctBase
+  T baseType.fromHex(hex)
+
 func fromTuple(_: type StorageRequest, tupl: tuple): StorageRequest =
   StorageRequest(
     client: tupl[0],

codex/node.nim

@@ -254,7 +254,7 @@ proc requestStorage*(
   ## - Run the PoR setup on the erasure dataset
   ## - Call into the marketplace and purchasing contracts
   ##
-  trace "Received a request for storage!", cid, duration, nodes, tolerance, reward
+  trace "Received a request for storage!", cid, duration, nodes, tolerance, reward, proofProbability, collateral, expiry
 
   without contracts =? self.contracts.client:
     trace "Purchasing not available"

codex/rest/api.nim

@@ -34,7 +34,7 @@ import pkg/codexdht/discv5/node as dn
 import ../node
 import ../blocktype
 import ../conf
-import ../contracts
+import ../contracts except `%*`, `%` # imported from contracts/marketplace (exporting ethers)
 import ../streams
 
 import ./coders
@@ -361,10 +361,15 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
 
       let body = await request.getBody()
 
-      without availability =? Availability.fromJson(body), error:
+      without restAv =? RestAvailability.fromJson(body), error:
         return RestApiResponse.error(Http400, error.msg)
 
       let reservations = contracts.sales.context.reservations
+      # assign id to availability via init
+      let availability = Availability.init(restAv.size,
+                                           restAv.duration,
+                                           restAv.minPrice,
+                                           restAv.maxCollateral)
 
       if not reservations.hasAvailable(availability.size.truncate(uint)):
         return RestApiResponse.error(Http422, "Not enough storage quota")
@@ -389,7 +394,12 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
       without purchase =? contracts.purchasing.getPurchase(id):
         return RestApiResponse.error(Http404)
 
-      let json = %purchase
+      let json = % RestPurchase(
+        state: purchase.state |? "none",
+        error: purchase.error.?msg,
+        request: purchase.request,
+        requestId: purchase.requestId
+      )
 
       return RestApiResponse.response($json, contentType="application/json")
 

codex/rest/json.nim

@@ -1,67 +1,33 @@
-import std/json
+import pkg/questionable
-import std/strutils
-import pkg/stew/byteutils
 import pkg/questionable/results
+import pkg/stew/byteutils
 import ../sales
 import ../purchasing
-import ../utils/stintutils
+import ../utils/json
 
 export json
 
 type
   StorageRequestParams* = object
-    duration*: UInt256
+    duration* {.serialize.}: UInt256
-    proofProbability*: UInt256
+    proofProbability* {.serialize.}: UInt256
-    reward*: UInt256
+    reward* {.serialize.}: UInt256
-    collateral*: UInt256
+    collateral* {.serialize.}: UInt256
-    expiry*: ?UInt256
+    expiry* {.serialize.}: ?UInt256
-    nodes*: ?uint
+    nodes* {.serialize.}: ?uint
-    tolerance*: ?uint
+    tolerance* {.serialize.}: ?uint
 
-proc fromJson*(
+  RestPurchase* = object
-    _: type Availability,
+    requestId* {.serialize.}: RequestId
-    bytes: seq[byte]
+    request* {.serialize.}: ?StorageRequest
-): ?!Availability =
+    state* {.serialize.}: string
-  let json = ?catch parseJson(string.fromBytes(bytes))
+    error* {.serialize.}: ?string
-  let size = ?catch UInt256.fromDecimal(json["size"].getStr)
-  let duration = ?catch UInt256.fromDecimal(json["duration"].getStr)
-  let minPrice = ?catch UInt256.fromDecimal(json["minPrice"].getStr)
-  let maxCollateral = ?catch UInt256.fromDecimal(json["maxCollateral"].getStr)
-  success Availability.init(size, duration, minPrice, maxCollateral)
 
-proc fromJson*(
+  RestAvailability* = object
-    _: type StorageRequestParams,
+    size* {.serialize.}: UInt256
-    bytes: seq[byte]
+    duration* {.serialize.}: UInt256
-): ?! StorageRequestParams =
+    minPrice* {.serialize.}: UInt256
-  let json = ?catch parseJson(string.fromBytes(bytes))
+    maxCollateral* {.serialize.}: UInt256
-  let duration = ?catch UInt256.fromDecimal(json["duration"].getStr)
-  let proofProbability = ?catch UInt256.fromDecimal(json["proofProbability"].getStr)
-  let reward = ?catch UInt256.fromDecimal(json["reward"].getStr)
-  let collateral = ?catch UInt256.fromDecimal(json["collateral"].getStr)
-  let expiry = UInt256.fromDecimal(json["expiry"].getStr).catch.option
-  let nodes = parseUInt(json["nodes"].getStr).catch.option
-  let tolerance = parseUInt(json["tolerance"].getStr).catch.option
-  success StorageRequestParams(
-    duration: duration,
-    proofProbability: proofProbability,
-    reward: reward,
-    collateral: collateral,
-    expiry: expiry,
-    nodes: nodes,
-    tolerance: tolerance
-  )
-
-func `%`*(address: Address): JsonNode =
-  % $address
-
-func `%`*(stint: StInt|StUint): JsonNode=
-  %(stint.toString)
-
-func `%`*(arr: openArray[byte]): JsonNode =
-  %("0x" & arr.toHex)
-
-func `%`*(id: RequestId | SlotId | Nonce | AvailabilityId): JsonNode =
-  % id.toArray
 
 func `%`*(obj: StorageRequest | Slot): JsonNode =
   let jsonObj = newJObject()
@@ -69,11 +35,3 @@ func `%`*(obj: StorageRequest | Slot): JsonNode =
   jsonObj["id"] = %(obj.id)
 
   return jsonObj
-
-func `%`*(purchase: Purchase): JsonNode =
-  %*{
-    "state": purchase.state |? "none",
-    "error": purchase.error.?msg,
-    "request": purchase.request,
-    "requestId": purchase.requestId
-  }

codex/sales/reservations.nim

@@ -19,6 +19,7 @@ import pkg/stew/byteutils
 import pkg/nimcrypto
 import pkg/questionable
 import pkg/questionable/results
+import ../utils/json
 
 push: {.upraises: [].}
 
@@ -34,11 +35,11 @@ logScope:
 type
   AvailabilityId* = distinct array[32, byte]
   Availability* = object
-    id*: AvailabilityId
+    id* {.serialize.}: AvailabilityId
-    size*: UInt256
+    size* {.serialize.}: UInt256
-    duration*: UInt256
+    duration* {.serialize.}: UInt256
-    minPrice*: UInt256
+    minPrice* {.serialize.}: UInt256
-    maxCollateral*: UInt256
+    maxCollateral* {.serialize.}: UInt256
     used*: bool
   Reservations* = ref object
     repo: RepoStore

codex/utils/json.nim

@@ -0,0 +1,339 @@
+
+import std/json except `%`, `%*`
+import std/macros
+import std/options
+import std/strutils
+import std/strformat
+import std/tables
+import std/typetraits
+import pkg/chronicles
+from pkg/libp2p import Cid, init
+import pkg/contractabi
+import pkg/stew/byteutils
+import pkg/stint
+import pkg/questionable/results
+import ../errors
+
+export json except `%`, `%*`
+
+logScope:
+  topics = "json serialization"
+
+type
+  SerializationError = object of CodexError
+  UnexpectedKindError = object of SerializationError
+
+template serialize* {.pragma.}
+
+proc newUnexpectedKindError(
+  expectedType: type,
+  expectedKinds: string,
+  json: JsonNode
+): ref UnexpectedKindError =
+  let kind = if json.isNil: "nil"
+             else: $json.kind
+  newException(UnexpectedKindError,
+    &"deserialization to {$expectedType} failed: expected {expectedKinds} " &
+    &"but got {kind}")
+
+proc newUnexpectedKindError(
+  expectedType: type,
+  expectedKinds: set[JsonNodeKind],
+  json: JsonNode
+): ref UnexpectedKindError =
+  newUnexpectedKindError(expectedType, $expectedKinds, json)
+
+proc newUnexpectedKindError(
+  expectedType: type,
+  expectedKind: JsonNodeKind,
+  json: JsonNode
+): ref UnexpectedKindError =
+  newUnexpectedKindError(expectedType, {expectedKind}, json)
+
+template expectJsonKind(
+  expectedType: type,
+  expectedKinds: set[JsonNodeKind],
+  json: JsonNode
+) =
+  if json.isNil or json.kind notin expectedKinds:
+    return failure(newUnexpectedKindError(expectedType, expectedKinds, json))
+
+template expectJsonKind(
+  expectedType: type,
+  expectedKind: JsonNodeKind,
+  json: JsonNode
+) =
+  expectJsonKind(expectedType, {expectedKind}, json)
+
+proc fromJson*(
+  _: type string,
+  json: JsonNode
+): ?!string =
+  if json.isNil:
+    let err = newException(ValueError, "'json' expected, but was nil")
+    return failure(err)
+  elif json.kind == JNull:
+    return success("null")
+  elif json.isNil or json.kind != JString:
+    return failure(newUnexpectedKindError(string, JString, json))
+  catch json.getStr
+
+proc fromJson*(
+  _: type bool,
+  json: JsonNode
+): ?!bool =
+  expectJsonKind(bool, JBool, json)
+  catch json.getBool
+
+proc fromJson*(
+  _: type int,
+  json: JsonNode
+): ?!int =
+  expectJsonKind(int, JInt, json)
+  catch json.getInt
+
+proc fromJson*[T: SomeInteger](
+  _: type T,
+  json: JsonNode
+): ?!T =
+  when T is uint|uint64 or (not defined(js) and int.sizeof == 4):
+    expectJsonKind(T, {JInt, JString}, json)
+    case json.kind
+    of JString:
+      let x = parseBiggestUInt(json.str)
+      return success cast[T](x)
+    else:
+      return success T(json.num)
+  else:
+    expectJsonKind(T, {JInt}, json)
+    return success cast[T](json.num)
+
+proc fromJson*[T: SomeFloat](
+  _: type T,
+  json: JsonNode
+): ?!T =
+  expectJsonKind(T, {JInt, JFloat, JString}, json)
+  if json.kind == JString:
+    case json.str
+    of "nan":
+      let b = NaN
+      return success T(b)
+      # dst = NaN # would fail some tests because range conversions would cause CT error
+      # in some cases; but this is not a hot-spot inside this branch and backend can optimize this.
+    of "inf":
+      let b = Inf
+      return success T(b)
+    of "-inf":
+      let b = -Inf
+      return success T(b)
+    else:
+      let err = newUnexpectedKindError(T, "'nan|inf|-inf'", json)
+      return failure(err)
+  else:
+    if json.kind == JFloat:
+      return success T(json.fnum)
+    else:
+      return success T(json.num)
+
+proc fromJson*(
+  _: type seq[byte],
+  json: JsonNode
+): ?!seq[byte] =
+  expectJsonKind(seq[byte], JString, json)
+  hexToSeqByte(json.getStr).catch
+
+proc fromJson*[N: static[int], T: array[N, byte]](
+  _: type T,
+  json: JsonNode
+): ?!T =
+  expectJsonKind(T, JString, json)
+  T.fromHex(json.getStr).catch
+
+proc fromJson*[T: distinct](
+  _: type T,
+  json: JsonNode
+): ?!T =
+  success T(? T.distinctBase.fromJson(json))
+
+proc fromJson*[N: static[int], T: StUint[N]](
+  _: type T,
+  json: JsonNode
+): ?!T =
+  expectJsonKind(T, JString, json)
+  catch parse(json.getStr, T)
+
+proc fromJson*[T](
+  _: type Option[T],
+  json: JsonNode
+): ?! Option[T] =
+  if json.isNil or json.kind == JNull:
+    return success(none T)
+  without val =? T.fromJson(json), error:
+    return failure(error)
+  success(val.some)
+
+proc fromJson*(
+  _: type Cid,
+  json: JsonNode
+): ?!Cid =
+  expectJsonKind(Cid, JString, json)
+  Cid.init($json).mapFailure
+
+proc fromJson*[T](
+  _: type seq[T],
+  json: JsonNode
+): ?! seq[T] =
+  expectJsonKind(seq[T], JArray, json)
+  var arr: seq[T] = @[]
+  for elem in json.elems:
+    arr.add(? T.fromJson(elem))
+  success arr
+
+proc fromJson*[T: object](
+  _: type T,
+  json: JsonNode
+): ?!T =
+  expectJsonKind(T, JObject, json)
+  var res = T.default
+  # Leave this in, it's good for debugging:
+  # trace "deserializing object", to = $T, json
+  for name, value in fieldPairs(res):
+    if json{name} != nil:
+      without parsed =? type(value).fromJson(json{name}), e:
+        error "error deserializing field",
+          field = $T & "." & name,
+          json = json{name},
+          error = e.msg
+        return failure(e)
+      value = parsed
+  success(res)
+
+proc fromJson*[T: ref object](
+  _: type T,
+  json: JsonNode
+): ?!T =
+  expectJsonKind(T, JObject, json)
+  var res = T.new()
+  # Leave this in, it's good for debugging:
+  # trace "deserializing object", to = $T, json
+  for name, value in fieldPairs(res[]):
+    if json{name} != nil:
+      without parsed =? type(value).fromJson(json{name}), e:
+        error "error deserializing field",
+          field = $T & "." & name,
+          json = json{name},
+          error = e.msg
+        return failure(e)
+      value = parsed
+  success(res)
+
+proc fromJson*[T: object](
+  _: type T,
+  bytes: seq[byte]
+): ?!T =
+  let json = ?catch parseJson(string.fromBytes(bytes))
+  T.fromJson(json)
+
+func `%`*(s: string): JsonNode = newJString(s)
+
+func `%`*(n: uint): JsonNode =
+  if n > cast[uint](int.high):
+    newJString($n)
+  else:
+    newJInt(BiggestInt(n))
+
+func `%`*(n: int): JsonNode = newJInt(n)
+
+func `%`*(n: BiggestUInt): JsonNode =
+  if n > cast[BiggestUInt](BiggestInt.high):
+    newJString($n)
+  else:
+    newJInt(BiggestInt(n))
+
+func `%`*(n: BiggestInt): JsonNode = newJInt(n)
+
+func `%`*(n: float): JsonNode =
+  if n != n: newJString("nan")
+  elif n == Inf: newJString("inf")
+  elif n == -Inf: newJString("-inf")
+  else: newJFloat(n)
+
+func `%`*(b: bool): JsonNode = newJBool(b)
+
+func `%`*(keyVals: openArray[tuple[key: string, val: JsonNode]]): JsonNode =
+  if keyVals.len == 0: return newJArray()
+  let jObj = newJObject()
+  for key, val in items(keyVals): jObj.fields[key] = val
+  jObj
+
+template `%`*(j: JsonNode): JsonNode = j
+
+func `%`*[T](table: Table[string, T]|OrderedTable[string, T]): JsonNode =
+  let jObj = newJObject()
+  for k, v in table: jObj[k] = ? %v
+  jObj
+
+func `%`*[T](opt: Option[T]): JsonNode =
+  if opt.isSome: %(opt.get) else: newJNull()
+
+func `%`*[T: object](obj: T): JsonNode =
+  let jsonObj = newJObject()
+  for name, value in obj.fieldPairs:
+    when value.hasCustomPragma(serialize):
+      jsonObj[name] = %value
+  jsonObj
+
+func `%`*[T: ref object](obj: T): JsonNode =
+  let jsonObj = newJObject()
+  for name, value in obj[].fieldPairs:
+    when value.hasCustomPragma(serialize):
+      jsonObj[name] = %(value)
+  jsonObj
+
+proc `%`*(o: enum): JsonNode = % $o
+
+func `%`*(stint: StInt|StUint): JsonNode = %stint.toString
+
+func `%`*(cstr: cstring): JsonNode = % $cstr
+
+func `%`*(arr: openArray[byte]): JsonNode = % arr.to0xHex
+
+func `%`*[T](elements: openArray[T]): JsonNode =
+  let jObj = newJArray()
+  for elem in elements: jObj.add(%elem)
+  jObj
+
+func `%`*[T: distinct](id: T): JsonNode =
+  type baseType = T.distinctBase
+  % baseType(id)
+
+proc toJsnImpl(x: NimNode): NimNode =
+  case x.kind
+  of nnkBracket: # array
+    if x.len == 0: return newCall(bindSym"newJArray")
+    result = newNimNode(nnkBracket)
+    for i in 0 ..< x.len:
+      result.add(toJsnImpl(x[i]))
+    result = newCall(bindSym("%", brOpen), result)
+  of nnkTableConstr: # object
+    if x.len == 0: return newCall(bindSym"newJObject")
+    result = newNimNode(nnkTableConstr)
+    for i in 0 ..< x.len:
+      x[i].expectKind nnkExprColonExpr
+      result.add newTree(nnkExprColonExpr, x[i][0], toJsnImpl(x[i][1]))
+    result = newCall(bindSym("%", brOpen), result)
+  of nnkCurly: # empty object
+    x.expectLen(0)
+    result = newCall(bindSym"newJObject")
+  of nnkNilLit:
+    result = newCall(bindSym"newJNull")
+  of nnkPar:
+    if x.len == 1: result = toJsnImpl(x[0])
+    else: result = newCall(bindSym("%", brOpen), x)
+  else:
+    result = newCall(bindSym("%", brOpen), x)
+
+macro `%*`*(x: untyped): JsonNode =
+  ## Convert an expression to a JsonNode directly, without having to specify
+  ## `%` for every element.
+  result = toJsnImpl(x)

tests/codex/testutils.nim

@@ -1,3 +1,4 @@
+import ./utils/testjson
 import ./utils/testoptionalcast
 import ./utils/testkeyutils
 import ./utils/testasyncstatemachine

tests/codex/utils/testjson.nim

@@ -0,0 +1,347 @@
+import std/math
+import std/options
+import std/strformat
+import std/strutils
+import std/unittest
+import pkg/chronicles
+import pkg/stew/byteutils
+import pkg/stint
+import pkg/codex/contracts/requests
+from pkg/codex/rest/json import RestPurchase
+import pkg/codex/utils/json as utilsjson
+import pkg/questionable
+import pkg/questionable/results
+import ../examples
+import ../helpers
+
+checksuite "json serialization":
+  var request: StorageRequest
+  var requestJson: JsonNode
+
+  func flatten(s: string): string =
+    s.replace(" ")
+     .replace("\n")
+
+  setup:
+    request = StorageRequest(
+      client: Address.init("0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174").get(),
+      ask: StorageAsk(
+        slots: 4,
+        slotSize: (1 * 1024 * 1024 * 1024).u256, # 1 Gigabyte
+        duration: (10 * 60 * 60).u256, # 10 hours
+        collateral: 200.u256,
+        proofProbability: 4.u256, # require a proof roughly once every 4 periods
+        reward: 84.u256,
+        maxSlotLoss: 2 # 2 slots can be freed without data considered to be lost
+      ),
+      content: StorageContent(
+        cid: "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob",
+        erasure: StorageErasure(
+          totalChunks: 12,
+        ),
+        por: StoragePoR(
+          u: @(array[480, byte].fromHex("0xc066dd7e405de5a795ce1e765209cfaa6de6c74829c607d1a2fe53107107a2981baccff95e2fc2e38d08303e8ab59a1169cfc9bfbfa2294c6df065456056a0d0106f66d6fe48300222758fd6fd286a1ac9060d1a295e19f931a8d3ad2c47eb131bea267fe942d460fda96fd4bf663148cd90fbb1b670dd97aae70394248b75cfbb98f71e8c69f50381e0558884d5d9aa75147923b55386c66f75f63024b698eeb0ff994bfdb610eea1b7c75e87bdb54843071bc64fbaf93e5dc214e875bd95bd30f167b69df1de819a34cc71a3a0465f5c1d1b7e5b98de6017ff3e3c059536f974471fe62e0f224eba8f96352a8ee51befbf4c31c88ad0fc8ff4e9d9da174a455a1c30fd61ac977145d3677a08167d508fae207f9458a9b19d4ceec2be30506e2d70cc0362c2bcdb0f73d63fa5e79f9b2901bc870ac8b2a264d50e1862ea177eb587bcd16ceb7d66f96f198cadec3f644af4d3cbe478bc1665818401f89107053d1750047fb7cfc47938bec2cd006db9c176ce337e41160077e353f87ab319e5b9df92282916ef99334c067f6ca20c3d7cbc12b95180b7bba762993a4dbdf4242032da8865988183738d279918906c3357701d74e5d8f5142315ae8f6d0f93537abc3545118e953f983317657a9d8b86e4305ea49e10f80ea07dc7ea7321b32c")),
+          publicKey: @(array[96, byte].fromHex("0xb231b19de641f678d250623b2b76099ab4bbd67aac19dcf42ded946831e3366d2a20af0fd9e841197e7e64d7639da4518b76c353db480087e21d55b470f24a180d6d6c8265bf3895e2e4e4e54b8ca9334d62b22feeeed8e77e54bfbc8fae6b62")),
+          name: @(array[512, byte].fromHex("0x75b2ac401efd21e60e84a69288da6fff28c7badaae885e417f35055a4e10cb514855f68a0ae18bf42861426c9fc34af13df2f2d04dc68933af78bf3fc396953f301b95f6d6af54ec9fc871c292096e45b91e836063f128c2d1469adbee49bc9b7d62985a858801e4df2cb77eb41ee7b50a8a4e5afb5b585f9034a2808f81bd95b9a3fbdd2579331023f1816a1ecbe7a31e386721a72e3d0ff6087326fba8442dfd22d1182c85906d796e697231c2d7d4a888ae256c79a9019974a4c729d981f3e554f48895e27fe8f45da46bc48c35cc74ae5a31dfea8baa1334fa7f106cdc4ec54452f39c823fa0af97769217cc16c78eb7d0c494c26d2f286f09a507bd04cb15963270bffefb28258176d9e10b7aaad76cdd86e0fe49437eb83c1c0650cb5920e32dc54f3a21a70308b7312b47ce57ef72c2c19eba5027612128b747e80b88c912d7fc10177e67beda0ed5bb8fdfc268bfa5a5c700da953c56bcc79b9186da99ee19a6fa954f44bdcbc7c7f4d208fb750bad587d5513fbaccd511b9b6e0cd798120de87b9c0c410b3b85c75a8a0f469d9973a1ec4c86982cf4fe1a2be21a9206aaabb1ad2fafa628d5156d2ec99ee30fc0ddb9dca6a4cd3a7987227315ceeaa832909853cabaf33c976b59cf5ed9643781d92ab769c0d0aa3bcef40b41b4b3e6fc5a00c9dfbf794047d9cfb97d9d669d00520b6492760a08dba65b0fd7e6d0"))
+        )
+      ),
+      expiry: 1691545330.u256,
+      nonce: Nonce array[32, byte].fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1")
+    )
+    requestJson = """{
+                      "client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174",
+                      "ask": {
+                        "slots": 4,
+                        "slotSize": "1073741824",
+                        "duration": "36000",
+                        "proofProbability": "4",
+                        "reward": "84",
+                        "collateral": "200",
+                        "maxSlotLoss": 2
+                      },
+                      "content": {
+                        "cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob",
+                        "erasure": {
+                          "totalChunks": 12
+                        },
+                        "por": {
+                          "u": "0xc066dd7e405de5a795ce1e765209cfaa6de6c74829c607d1a2fe53107107a2981baccff95e2fc2e38d08303e8ab59a1169cfc9bfbfa2294c6df065456056a0d0106f66d6fe48300222758fd6fd286a1ac9060d1a295e19f931a8d3ad2c47eb131bea267fe942d460fda96fd4bf663148cd90fbb1b670dd97aae70394248b75cfbb98f71e8c69f50381e0558884d5d9aa75147923b55386c66f75f63024b698eeb0ff994bfdb610eea1b7c75e87bdb54843071bc64fbaf93e5dc214e875bd95bd30f167b69df1de819a34cc71a3a0465f5c1d1b7e5b98de6017ff3e3c059536f974471fe62e0f224eba8f96352a8ee51befbf4c31c88ad0fc8ff4e9d9da174a455a1c30fd61ac977145d3677a08167d508fae207f9458a9b19d4ceec2be30506e2d70cc0362c2bcdb0f73d63fa5e79f9b2901bc870ac8b2a264d50e1862ea177eb587bcd16ceb7d66f96f198cadec3f644af4d3cbe478bc1665818401f89107053d1750047fb7cfc47938bec2cd006db9c176ce337e41160077e353f87ab319e5b9df92282916ef99334c067f6ca20c3d7cbc12b95180b7bba762993a4dbdf4242032da8865988183738d279918906c3357701d74e5d8f5142315ae8f6d0f93537abc3545118e953f983317657a9d8b86e4305ea49e10f80ea07dc7ea7321b32c",
+                          "publicKey": "0xb231b19de641f678d250623b2b76099ab4bbd67aac19dcf42ded946831e3366d2a20af0fd9e841197e7e64d7639da4518b76c353db480087e21d55b470f24a180d6d6c8265bf3895e2e4e4e54b8ca9334d62b22feeeed8e77e54bfbc8fae6b62",
+                          "name": "0x75b2ac401efd21e60e84a69288da6fff28c7badaae885e417f35055a4e10cb514855f68a0ae18bf42861426c9fc34af13df2f2d04dc68933af78bf3fc396953f301b95f6d6af54ec9fc871c292096e45b91e836063f128c2d1469adbee49bc9b7d62985a858801e4df2cb77eb41ee7b50a8a4e5afb5b585f9034a2808f81bd95b9a3fbdd2579331023f1816a1ecbe7a31e386721a72e3d0ff6087326fba8442dfd22d1182c85906d796e697231c2d7d4a888ae256c79a9019974a4c729d981f3e554f48895e27fe8f45da46bc48c35cc74ae5a31dfea8baa1334fa7f106cdc4ec54452f39c823fa0af97769217cc16c78eb7d0c494c26d2f286f09a507bd04cb15963270bffefb28258176d9e10b7aaad76cdd86e0fe49437eb83c1c0650cb5920e32dc54f3a21a70308b7312b47ce57ef72c2c19eba5027612128b747e80b88c912d7fc10177e67beda0ed5bb8fdfc268bfa5a5c700da953c56bcc79b9186da99ee19a6fa954f44bdcbc7c7f4d208fb750bad587d5513fbaccd511b9b6e0cd798120de87b9c0c410b3b85c75a8a0f469d9973a1ec4c86982cf4fe1a2be21a9206aaabb1ad2fafa628d5156d2ec99ee30fc0ddb9dca6a4cd3a7987227315ceeaa832909853cabaf33c976b59cf5ed9643781d92ab769c0d0aa3bcef40b41b4b3e6fc5a00c9dfbf794047d9cfb97d9d669d00520b6492760a08dba65b0fd7e6d0"
+                        }
+                      },
+                      "expiry": "1691545330",
+                      "nonce": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"
+                    }""".parseJson
+
+  test "serializes UInt256 to non-hex string representation":
+    check (% 100000.u256) == newJString("100000")
+
+  test "serializes sequence to an array":
+    let json = % @[1, 2, 3]
+    let expected = "[1,2,3]"
+    check $json == expected
+
+  test "serializes Option[T] when has a value":
+    let obj = %(some 1)
+    let expected = "1"
+    check $obj == expected
+
+  test "serializes Option[T] when doesn't have a value":
+    let obj = %(none int)
+    let expected = "null"
+    check $obj == expected
+
+  test "serializes uints int.high or smaller":
+    let largeUInt: uint = uint(int.high)
+    check %largeUInt == newJInt(BiggestInt(largeUInt))
+
+  test "serializes large uints":
+    let largeUInt: uint = uint(int.high) + 1'u
+    check %largeUInt == newJString($largeUInt)
+
+
+  test "serializes Inf float":
+    check %Inf == newJString("inf")
+
+  test "serializes -Inf float":
+    check %(-Inf) == newJString("-inf")
+
+  test "deserializes NaN float":
+    check %NaN == newJString("nan")
+
+  test "can construct json objects with %*":
+    type MyObj = object
+      mystring {.serialize.}: string
+      myint {.serialize.}: int
+      myoption {.serialize.}: ?bool
+
+    let myobj = MyObj(mystring: "abc", myint: 123, myoption: some true)
+    let mystuint = 100000.u256
+
+    let json = %*{
+      "myobj": myobj,
+      "mystuint": mystuint
+    }
+
+    let expected = """{
+                        "myobj": {
+                          "mystring": "abc",
+                          "myint": 123,
+                          "myoption": true
+                        },
+                        "mystuint": "100000"
+                      }""".flatten
+
+    check $json == expected
+
+  test "only serializes marked fields":
+    type MyObj = object
+      mystring {.serialize.}: string
+      myint {.serialize.}: int
+      mybool: bool
+
+    let obj = % MyObj(mystring: "abc", myint: 1, mybool: true)
+
+    let expected = """{
+                        "mystring": "abc",
+                        "myint": 1
+                      }""".flatten
+
+    check $obj == expected
+
+  test "serializes ref objects":
+    type MyRef = ref object
+      mystring {.serialize.}: string
+      myint {.serialize.}: int
+
+    let obj = % MyRef(mystring: "abc", myint: 1)
+
+    let expected = """{
+                        "mystring": "abc",
+                        "myint": 1
+                      }""".flatten
+
+    check $obj == expected
+
+  test "serializes RestPurchase":
+    let request = % RestPurchase(
+      request: some request,
+      requestId: RequestId.fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"),
+      error: some "error",
+      state: "state"
+    )
+    let expected = """{
+                        "requestId": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1",
+                        "request": {
+                          "client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174",
+                          "ask": {
+                            "slots": 4,
+                            "slotSize": "1073741824",
+                            "duration": "36000",
+                            "proofProbability": "4",
+                            "reward": "84",
+                            "collateral": "200",
+                            "maxSlotLoss": 2
+                          },
+                          "content": {
+                            "cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob"
+                          },
+                          "expiry": "1691545330"
+                        },
+                        "state": "state",
+                        "error": "error"
+                      }""".flatten
+    check $request == expected
+
+  test "serializes StorageRequest":
+    let expected = """{
+                        "client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174",
+                        "ask": {
+                          "slots": 4,
+                          "slotSize": "1073741824",
+                          "duration": "36000",
+                          "proofProbability": "4",
+                          "reward": "84",
+                          "collateral": "200",
+                          "maxSlotLoss": 2
+                        },
+                        "content": {
+                          "cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob"
+                        },
+                        "expiry": "1691545330"
+                      }""".flatten
+    check $(%request) == expected
+
+  test "deserializes UInt256 from non-hex string representation":
+    let json = newJString("100000")
+    check !UInt256.fromJson(json) == 100000.u256
+
+  test "deserializes Option[T] when has a value":
+    let json = newJInt(1)
+    check (!fromJson(?int, json) == some 1)
+
+  test "deserializes Option[T] when doesn't have a value":
+    let json = newJNull()
+    check !fromJson(?int, json) == none int
+
+  test "deserializes float":
+    let json = newJFloat(1.234)
+    check !float.fromJson(json) == 1.234
+
+  test "deserializes Inf float":
+    let json = newJString("inf")
+    check !float.fromJson(json) == Inf
+
+  test "deserializes -Inf float":
+    let json = newJString("-inf")
+    check !float.fromJson(json) == -Inf
+
+  test "deserializes NaN float":
+    let json = newJString("nan")
+    check float.fromJson(json).get.isNaN
+
+  test "deserializes array to sequence":
+    let expected = @[1, 2, 3]
+    let json = "[1,2,3]".parseJson
+    check !seq[int].fromJson(json) == expected
+
+  test "deserializes uints int.high or smaller":
+    let largeUInt: uint = uint(int.high)
+    let json = newJInt(BiggestInt(largeUInt))
+    check !uint.fromJson(json) == largeUInt
+
+  test "deserializes large uints":
+    let largeUInt: uint = uint(int.high) + 1'u
+    let json = newJString($BiggestUInt(largeUInt))
+    check !uint.fromJson(json) == largeUInt
+
+  test "can deserialize json objects":
+    type MyObj = object
+      mystring: string
+      myint: int
+      myoption: ?bool
+
+    let expected = MyObj(mystring: "abc", myint: 123, myoption: some true)
+
+    let json = parseJson("""{
+                              "mystring": "abc",
+                              "myint": 123,
+                              "myoption": true
+                            }""")
+    check !MyObj.fromJson(json) == expected
+
+  test "ignores serialize pragma when deserializing":
+    type MyObj = object
+      mystring {.serialize.}: string
+      mybool: bool
+
+    let expected = MyObj(mystring: "abc", mybool: true)
+
+    let json = parseJson("""{
+                              "mystring": "abc",
+                              "mybool": true
+                            }""")
+
+    check !MyObj.fromJson(json) == expected
+
+  test "deserializes objects with extra fields":
+    type MyObj = object
+      mystring: string
+      mybool: bool
+
+    let expected = MyObj(mystring: "abc", mybool: true)
+
+    let json = """{
+                    "mystring": "abc",
+                    "mybool": true,
+                    "extra": "extra"
+                  }""".parseJson
+    check !MyObj.fromJson(json) == expected
+
+  test "deserializes objects with less fields":
+    type MyObj = object
+      mystring: string
+      mybool: bool
+
+    let expected = MyObj(mystring: "abc", mybool: false)
+
+    let json = """{
+                    "mystring": "abc"
+                  }""".parseJson
+    check !MyObj.fromJson(json) == expected
+
+  test "deserializes ref objects":
+    type MyRef = ref object
+      mystring: string
+      myint: int
+
+    let expected = MyRef(mystring: "abc", myint: 1)
+
+    let json = """{
+                    "mystring": "abc",
+                    "myint": 1
+                  }""".parseJson
+
+    let deserialized = !MyRef.fromJson(json)
+    check deserialized.mystring == expected.mystring
+    check deserialized.myint == expected.myint
+
+  test "deserializes StorageRequest":
+    check !StorageRequest.fromJson(requestJson) == request
+
+  test "deserializes RestPurchase":
+    let json = """{
+                    "requestId": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1",
+                    "state": "state",
+                    "error": "error"
+                  }""".parseJson
+    json["request"] = requestJson
+
+    let expected = RestPurchase(
+      requestId: RequestId.fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"),
+      state: "state",
+      error: some "error",
+      request: some request
+    )
+    check !RestPurchase.fromJson(json) == expected

tests/integration/codexclient.nim

@@ -1,8 +1,13 @@
 import std/httpclient
-import std/json
 import std/strutils
+from pkg/libp2p import Cid, `$`, init
+import pkg/chronicles
 import pkg/stint
 import pkg/questionable/results
+import pkg/codex/rest/json
+import pkg/codex/purchasing
+import pkg/codex/errors
+import pkg/codex/sales/reservations
 
 type CodexClient* = ref object
   http: HttpClient
@@ -21,38 +26,43 @@ proc setLogLevel*(client: CodexClient, level: string) =
   let response = client.http.request(url, httpMethod=HttpPost, headers=headers)
   assert response.status == "200 OK"
 
-proc upload*(client: CodexClient, contents: string): string =
+proc upload*(client: CodexClient, contents: string): ?!Cid =
   let response = client.http.post(client.baseurl & "/upload", contents)
   assert response.status == "200 OK"
-  response.body
+  Cid.init(response.body).mapFailure
 
 proc requestStorage*(
     client: CodexClient,
-    cid: string,
+    cid: Cid,
-    duration: uint64,
+    duration: UInt256,
-    reward: uint64,
+    reward: UInt256,
-    proofProbability: uint64,
+    proofProbability: UInt256,
     expiry: UInt256,
-    collateral: uint64
+    collateral: UInt256,
-): string =
+    nodes: uint = 1,
+    tolerance: uint = 0
+): ?!PurchaseId =
   ## Call request storage REST endpoint
   ##
-  let url = client.baseurl & "/storage/request/" & cid
+  let url = client.baseurl & "/storage/request/" & $cid
   let json = %*{
-    "duration": $duration,
+    "duration": duration,
-    "reward": $reward,
+    "reward": reward,
-    "proofProbability": $proofProbability,
+    "proofProbability": proofProbability,
-    "expiry": $expiry,
+    "expiry": expiry,
-    "collateral": $collateral,
+    "collateral": collateral,
+    "nodes": nodes,
+    "tolerance": tolerance
   }
   let response = client.http.post(url, $json)
   assert response.status == "200 OK"
-  response.body
+  PurchaseId.fromHex(response.body).catch
 
-proc getPurchase*(client: CodexClient, purchase: string): JsonNode =
+proc getPurchase*(client: CodexClient, purchaseId: PurchaseId): ?!RestPurchase =
-  let url = client.baseurl & "/storage/purchases/" & purchase
+  let url = client.baseurl & "/storage/purchases/" & purchaseId.toHex
   let body = client.http.getContent(url)
-  parseJson(body).catch |? nil
+  let json = ? parseJson(body).catch
+  RestPurchase.fromJson(json)
 
 proc getSlots*(client: CodexClient): JsonNode =
   let url = client.baseurl & "/sales/slots"
@@ -61,27 +71,26 @@ proc getSlots*(client: CodexClient): JsonNode =
 
 proc postAvailability*(
     client: CodexClient,
-    size, duration, minPrice: uint64,
+    size, duration, minPrice, maxCollateral: UInt256
-    maxCollateral: uint64
+): ?!Availability =
-): JsonNode =
   ## Post sales availability endpoint
   ##
   let url = client.baseurl & "/sales/availability"
   let json = %*{
-    "size": $size,
+    "size": size,
-    "duration": $duration,
+    "duration": duration,
-    "minPrice": $minPrice,
+    "minPrice": minPrice,
-    "maxCollateral": $maxCollateral,
+    "maxCollateral": maxCollateral,
   }
   let response = client.http.post(url, $json)
   assert response.status == "200 OK"
-  parseJson(response.body)
+  Availability.fromJson(response.body.parseJson)
 
-proc getAvailabilities*(client: CodexClient): JsonNode =
+proc getAvailabilities*(client: CodexClient): ?!seq[Availability] =
   ## Call sales availability REST endpoint
   let url = client.baseurl & "/sales/availability"
   let body = client.http.getContent(url)
-  parseJson(body)
+  seq[Availability].fromJson(parseJson(body))
 
 proc close*(client: CodexClient) =
   client.http.close()

tests/integration/testIntegration.nim

@@ -1,4 +1,5 @@
-import std/json
+import std/options
+from pkg/libp2p import `==`
 import pkg/chronos
 import pkg/stint
 import pkg/ethers/erc20
@@ -9,11 +10,16 @@ import ../contracts/deployment
 import ../codex/helpers/eventually
 import ./twonodes
 
+
 # For debugging you can enable logging output with debugX = true
 # You can also pass a string in same format like for the `--log-level` parameter
 # to enable custom logging levels for specific topics like: debug2 = "INFO; TRACE: marketplace"
 
 twonodessuite "Integration tests", debug1 = false, debug2 = false:
+
+  proc purchaseStateIs(client: CodexClient, id: PurchaseId, state: string): bool =
+    client.getPurchase(id).option.?state == some state
+
   setup:
     # Our Hardhat configuration does use automine, which means that time tracked by `provider.currentTime()` is not
     # advanced until blocks are mined and that happens only when transaction is submitted.
@@ -27,87 +33,105 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
     client1.setLogLevel("DEBUG;TRACE:codex")
 
   test "node accepts file uploads":
-    let cid1 = client1.upload("some file contents")
+    let cid1 = client1.upload("some file contents").get
-    let cid2 = client1.upload("some other contents")
+    let cid2 = client1.upload("some other contents").get
     check cid1 != cid2
 
   test "node handles new storage availability":
-    let availability1 = client1.postAvailability(size=1, duration=2, minPrice=3, maxCollateral=4)
+    let availability1 = client1.postAvailability(size=1.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get
-    let availability2 = client1.postAvailability(size=4, duration=5, minPrice=6, maxCollateral=7)
+    let availability2 = client1.postAvailability(size=4.u256, duration=5.u256, minPrice=6.u256, maxCollateral=7.u256).get
     check availability1 != availability2
 
   test "node lists storage that is for sale":
-    let availability = client1.postAvailability(size=1, duration=2, minPrice=3, maxCollateral=4)
+    let availability = client1.postAvailability(size=1.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get
-    check availability in client1.getAvailabilities()
+    check availability in client1.getAvailabilities().get
 
   test "node handles storage request":
     let expiry = (await provider.currentTime()) + 30
-    let cid = client1.upload("some file contents")
+    let cid = client1.upload("some file contents").get
-    let id1 = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry, collateral=200)
+    let id1 = client1.requestStorage(cid, duration=1.u256, reward=2.u256, proofProbability=3.u256, expiry=expiry, collateral=200.u256).get
-    let id2 = client1.requestStorage(cid, duration=4, reward=5, proofProbability=6, expiry=expiry, collateral=201)
+    let id2 = client1.requestStorage(cid, duration=4.u256, reward=5.u256, proofProbability=6.u256, expiry=expiry, collateral=201.u256).get
     check id1 != id2
 
   test "node retrieves purchase status":
     let expiry = (await provider.currentTime()) + 30
-    let cid = client1.upload("some file contents")
+    let cid = client1.upload("some file contents").get
-    let id = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry, collateral=200)
+    let id = client1.requestStorage(cid, duration=1.u256, reward=2.u256, proofProbability=3.u256, expiry=expiry, collateral=200.u256, nodes=2, tolerance=1).get
-    let purchase = client1.getPurchase(id)
+    let request = client1.getPurchase(id).get.request.get
-    check purchase{"request"}{"ask"}{"duration"} == %"1"
+    check request.ask.duration == 1.u256
-    check purchase{"request"}{"ask"}{"reward"} == %"2"
+    check request.ask.reward == 2.u256
-    check purchase{"request"}{"ask"}{"proofProbability"} == %"3"
+    check request.ask.proofProbability == 3.u256
+    check request.expiry == expiry
+    check request.ask.collateral == 200.u256
+    check request.ask.slots == 3'u64
+    check request.ask.maxSlotLoss == 1'u64
 
   test "node remembers purchase status after restart":
     let expiry = (await provider.currentTime()) + 30
-    let cid = client1.upload("some file contents")
+    let cid = client1.upload("some file contents").get
-    let id = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry, collateral=200)
+    let id = client1.requestStorage(cid,
-    check eventually client1.getPurchase(id){"state"}.getStr() == "submitted"
+                                    duration=1.u256,
+                                    reward=2.u256,
+                                    proofProbability=3.u256,
+                                    expiry=expiry,
+                                    collateral=200.u256).get
+    check eventually client1.purchaseStateIs(id, "submitted")
 
     node1.restart()
     client1.restart()
 
-    check eventually (not isNil client1.getPurchase(id){"request"}{"ask"})
+    check eventually client1.purchaseStateIs(id, "submitted")
-    check client1.getPurchase(id){"request"}{"ask"}{"duration"} == %"1"
+    let request = client1.getPurchase(id).get.request.get
-    check client1.getPurchase(id){"request"}{"ask"}{"reward"} == %"2"
+    check request.ask.duration == 1.u256
+    check request.ask.reward == 2.u256
+    check request.ask.proofProbability == 3.u256
+    check request.expiry == expiry
+    check request.ask.collateral == 200.u256
+    check request.ask.slots == 1'u64
+    check request.ask.maxSlotLoss == 0'u64
+
 
   test "nodes negotiate contracts on the marketplace":
-    let size: uint64 = 0xFFFFF
+    let size = 0xFFFFF.u256
     # client 2 makes storage available
-    discard client2.postAvailability(size=size, duration=200, minPrice=300, maxCollateral=300)
+    discard client2.postAvailability(size=size, duration=200.u256, minPrice=300.u256, maxCollateral=300.u256)
 
     # client 1 requests storage
     let expiry = (await provider.currentTime()) + 30
-    let cid = client1.upload("some file contents")
+    let cid = client1.upload("some file contents").get
-    let purchase = client1.requestStorage(cid, duration=100, reward=400, proofProbability=3, expiry=expiry, collateral=200)
+    let id = client1.requestStorage(cid, duration=100.u256, reward=400.u256, proofProbability=3.u256, expiry=expiry, collateral=200.u256).get
 
-    check eventually client1.getPurchase(purchase){"state"} == %"started"
+    check eventually client1.purchaseStateIs(id, "started")
-    check client1.getPurchase(purchase){"error"} == newJNull()
+    let purchase = client1.getPurchase(id).get
-    let availabilities = client2.getAvailabilities()
+    check purchase.error == none string
+    let availabilities = client2.getAvailabilities().get
     check availabilities.len == 1
-    let newSize = UInt256.fromDecimal(availabilities[0]{"size"}.getStr)
+    let newSize = availabilities[0].size
-    check newSize > 0 and newSize < size.u256
+    check newSize > 0 and newSize < size
 
   test "node slots gets paid out":
     let marketplace = Marketplace.new(Marketplace.address, provider.getSigner())
     let tokenAddress = await marketplace.token()
     let token = Erc20Token.new(tokenAddress, provider.getSigner())
-    let reward: uint64 = 400
+    let reward = 400.u256
-    let duration: uint64 = 100
+    let duration = 100.u256
 
     # client 2 makes storage available
     let startBalance = await token.balanceOf(account2)
-    discard client2.postAvailability(size=0xFFFFF, duration=200, minPrice=300, maxCollateral=300)
+    discard client2.postAvailability(size=0xFFFFF.u256, duration=200.u256, minPrice=300.u256, maxCollateral=300.u256).get
 
     # client 1 requests storage
     let expiry = (await provider.currentTime()) + 30
-    let cid = client1.upload("some file contents")
+    let cid = client1.upload("some file contents").get
-    let purchase = client1.requestStorage(cid, duration=duration, reward=reward, proofProbability=3, expiry=expiry, collateral=200)
+    let id = client1.requestStorage(cid, duration=duration, reward=reward, proofProbability=3.u256, expiry=expiry, collateral=200.u256).get
 
-    check eventually client1.getPurchase(purchase){"state"} == %"started"
+    check eventually client1.purchaseStateIs(id, "started")
-    check client1.getPurchase(purchase){"error"} == newJNull()
+    let purchase = client1.getPurchase(id).get
+    check purchase.error == none string
 
     # Proving mechanism uses blockchain clock to do proving/collect/cleanup round
     # hence we must use `advanceTime` over `sleepAsync` as Hardhat does mine new blocks
     # only with new transaction
-    await provider.advanceTime(duration.u256)
+    await provider.advanceTime(duration)
 
-    check eventually (await token.balanceOf(account2)) - startBalance == duration.u256*reward.u256
+    check eventually (await token.balanceOf(account2)) - startBalance == duration*reward

tests/integration/testproofs.nim

@@ -2,7 +2,7 @@ import std/sequtils
 import std/os
 from std/times import getTime, toUnix
 import pkg/chronicles
-import codex/contracts/marketplace
+import codex/contracts
 import codex/periods
 import ../contracts/time
 import ../contracts/deployment
@@ -19,6 +19,9 @@ twonodessuite "Proving integration test", debug1=false, debug2=false:
   var marketplace: Marketplace
   var period: uint64
 
+  proc purchaseStateIs(client: CodexClient, id: PurchaseId, state: string): bool =
+    client.getPurchase(id).option.?state == some state
+
   setup:
     marketplace = Marketplace.new(Marketplace.address, provider)
     period = (await marketplace.config()).proofs.period.truncate(uint64)
@@ -32,22 +35,22 @@ twonodessuite "Proving integration test", debug1=false, debug2=false:
                                   duration: uint64 = 100 * period,
                                   expiry: uint64 = 30) {.async.} =
     discard client2.postAvailability(
-      size=0xFFFFF,
+      size=0xFFFFF.u256,
-      duration=duration,
+      duration=duration.u256,
-      minPrice=300,
+      minPrice=300.u256,
-      maxCollateral=200
+      maxCollateral=200.u256
     )
-    let cid = client1.upload("some file contents")
+    let cid = client1.upload("some file contents").get
     let expiry = (await provider.currentTime()) + expiry.u256
-    let purchase = client1.requestStorage(
+    let id = client1.requestStorage(
       cid,
       expiry=expiry,
-      duration=duration,
+      duration=duration.u256,
-      proofProbability=proofProbability,
+      proofProbability=proofProbability.u256,
-      collateral=100,
+      collateral=100.u256,
-      reward=400
+      reward=400.u256
-    )
+    ).get
-    check eventually client1.getPurchase(purchase){"state"} == %"started"
+    check eventually client1.purchaseStateIs(id, "started")
 
   proc advanceToNextPeriod {.async.} =
     let periodicity = Periodicity(seconds: period.u256)
@@ -105,6 +108,9 @@ multinodesuite "Simulate invalid proofs",
   StartNodes.init(clients=1'u, providers=0'u, validators=1'u),
   DebugNodes.init(client=false, provider=false, validator=false):
 
+  proc purchaseStateIs(client: CodexClient, id: PurchaseId, state: string): bool =
+    client.getPurchase(id).option.?state == some state
+
   var marketplace: Marketplace
   var period: uint64
   var slotId: SlotId
@@ -142,26 +148,26 @@ multinodesuite "Simulate invalid proofs",
     let storageProvider = providers()[0].restClient
 
     discard storageProvider.postAvailability(
-      size=0xFFFFF,
+      size=0xFFFFF.u256,
-      duration=duration,
+      duration=duration.u256,
-      minPrice=300,
+      minPrice=300.u256,
-      maxCollateral=200
+      maxCollateral=200.u256
     )
-    let cid = client.upload("some file contents " & $ getTime().toUnix)
+    let cid = client.upload("some file contents " & $ getTime().toUnix).get
     let expiry = (await provider.currentTime()) + expiry.u256
     # avoid timing issues by filling the slot at the start of the next period
     await advanceToNextPeriod()
-    let purchase = client.requestStorage(
+    let id = client.requestStorage(
       cid,
       expiry=expiry,
-      duration=duration,
+      duration=duration.u256,
-      proofProbability=proofProbability,
+      proofProbability=proofProbability.u256,
-      collateral=100,
+      collateral=100.u256,
-      reward=400
+      reward=400.u256
-    )
+    ).get
-    check eventually client.getPurchase(purchase){"state"} == %"started"
+    check eventually client.purchaseStateIs(id, "started")
-    let requestId = RequestId.fromHex client.getPurchase(purchase){"requestId"}.getStr
+    let purchase = client.getPurchase(id).get
-    slotId = slotId(requestId, 0.u256)
+    slotId = slotId(purchase.requestId, 0.u256)
 
   # TODO: these are very loose tests in that they are not testing EXACTLY how
   # proofs were marked as missed by the validator. These tests should be