status-im
/
nim-codex
mirror of https://github.com/status-im/nim-codex.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor: use serde for json de/serialization instead of utils/json (#704) 

* json > nim-serde bump

Should wait until serde is integrated into nim-ethers before making these changes as there will be less import exceptions required.

* bump nim-serde

* change func to proc due to chronicles side effects

* import serde into utils/json, use as proxy

import nim-serde into utils/json and use utils/json as a proxy for serde functions, including overloading `%` and `fromJson` for application types.

* update tests to use serde

* bump serde to latest

* remove testjson -- no longer needed

* bump serde in nimble

* updates to reconcile rebase with master
This commit is contained in:
Eric 2024-03-19 14:25:13 +11:00 committed by GitHub
parent b4de53f436
commit f567f4ec15
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 43 additions and 701 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@ -209,3 +209,6 @@
url = https://github.com/codex-storage/codex-storage-proofs-circuits.git url = https://github.com/codex-storage/codex-storage-proofs-circuits.git
ignore = untracked ignore = untracked
branch = master branch = master
[submodule "vendor/nim-serde"]
path = vendor/nim-serde
url = https://github.com/codex-storage/nim-serde.git

1
codex.nimble
View File

@ -22,6 +22,7 @@ requires "presto"
requires "protobuf_serialization >= 0.2.0 & < 0.3.0" requires "protobuf_serialization >= 0.2.0 & < 0.3.0"
requires "questionable >= 0.10.13 & < 0.11.0" requires "questionable >= 0.10.13 & < 0.11.0"
requires "secp256k1" requires "secp256k1"
requires "serde >= 1.0.0 & < 2.0.0"
requires "stew" requires "stew"
requires "upraises >= 0.1.0 & < 0.2.0" requires "upraises >= 0.1.0 & < 0.2.0"
requires "toml_serialization" requires "toml_serialization"

15
codex/logutils.nim
View File

@ -17,16 +17,16 @@
## module, and specifying `formatIt`. If textlines log output and json log output ## module, and specifying `formatIt`. If textlines log output and json log output
## need to be different, overload `formatIt` and specify a `LogFormat`. If json ## need to be different, overload `formatIt` and specify a `LogFormat`. If json
## serialization is needed, it can be declared with a `%` proc. `logutils` ## serialization is needed, it can be declared with a `%` proc. `logutils`
## imports and exports `utils/json` which handles the de/serialization, examples ## imports and exports `nim-serde` which handles the de/serialization, examples
## below. **Only `codex/logutils` needs to be imported.** ## below. **Only `codex/logutils` needs to be imported.**
## ##
## Using `logutils` in the Codex codebase: ## Using `logutils` in the Codex codebase:
## - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils` ## - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils`
## - most of `chronicles` is exported by `logutils` ## - most of `chronicles` is exported by `logutils`
## - Instead of importing `std/json`, import `pkg/codex/utils/json` ## - Instead of importing `std/json`, import `pkg/serde/json`
## - `std/json` is exported by `utils/json` which is exported by `logutils` ## - `std/json` is exported by `serde` which is exported by `logutils`
## - Instead of importing `pkg/nim-json-serialization`, import ## - Instead of importing `pkg/nim-json-serialization`, import
## `pkg/codex/utils/json` ## `pkg/serde/json` or use codex-specific overloads by importing `utils/json`
## - one of the goals is to remove the use of `nim-json-serialization` ## - one of the goals is to remove the use of `nim-json-serialization`
## ##
## ```nim ## ```nim
@ -54,7 +54,7 @@
## # chronicles json output ## # chronicles json output
## {"lvl":"TRC","msg":"test","tid":14397405,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}} ## {"lvl":"TRC","msg":"test","tid":14397405,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}}
## ``` ## ```
## In this case, `BlockAddress` is just an object, so `utils/json` can handle ## In this case, `BlockAddress` is just an object, so `nim-serde` can handle
## serializing it without issue (only fields annotated with `{.serialize.}` will ## serializing it without issue (only fields annotated with `{.serialize.}` will
## serialize (aka opt-in serialization)). ## serialize (aka opt-in serialization)).
## ##
@ -95,20 +95,19 @@ import pkg/chronicles except toJson, `%`
from pkg/libp2p import Cid, MultiAddress, `$` from pkg/libp2p import Cid, MultiAddress, `$`
import pkg/questionable import pkg/questionable
import pkg/questionable/results import pkg/questionable/results
import ./utils/json except formatIt # TODO: remove exception?
import pkg/stew/byteutils import pkg/stew/byteutils
import pkg/stint import pkg/stint
import pkg/upraises import pkg/upraises
import ./utils/json
export byteutils export byteutils
export chronicles except toJson, formatIt, `%` export chronicles except toJson, formatIt, `%`
export questionable export questionable
export sequtils export sequtils
export json except formatIt
export strutils export strutils
export sugar export sugar
export upraises export upraises
export json
export results export results
func shortLog*(long: string, ellipses = "*", start = 3, stop = 6): string = func shortLog*(long: string, ellipses = "*", start = 3, stop = 6): string =

4
codex/rest/json.nim
View File

@ -104,11 +104,11 @@ proc init*(_: type RestNodeId, id: NodeId): RestNodeId =
id: id id: id
) )
func `%`*(obj: StorageRequest | Slot): JsonNode = proc `%`*(obj: StorageRequest | Slot): JsonNode =
let jsonObj = newJObject() let jsonObj = newJObject()
for k, v in obj.fieldPairs: jsonObj[k] = %v for k, v in obj.fieldPairs: jsonObj[k] = %v
jsonObj["id"] = %(obj.id) jsonObj["id"] = %(obj.id)
return jsonObj return jsonObj
func `%`*(obj: RestNodeId): JsonNode = % $obj.id proc `%`*(obj: RestNodeId): JsonNode = % $obj.id

2
codex/slots/builder/builder.nim
View File

@ -15,11 +15,11 @@ import std/sugar
import pkg/libp2p import pkg/libp2p
import pkg/chronos import pkg/chronos
import pkg/chronicles
import pkg/questionable import pkg/questionable
import pkg/questionable/results import pkg/questionable/results
import pkg/constantine/math/io/io_fields import pkg/constantine/math/io/io_fields
import ../../logutils
import ../../utils import ../../utils
import ../../stores import ../../stores
import ../../manifest import ../../manifest

2
codex/slots/sampler/sampler.nim
View File

@ -9,12 +9,12 @@
import std/sugar import std/sugar
import pkg/chronicles
import pkg/chronos import pkg/chronos
import pkg/questionable import pkg/questionable
import pkg/questionable/results import pkg/questionable/results
import pkg/stew/arrayops import pkg/stew/arrayops
import ../../logutils
import ../../market import ../../market
import ../../blocktype as bt import ../../blocktype as bt
import ../../merkletree import ../../merkletree

319
codex/utils/json.nim
View File

@ -1,181 +1,16 @@
import std/json except `%`, `%*`
import std/macros
import std/options import std/options
import std/strutils
import std/strformat
import std/tables
import std/typetraits import std/typetraits
from pkg/ethers import Address from pkg/ethers import Address
from pkg/libp2p import Cid, PeerId, SignedPeerRecord, MultiAddress, AddressInfo, init, `$` from pkg/libp2p import Cid, PeerId, SignedPeerRecord, MultiAddress, AddressInfo, init, `$`
import pkg/contractabi import pkg/contractabi
import pkg/codexdht/discv5/node as dn import pkg/codexdht/discv5/node as dn
import pkg/stew/byteutils import pkg/serde/json
import pkg/stint
import pkg/questionable/results import pkg/questionable/results
import ../errors import ../errors
export json except `%`, `%*` export json
type
SerializationError = object of CodexError
UnexpectedKindError = object of SerializationError
template serialize* {.pragma.}
proc newUnexpectedKindError(
expectedType: type,
expectedKinds: string,
json: JsonNode
): ref UnexpectedKindError =
let kind = if json.isNil: "nil"
else: $json.kind
newException(UnexpectedKindError,
&"deserialization to {$expectedType} failed: expected {expectedKinds} " &
&"but got {kind}")
proc newUnexpectedKindError(
expectedType: type,
expectedKinds: set[JsonNodeKind],
json: JsonNode
): ref UnexpectedKindError =
newUnexpectedKindError(expectedType, $expectedKinds, json)
proc newUnexpectedKindError(
expectedType: type,
expectedKind: JsonNodeKind,
json: JsonNode
): ref UnexpectedKindError =
newUnexpectedKindError(expectedType, {expectedKind}, json)
template expectJsonKind(
expectedType: type,
expectedKinds: set[JsonNodeKind],
json: JsonNode
) =
if json.isNil or json.kind notin expectedKinds:
return failure(newUnexpectedKindError(expectedType, expectedKinds, json))
template expectJsonKind(
expectedType: type,
expectedKind: JsonNodeKind,
json: JsonNode
) =
expectJsonKind(expectedType, {expectedKind}, json)
proc fromJson*(
T: type enum,
json: JsonNode
): ?!T =
expectJsonKind(string, JString, json)
catch parseEnum[T](json.str)
proc fromJson*(
_: type string,
json: JsonNode
): ?!string =
if json.isNil:
let err = newException(ValueError, "'json' expected, but was nil")
return failure(err)
elif json.kind == JNull:
return success("null")
elif json.isNil or json.kind != JString:
return failure(newUnexpectedKindError(string, JString, json))
catch json.getStr
proc fromJson*(
_: type bool,
json: JsonNode
): ?!bool =
expectJsonKind(bool, JBool, json)
catch json.getBool
proc fromJson*(
_: type int,
json: JsonNode
): ?!int =
expectJsonKind(int, JInt, json)
catch json.getInt
proc fromJson*[T: SomeInteger](
_: type T,
json: JsonNode
): ?!T =
when T is uint|uint64 or (not defined(js) and int.sizeof == 4):
expectJsonKind(T, {JInt, JString}, json)
case json.kind
of JString:
let x = parseBiggestUInt(json.str)
return success cast[T](x)
else:
return success T(json.num)
else:
expectJsonKind(T, {JInt}, json)
return success cast[T](json.num)
proc fromJson*[T: SomeFloat](
_: type T,
json: JsonNode
): ?!T =
expectJsonKind(T, {JInt, JFloat, JString}, json)
if json.kind == JString:
case json.str
of "nan":
let b = NaN
return success T(b)
# dst = NaN # would fail some tests because range conversions would cause CT error
# in some cases; but this is not a hot-spot inside this branch and backend can optimize this.
of "inf":
let b = Inf
return success T(b)
of "-inf":
let b = -Inf
return success T(b)
else:
let err = newUnexpectedKindError(T, "'nan|inf|-inf'", json)
return failure(err)
else:
if json.kind == JFloat:
return success T(json.fnum)
else:
return success T(json.num)
proc fromJson*(
_: type seq[byte],
json: JsonNode
): ?!seq[byte] =
expectJsonKind(seq[byte], JString, json)
hexToSeqByte(json.getStr).catch
proc fromJson*[N: static[int], T: array[N, byte]](
_: type T,
json: JsonNode
): ?!T =
expectJsonKind(T, JString, json)
T.fromHex(json.getStr).catch
proc fromJson*[T: distinct](
_: type T,
json: JsonNode
): ?!T =
success T(? T.distinctBase.fromJson(json))
proc fromJson*[N: static[int], T: StUint[N]](
_: type T,
json: JsonNode
): ?!T =
expectJsonKind(T, JString, json)
catch parse(json.getStr, T)
proc fromJson*[T](
_: type Option[T],
json: JsonNode
): ?! Option[T] =
if json.isNil or json.kind == JNull:
return success(none T)
without val =? T.fromJson(json), error:
return failure(error)
success(val.some)
proc fromJson*( proc fromJson*(
_: type Cid, _: type Cid,
@ -184,123 +19,6 @@ proc fromJson*(
expectJsonKind(Cid, JString, json) expectJsonKind(Cid, JString, json)
Cid.init(json.str).mapFailure Cid.init(json.str).mapFailure
proc fromJson*[T](
_: type seq[T],
json: JsonNode
): ?! seq[T] =
expectJsonKind(seq[T], JArray, json)
var arr: seq[T] = @[]
for elem in json.elems:
arr.add(? T.fromJson(elem))
success arr
proc fromJson*[T: ref object or object](
_: type T,
json: JsonNode
): ?!T =
expectJsonKind(T, JObject, json)
var res = when type(T) is ref: T.new() else: T.default
# Leave this in, it's good for debugging:
# trace "deserializing object", to = $T, json
for name, value in fieldPairs(when type(T) is ref: res[] else: res):
if json{name} != nil:
without parsed =? type(value).fromJson(json{name}), e:
error "error deserializing field",
field = $T & "." & name,
json = json{name},
error = e.msg
return failure(e)
value = parsed
success(res)
proc fromJson*[T: object](
_: type T,
bytes: seq[byte]
): ?!T =
let json = ?catch parseJson(string.fromBytes(bytes))
T.fromJson(json)
proc fromJson*[T: ref object](
_: type T,
bytes: seq[byte]
): ?!T =
let json = ?catch parseJson(string.fromBytes(bytes))
T.fromJson(json)
func `%`*(s: string): JsonNode = newJString(s)
func `%`*(n: uint): JsonNode =
if n > cast[uint](int.high):
newJString($n)
else:
newJInt(BiggestInt(n))
func `%`*(n: int): JsonNode = newJInt(n)
func `%`*(n: BiggestUInt): JsonNode =
if n > cast[BiggestUInt](BiggestInt.high):
newJString($n)
else:
newJInt(BiggestInt(n))
func `%`*(n: BiggestInt): JsonNode = newJInt(n)
func `%`*(n: float): JsonNode =
if n != n: newJString("nan")
elif n == Inf: newJString("inf")
elif n == -Inf: newJString("-inf")
else: newJFloat(n)
func `%`*(b: bool): JsonNode = newJBool(b)
func `%`*(keyVals: openArray[tuple[key: string, val: JsonNode]]): JsonNode =
if keyVals.len == 0: return newJArray()
let jObj = newJObject()
for key, val in items(keyVals): jObj.fields[key] = val
jObj
template `%`*(j: JsonNode): JsonNode = j
func `%`*[T](table: Table[string, T]|OrderedTable[string, T]): JsonNode =
let jObj = newJObject()
for k, v in table: jObj[k] = ? %v
jObj
func `%`*[T](opt: Option[T]): JsonNode =
if opt.isSome: %(opt.get) else: newJNull()
func `%`*[T: object](obj: T): JsonNode =
let jsonObj = newJObject()
for name, value in obj.fieldPairs:
when value.hasCustomPragma(serialize):
jsonObj[name] = %value
jsonObj
func `%`*[T: ref object](obj: T): JsonNode =
let jsonObj = newJObject()
for name, value in obj[].fieldPairs:
when value.hasCustomPragma(serialize):
jsonObj[name] = %(value)
jsonObj
proc `%`*(o: enum): JsonNode = % $o
func `%`*(stint: StInt|StUint): JsonNode = %stint.toString
func `%`*(cstr: cstring): JsonNode = % $cstr
func `%`*(arr: openArray[byte]): JsonNode = % arr.to0xHex
func `%`*[T](elements: openArray[T]): JsonNode =
let jObj = newJArray()
for elem in elements: jObj.add(%elem)
jObj
func `%`*[T: distinct](id: T): JsonNode =
type baseType = T.distinctBase
% baseType(id)
func `%`*(cid: Cid): JsonNode = % $cid func `%`*(cid: Cid): JsonNode = % $cid
func `%`*(obj: PeerId): JsonNode = % $obj func `%`*(obj: PeerId): JsonNode = % $obj
@ -314,36 +32,3 @@ func `%`*(obj: AddressInfo): JsonNode = % $obj.address
func `%`*(obj: MultiAddress): JsonNode = % $obj func `%`*(obj: MultiAddress): JsonNode = % $obj
func `%`*(address: ethers.Address): JsonNode = % $address func `%`*(address: ethers.Address): JsonNode = % $address
func toJson*[T](item: T): string = $(%item)
proc toJsnImpl(x: NimNode): NimNode =
case x.kind
of nnkBracket: # array
if x.len == 0: return newCall(bindSym"newJArray")
result = newNimNode(nnkBracket)
for i in 0 ..< x.len:
result.add(toJsnImpl(x[i]))
result = newCall(bindSym("%", brOpen), result)
of nnkTableConstr: # object
if x.len == 0: return newCall(bindSym"newJObject")
result = newNimNode(nnkTableConstr)
for i in 0 ..< x.len:
x[i].expectKind nnkExprColonExpr
result.add newTree(nnkExprColonExpr, x[i][0], toJsnImpl(x[i][1]))
result = newCall(bindSym("%", brOpen), result)
of nnkCurly: # empty object
x.expectLen(0)
result = newCall(bindSym"newJObject")
of nnkNilLit:
result = newCall(bindSym"newJNull")
of nnkPar:
if x.len == 1: result = toJsnImpl(x[0])
else: result = newCall(bindSym("%", brOpen), x)
else:
result = newCall(bindSym("%", brOpen), x)
macro `%*`*(x: untyped): JsonNode =
## Convert an expression to a JsonNode directly, without having to specify
## `%` for every element.
result = toJsnImpl(x)

1
tests/codex/sales/testslotqueue.nim
View File

@ -1,5 +1,4 @@
import std/sequtils import std/sequtils
import pkg/chronicles
import pkg/chronos import pkg/chronos
import pkg/datastore import pkg/datastore
import pkg/questionable import pkg/questionable

4
tests/codex/slots/backends/testcircomcompat.nim
View File

@ -8,11 +8,11 @@ import ../../../asynctest
import pkg/chronos import pkg/chronos
import pkg/poseidon2 import pkg/poseidon2
import pkg/datastore import pkg/datastore
import pkg/serde/json
import pkg/codex/slots {.all.} import pkg/codex/slots {.all.}
import pkg/codex/slots/types {.all.} import pkg/codex/slots/types {.all.}
import pkg/codex/merkletree import pkg/codex/merkletree
import pkg/codex/utils/json
import pkg/codex/codextypes import pkg/codex/codextypes
import pkg/codex/manifest import pkg/codex/manifest
import pkg/codex/stores import pkg/codex/stores
@ -33,7 +33,7 @@ suite "Test Circom Compat Backend - control inputs":
setup: setup:
let let
inputData = readFile("tests/circuits/fixtures/input.json") inputData = readFile("tests/circuits/fixtures/input.json")
inputJson = parseJson(inputData) inputJson = !JsonNode.parse(inputData)
proofInputs = Poseidon2Hash.jsonToProofInput(inputJson) proofInputs = Poseidon2Hash.jsonToProofInput(inputJson)
circom = CircomCompat.init(r1cs, wasm, zkey) circom = CircomCompat.init(r1cs, wasm, zkey)

2
tests/codex/slots/sampler/testsampler.nim
View File

@ -35,7 +35,7 @@ suite "Test Sampler - control samples":
setup: setup:
inputData = readFile("tests/circuits/fixtures/input.json") inputData = readFile("tests/circuits/fixtures/input.json")
inputJson = parseJson(inputData) inputJson = !JsonNode.parse(inputData)
proofInput = Poseidon2Hash.jsonToProofInput(inputJson) proofInput = Poseidon2Hash.jsonToProofInput(inputJson)
test "Should verify control samples": test "Should verify control samples":

4
tests/codex/slots/sampler/testutils.nim
View File

@ -18,9 +18,9 @@ import pkg/codex/contracts/requests
import pkg/codex/contracts import pkg/codex/contracts
import pkg/codex/merkletree import pkg/codex/merkletree
import pkg/codex/stores/cachestore import pkg/codex/stores/cachestore
import pkg/codex/slots/types import pkg/codex/slots/types
import pkg/codex/slots/sampler/utils import pkg/codex/slots/sampler/utils
import pkg/codex/utils/json
import ../backends/helpers import ../backends/helpers
import ../../helpers import ../../helpers
@ -38,7 +38,7 @@ asyncchecksuite "Test proof sampler utils":
setup: setup:
inputData = readFile("tests/circuits/fixtures/input.json") inputData = readFile("tests/circuits/fixtures/input.json")
inputJson = parseJson(inputData) inputJson = !JsonNode.parse(inputData)
proofInput = Poseidon2Hash.jsonToProofInput(inputJson) proofInput = Poseidon2Hash.jsonToProofInput(inputJson)
test "Extract low bits": test "Extract low bits":

5
tests/codex/testlogutils.nim
View File

@ -7,6 +7,7 @@ import pkg/codex/contracts/requests
import pkg/codex/logutils import pkg/codex/logutils
import pkg/codex/purchasing/purchaseid import pkg/codex/purchasing/purchaseid
import pkg/codex/units import pkg/codex/units
import pkg/codex/utils/json
import pkg/libp2p/cid import pkg/libp2p/cid
import pkg/libp2p/multiaddress import pkg/libp2p/multiaddress
import pkg/questionable import pkg/questionable
@ -61,8 +62,8 @@ checksuite "Test logging output":
outputLines.contains(toFind) outputLines.contains(toFind)
template loggedJson(prop, expected): auto = template loggedJson(prop, expected): auto =
let json = $ parseJson(outputJson){prop} let jsonVal = !JsonNode.parse(outputJson)
json == expected $ jsonVal{prop} == expected
template log(val) = template log(val) =
testlines.trace "test", val testlines.trace "test", val

1
tests/codex/testutils.nim
View File

@ -1,4 +1,3 @@
import ./utils/testjson
import ./utils/testoptionalcast import ./utils/testoptionalcast
import ./utils/testkeyutils import ./utils/testkeyutils
import ./utils/testasyncstatemachine import ./utils/testasyncstatemachine

345
tests/codex/utils/testjson.nim
View File

@ -1,345 +0,0 @@
import std/math
import std/options
import std/strformat
import std/strutils
import std/unittest
import pkg/stew/byteutils
import pkg/stint
import pkg/codex/contracts/requests
from pkg/codex/rest/json import RestPurchase
import pkg/codex/logutils
import pkg/codex/utils/json as utilsjson
import pkg/questionable
import pkg/questionable/results
import pkg/libp2p
import ../helpers
checksuite "json serialization":
var request: StorageRequest
var requestJson: JsonNode
func flatten(s: string): string =
s.replace(" ")
.replace("\n")
setup:
request = StorageRequest(
client: Address.init("0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174").get(),
ask: StorageAsk(
slots: 4,
slotSize: (1 * 1024 * 1024 * 1024).u256, # 1 Gigabyte
duration: (10 * 60 * 60).u256, # 10 hours
collateral: 200.u256,
proofProbability: 4.u256, # require a proof roughly once every 4 periods
reward: 84.u256,
maxSlotLoss: 2 # 2 slots can be freed without data considered to be lost
),
content: StorageContent(
cid: "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob",
merkleRoot: array[32, byte].fromHex("0xc066dd7e405de5a795ce1e765209cfaa6de6c74829c607d1a2fe53107107a298")
),
expiry: 1691545330.u256,
nonce: Nonce array[32, byte].fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1")
)
requestJson = """{
"client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174",
"ask": {
"slots": 4,
"slotSize": "1073741824",
"duration": "36000",
"proofProbability": "4",
"reward": "84",
"collateral": "200",
"maxSlotLoss": 2
},
"content": {
"cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob",
"merkleRoot": "0xc066dd7e405de5a795ce1e765209cfaa6de6c74829c607d1a2fe53107107a298"
},
"expiry": "1691545330",
"nonce": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"
}""".parseJson
test "serializes UInt256 to non-hex string representation":
check (% 100000.u256) == newJString("100000")
test "serializes sequence to an array":
let json = % @[1, 2, 3]
let expected = "[1,2,3]"
check $json == expected
test "serializes Option[T] when has a value":
let obj = %(some 1)
let expected = "1"
check $obj == expected
test "serializes Option[T] when doesn't have a value":
let obj = %(none int)
let expected = "null"
check $obj == expected
test "serializes uints int.high or smaller":
let largeUInt: uint = uint(int.high)
check %largeUInt == newJInt(BiggestInt(largeUInt))
test "serializes large uints":
let largeUInt: uint = uint(int.high) + 1'u
check %largeUInt == newJString($largeUInt)
test "serializes Inf float":
check %Inf == newJString("inf")
test "serializes -Inf float":
check %(-Inf) == newJString("-inf")
test "deserializes NaN float":
check %NaN == newJString("nan")
test "can construct json objects with %*":
type MyObj = object
mystring {.serialize.}: string
myint {.serialize.}: int
myoption {.serialize.}: ?bool
let myobj = MyObj(mystring: "abc", myint: 123, myoption: some true)
let mystuint = 100000.u256
let json = %*{
"myobj": myobj,
"mystuint": mystuint
}
let expected = """{
"myobj": {
"mystring": "abc",
"myint": 123,
"myoption": true
},
"mystuint": "100000"
}""".flatten
check $json == expected
test "only serializes marked fields":
type MyObj = object
mystring {.serialize.}: string
myint {.serialize.}: int
mybool: bool
let obj = % MyObj(mystring: "abc", myint: 1, mybool: true)
let expected = """{
"mystring": "abc",
"myint": 1
}""".flatten
check $obj == expected
test "serializes ref objects":
type MyRef = ref object
mystring {.serialize.}: string
myint {.serialize.}: int
let obj = % MyRef(mystring: "abc", myint: 1)
let expected = """{
"mystring": "abc",
"myint": 1
}""".flatten
check $obj == expected
test "serializes RestPurchase":
let request = % RestPurchase(
request: some request,
requestId: RequestId.fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"),
error: some "error",
state: "state"
)
let expected = """{
"requestId": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1",
"request": {
"client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174",
"ask": {
"slots": 4,
"slotSize": "1073741824",
"duration": "36000",
"proofProbability": "4",
"reward": "84",
"collateral": "200",
"maxSlotLoss": 2
},
"content": {
"cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob"
},
"expiry": "1691545330"
},
"state": "state",
"error": "error"
}""".flatten
check $request == expected
test "serializes StorageRequest":
let expected = """{
"client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174",
"ask": {
"slots": 4,
"slotSize": "1073741824",
"duration": "36000",
"proofProbability": "4",
"reward": "84",
"collateral": "200",
"maxSlotLoss": 2
},
"content": {
"cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob"
},
"expiry": "1691545330"
}""".flatten
check request.toJson == expected
test "deserialize enum":
let json = newJString($CidVersion.CIDv1)
check !CidVersion.fromJson(json) == CIDv1
test "deserializes UInt256 from non-hex string representation":
let json = newJString("100000")
check !UInt256.fromJson(json) == 100000.u256
test "deserializes Option[T] when has a value":
let json = newJInt(1)
check (!fromJson(?int, json) == some 1)
test "deserializes Option[T] when doesn't have a value":
let json = newJNull()
check !fromJson(?int, json) == none int
test "deserializes float":
let json = newJFloat(1.234)
check !float.fromJson(json) == 1.234
test "deserializes Inf float":
let json = newJString("inf")
check !float.fromJson(json) == Inf
test "deserializes -Inf float":
let json = newJString("-inf")
check !float.fromJson(json) == -Inf
test "deserializes NaN float":
let json = newJString("nan")
check float.fromJson(json).get.isNaN
test "deserializes array to sequence":
let expected = @[1, 2, 3]
let json = "[1,2,3]".parseJson
check !seq[int].fromJson(json) == expected
test "deserializes uints int.high or smaller":
let largeUInt: uint = uint(int.high)
let json = newJInt(BiggestInt(largeUInt))
check !uint.fromJson(json) == largeUInt
test "deserializes large uints":
let largeUInt: uint = uint(int.high) + 1'u
let json = newJString($BiggestUInt(largeUInt))
check !uint.fromJson(json) == largeUInt
test "can deserialize json objects":
type MyObj = object
mystring: string
myint: int
myoption: ?bool
let expected = MyObj(mystring: "abc", myint: 123, myoption: some true)
let json = parseJson("""{
"mystring": "abc",
"myint": 123,
"myoption": true
}""")
check !MyObj.fromJson(json) == expected
test "ignores serialize pragma when deserializing":
type MyObj = object
mystring {.serialize.}: string
mybool: bool
let expected = MyObj(mystring: "abc", mybool: true)
let json = parseJson("""{
"mystring": "abc",
"mybool": true
}""")
check !MyObj.fromJson(json) == expected
test "deserializes objects with extra fields":
type MyObj = object
mystring: string
mybool: bool
let expected = MyObj(mystring: "abc", mybool: true)
let json = """{
"mystring": "abc",
"mybool": true,
"extra": "extra"
}""".parseJson
check !MyObj.fromJson(json) == expected
test "deserializes objects with less fields":
type MyObj = object
mystring: string
mybool: bool
let expected = MyObj(mystring: "abc", mybool: false)
let json = """{
"mystring": "abc"
}""".parseJson
check !MyObj.fromJson(json) == expected
test "deserializes ref objects":
type MyRef = ref object
mystring: string
myint: int
let expected = MyRef(mystring: "abc", myint: 1)
let json = """{
"mystring": "abc",
"myint": 1
}""".parseJson
let deserialized = !MyRef.fromJson(json)
check deserialized.mystring == expected.mystring
check deserialized.myint == expected.myint
test "deserializes Cid":
let
jCid = newJString("zdj7Wakya26ggQWkvMdHYFcPgZ7Qh2HdMooQDDFDHkk4uHS14")
cid = "zdj7Wakya26ggQWkvMdHYFcPgZ7Qh2HdMooQDDFDHkk4uHS14"
check:
!Cid.fromJson(jCid) == !Cid.init(cid).mapFailure
test "deserializes StorageRequest":
check !StorageRequest.fromJson(requestJson) == request
test "deserializes RestPurchase":
let json = """{
"requestId": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1",
"state": "state",
"error": "error"
}""".parseJson
json["request"] = requestJson
let expected = RestPurchase(
requestId: RequestId.fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"),
state: "state",
error: some "error",
request: some request
)
check !RestPurchase.fromJson(json) == expected

23
tests/integration/codexclient.nim
View File

@ -19,9 +19,9 @@ type CodexClient* = ref object
proc new*(_: type CodexClient, baseurl: string): CodexClient = proc new*(_: type CodexClient, baseurl: string): CodexClient =
CodexClient(http: newHttpClient(), baseurl: baseurl) CodexClient(http: newHttpClient(), baseurl: baseurl)
proc info*(client: CodexClient): JsonNode = proc info*(client: CodexClient): ?!JsonNode =
let url = client.baseurl & "/debug/info" let url = client.baseurl & "/debug/info"
client.http.getContent(url).parseJson() JsonNode.parse( client.http.getContent(url) )
proc setLogLevel*(client: CodexClient, level: string) = proc setLogLevel*(client: CodexClient, level: string) =
let url = client.baseurl & "/debug/chronicles/loglevel?level=" & level let url = client.baseurl & "/debug/chronicles/loglevel?level=" & level
@ -52,8 +52,7 @@ proc list*(client: CodexClient): ?!seq[RestContent] =
if response.status != "200 OK": if response.status != "200 OK":
return failure(response.status) return failure(response.status)
let json = ? parseJson(response.body).catch seq[RestContent].fromJson(response.body)
seq[RestContent].fromJson(json)
proc space*(client: CodexClient): ?!RestRepoStore = proc space*(client: CodexClient): ?!RestRepoStore =
let url = client.baseurl & "/space" let url = client.baseurl & "/space"
@ -62,8 +61,7 @@ proc space*(client: CodexClient): ?!RestRepoStore =
if response.status != "200 OK": if response.status != "200 OK":
return failure(response.status) return failure(response.status)
let json = ? parseJson(response.body).catch RestRepoStore.fromJson(response.body)
RestRepoStore.fromJson(json)
proc requestStorageRaw*( proc requestStorageRaw*(
client: CodexClient, client: CodexClient,
@ -116,8 +114,7 @@ proc getPurchase*(client: CodexClient, purchaseId: PurchaseId): ?!RestPurchase =
let url = client.baseurl & "/storage/purchases/" & purchaseId.toHex let url = client.baseurl & "/storage/purchases/" & purchaseId.toHex
try: try:
let body = client.http.getContent(url) let body = client.http.getContent(url)
let json = ? parseJson(body).catch return RestPurchase.fromJson(body)
return RestPurchase.fromJson(json)
except CatchableError as e: except CatchableError as e:
return failure e.msg return failure e.msg
@ -125,16 +122,14 @@ proc getSalesAgent*(client: CodexClient, slotId: SlotId): ?!RestSalesAgent =
let url = client.baseurl & "/sales/slots/" & slotId.toHex let url = client.baseurl & "/sales/slots/" & slotId.toHex
try: try:
let body = client.http.getContent(url) let body = client.http.getContent(url)
let json = ? parseJson(body).catch return RestSalesAgent.fromJson(body)
return RestSalesAgent.fromJson(json)
except CatchableError as e: except CatchableError as e:
return failure e.msg return failure e.msg
proc getSlots*(client: CodexClient): ?!seq[Slot] = proc getSlots*(client: CodexClient): ?!seq[Slot] =
let url = client.baseurl & "/sales/slots" let url = client.baseurl & "/sales/slots"
let body = client.http.getContent(url) let body = client.http.getContent(url)
let json = ? parseJson(body).catch seq[Slot].fromJson(body)
seq[Slot].fromJson(json)
proc postAvailability*( proc postAvailability*(
client: CodexClient, client: CodexClient,
@ -151,13 +146,13 @@ proc postAvailability*(
} }
let response = client.http.post(url, $json) let response = client.http.post(url, $json)
doAssert response.status == "200 OK", "expected 200 OK, got " & response.status & ", body: " & response.body doAssert response.status == "200 OK", "expected 200 OK, got " & response.status & ", body: " & response.body
Availability.fromJson(response.body.parseJson) Availability.fromJson(response.body)
proc getAvailabilities*(client: CodexClient): ?!seq[Availability] = proc getAvailabilities*(client: CodexClient): ?!seq[Availability] =
## Call sales availability REST endpoint ## Call sales availability REST endpoint
let url = client.baseurl & "/sales/availability" let url = client.baseurl & "/sales/availability"
let body = client.http.getContent(url) let body = client.http.getContent(url)
seq[Availability].fromJson(parseJson(body)) seq[Availability].fromJson(body)
proc close*(client: CodexClient) = proc close*(client: CodexClient) =
client.http.close() client.http.close()

6
tests/integration/multinodes.nim
View File

@ -288,7 +288,11 @@ template multinodesuite*(name: string, body: untyped) =
node: node node: node
) )
if clients().len == 1: if clients().len == 1:
bootstrap = CodexProcess(node).client.info()["spr"].getStr() without ninfo =? CodexProcess(node).client.info():
# raise CatchableError instead of Defect (with .get or !) so we
# can gracefully shutdown and prevent zombies
raiseMultiNodeSuiteError "Failed to get node info"
bootstrap = ninfo["spr"].getStr()
if var providers =? nodeConfigs.providers: if var providers =? nodeConfigs.providers:
failAndTeardownOnError "failed to start provider nodes": failAndTeardownOnError "failed to start provider nodes":

2
tests/integration/testIntegration.nim
View File

@ -26,7 +26,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
await ethProvider.advanceTime(1.u256) await ethProvider.advanceTime(1.u256)
test "nodes can print their peer information": test "nodes can print their peer information":
check client1.info() != client2.info() check !client1.info() != !client2.info()
test "nodes can set chronicles log level": test "nodes can set chronicles log level":
client1.setLogLevel("DEBUG;TRACE:codex") client1.setLogLevel("DEBUG;TRACE:codex")

2
tests/integration/twonodes.nim
View File

@ -52,7 +52,7 @@ template twonodessuite*(name: string, debug1, debug2: string, body) =
node1 = startNode(node1Args, debug = debug1) node1 = startNode(node1Args, debug = debug1)
node1.waitUntilStarted() node1.waitUntilStarted()
let bootstrap = client1.info()["spr"].getStr() let bootstrap = (!client1.info()["spr"]).getStr()
var node2Args = @[ var node2Args = @[
"--api-port=8081", "--api-port=8081",

2
tests/testTaiko.nim
View File

@ -29,7 +29,7 @@ suite "Taiko L2 Integration Tests":
]) ])
node1.waitUntilStarted() node1.waitUntilStarted()
let bootstrap = node1.client.info()["spr"].getStr() let bootstrap = (!node1.client.info())["spr"].getStr()
node2 = startNode([ node2 = startNode([
"--data-dir=" & createTempDir("", ""), "--data-dir=" & createTempDir("", ""),

1
vendor/nim-serde vendored Submodule

@ -0,0 +1 @@
Subproject commit a261c3d214b87d9de0338d6cfd5d134da03d2dc3