From f567f4ec15dbdf0f208aea5370bf3ba6bd6c14a4 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Tue, 19 Mar 2024 14:25:13 +1100 Subject: [PATCH] refactor: use serde for json de/serialization instead of utils/json (#704) * json > nim-serde bump Should wait until serde is integrated into nim-ethers before making these changes as there will be less import exceptions required. * bump nim-serde * change func to proc due to chronicles side effects * import serde into utils/json, use as proxy import nim-serde into utils/json and use utils/json as a proxy for serde functions, including overloading `%` and `fromJson` for application types. * update tests to use serde * bump serde to latest * remove testjson -- no longer needed * bump serde in nimble * updates to reconcile rebase with master --- .gitmodules | 3 + codex.nimble | 1 + codex/logutils.nim | 15 +- codex/rest/json.nim | 4 +- codex/slots/builder/builder.nim | 2 +- codex/slots/sampler/sampler.nim | 2 +- codex/utils/json.nim | 319 +--------------- tests/codex/sales/testslotqueue.nim | 1 - .../codex/slots/backends/testcircomcompat.nim | 4 +- tests/codex/slots/sampler/testsampler.nim | 2 +- tests/codex/slots/sampler/testutils.nim | 4 +- tests/codex/testlogutils.nim | 5 +- tests/codex/testutils.nim | 1 - tests/codex/utils/testjson.nim | 345 ------------------ tests/integration/codexclient.nim | 23 +- tests/integration/multinodes.nim | 6 +- tests/integration/testIntegration.nim | 2 +- tests/integration/twonodes.nim | 2 +- tests/testTaiko.nim | 2 +- vendor/nim-serde | 1 + 20 files changed, 43 insertions(+), 701 deletions(-) delete mode 100644 tests/codex/utils/testjson.nim create mode 160000 vendor/nim-serde diff --git a/.gitmodules b/.gitmodules index 1089a309..c5162818 100644 --- a/.gitmodules +++ b/.gitmodules @@ -209,3 +209,6 @@ url = https://github.com/codex-storage/codex-storage-proofs-circuits.git ignore = untracked branch = master +[submodule "vendor/nim-serde"] + path = vendor/nim-serde + url = https://github.com/codex-storage/nim-serde.git diff --git a/codex.nimble b/codex.nimble index 42347710..7a99779f 100644 --- a/codex.nimble +++ b/codex.nimble @@ -22,6 +22,7 @@ requires "presto" requires "protobuf_serialization >= 0.2.0 & < 0.3.0" requires "questionable >= 0.10.13 & < 0.11.0" requires "secp256k1" +requires "serde >= 1.0.0 & < 2.0.0" requires "stew" requires "upraises >= 0.1.0 & < 0.2.0" requires "toml_serialization" diff --git a/codex/logutils.nim b/codex/logutils.nim index 6abc21a5..eb75e906 100644 --- a/codex/logutils.nim +++ b/codex/logutils.nim @@ -17,16 +17,16 @@ ## module, and specifying `formatIt`. If textlines log output and json log output ## need to be different, overload `formatIt` and specify a `LogFormat`. If json ## serialization is needed, it can be declared with a `%` proc. `logutils` -## imports and exports `utils/json` which handles the de/serialization, examples +## imports and exports `nim-serde` which handles the de/serialization, examples ## below. **Only `codex/logutils` needs to be imported.** ## ## Using `logutils` in the Codex codebase: ## - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils` ## - most of `chronicles` is exported by `logutils` -## - Instead of importing `std/json`, import `pkg/codex/utils/json` -## - `std/json` is exported by `utils/json` which is exported by `logutils` +## - Instead of importing `std/json`, import `pkg/serde/json` +## - `std/json` is exported by `serde` which is exported by `logutils` ## - Instead of importing `pkg/nim-json-serialization`, import -## `pkg/codex/utils/json` +## `pkg/serde/json` or use codex-specific overloads by importing `utils/json` ## - one of the goals is to remove the use of `nim-json-serialization` ## ## ```nim @@ -54,7 +54,7 @@ ## # chronicles json output ## {"lvl":"TRC","msg":"test","tid":14397405,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}} ## ``` -## In this case, `BlockAddress` is just an object, so `utils/json` can handle +## In this case, `BlockAddress` is just an object, so `nim-serde` can handle ## serializing it without issue (only fields annotated with `{.serialize.}` will ## serialize (aka opt-in serialization)). ## @@ -95,20 +95,19 @@ import pkg/chronicles except toJson, `%` from pkg/libp2p import Cid, MultiAddress, `$` import pkg/questionable import pkg/questionable/results +import ./utils/json except formatIt # TODO: remove exception? import pkg/stew/byteutils import pkg/stint import pkg/upraises -import ./utils/json - export byteutils export chronicles except toJson, formatIt, `%` export questionable export sequtils +export json except formatIt export strutils export sugar export upraises -export json export results func shortLog*(long: string, ellipses = "*", start = 3, stop = 6): string = diff --git a/codex/rest/json.nim b/codex/rest/json.nim index 181ea584..748a8b5b 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -104,11 +104,11 @@ proc init*(_: type RestNodeId, id: NodeId): RestNodeId = id: id ) -func `%`*(obj: StorageRequest | Slot): JsonNode = +proc `%`*(obj: StorageRequest | Slot): JsonNode = let jsonObj = newJObject() for k, v in obj.fieldPairs: jsonObj[k] = %v jsonObj["id"] = %(obj.id) return jsonObj -func `%`*(obj: RestNodeId): JsonNode = % $obj.id +proc `%`*(obj: RestNodeId): JsonNode = % $obj.id diff --git a/codex/slots/builder/builder.nim b/codex/slots/builder/builder.nim index 674fd28e..f999a514 100644 --- a/codex/slots/builder/builder.nim +++ b/codex/slots/builder/builder.nim @@ -15,11 +15,11 @@ import std/sugar import pkg/libp2p import pkg/chronos -import pkg/chronicles import pkg/questionable import pkg/questionable/results import pkg/constantine/math/io/io_fields +import ../../logutils import ../../utils import ../../stores import ../../manifest diff --git a/codex/slots/sampler/sampler.nim b/codex/slots/sampler/sampler.nim index aef3545d..d22121c2 100644 --- a/codex/slots/sampler/sampler.nim +++ b/codex/slots/sampler/sampler.nim @@ -9,12 +9,12 @@ import std/sugar -import pkg/chronicles import pkg/chronos import pkg/questionable import pkg/questionable/results import pkg/stew/arrayops +import ../../logutils import ../../market import ../../blocktype as bt import ../../merkletree diff --git a/codex/utils/json.nim b/codex/utils/json.nim index 080d4e89..4113b632 100644 --- a/codex/utils/json.nim +++ b/codex/utils/json.nim @@ -1,181 +1,16 @@ -import std/json except `%`, `%*` -import std/macros import std/options -import std/strutils -import std/strformat -import std/tables import std/typetraits from pkg/ethers import Address from pkg/libp2p import Cid, PeerId, SignedPeerRecord, MultiAddress, AddressInfo, init, `$` import pkg/contractabi import pkg/codexdht/discv5/node as dn -import pkg/stew/byteutils -import pkg/stint +import pkg/serde/json import pkg/questionable/results import ../errors -export json except `%`, `%*` +export json -type - SerializationError = object of CodexError - UnexpectedKindError = object of SerializationError - -template serialize* {.pragma.} - -proc newUnexpectedKindError( - expectedType: type, - expectedKinds: string, - json: JsonNode -): ref UnexpectedKindError = - let kind = if json.isNil: "nil" - else: $json.kind - newException(UnexpectedKindError, - &"deserialization to {$expectedType} failed: expected {expectedKinds} " & - &"but got {kind}") - -proc newUnexpectedKindError( - expectedType: type, - expectedKinds: set[JsonNodeKind], - json: JsonNode -): ref UnexpectedKindError = - newUnexpectedKindError(expectedType, $expectedKinds, json) - -proc newUnexpectedKindError( - expectedType: type, - expectedKind: JsonNodeKind, - json: JsonNode -): ref UnexpectedKindError = - newUnexpectedKindError(expectedType, {expectedKind}, json) - -template expectJsonKind( - expectedType: type, - expectedKinds: set[JsonNodeKind], - json: JsonNode -) = - if json.isNil or json.kind notin expectedKinds: - return failure(newUnexpectedKindError(expectedType, expectedKinds, json)) - -template expectJsonKind( - expectedType: type, - expectedKind: JsonNodeKind, - json: JsonNode -) = - expectJsonKind(expectedType, {expectedKind}, json) - -proc fromJson*( - T: type enum, - json: JsonNode -): ?!T = - expectJsonKind(string, JString, json) - catch parseEnum[T](json.str) - -proc fromJson*( - _: type string, - json: JsonNode -): ?!string = - if json.isNil: - let err = newException(ValueError, "'json' expected, but was nil") - return failure(err) - elif json.kind == JNull: - return success("null") - elif json.isNil or json.kind != JString: - return failure(newUnexpectedKindError(string, JString, json)) - catch json.getStr - -proc fromJson*( - _: type bool, - json: JsonNode -): ?!bool = - expectJsonKind(bool, JBool, json) - catch json.getBool - -proc fromJson*( - _: type int, - json: JsonNode -): ?!int = - expectJsonKind(int, JInt, json) - catch json.getInt - -proc fromJson*[T: SomeInteger]( - _: type T, - json: JsonNode -): ?!T = - when T is uint|uint64 or (not defined(js) and int.sizeof == 4): - expectJsonKind(T, {JInt, JString}, json) - case json.kind - of JString: - let x = parseBiggestUInt(json.str) - return success cast[T](x) - else: - return success T(json.num) - else: - expectJsonKind(T, {JInt}, json) - return success cast[T](json.num) - -proc fromJson*[T: SomeFloat]( - _: type T, - json: JsonNode -): ?!T = - expectJsonKind(T, {JInt, JFloat, JString}, json) - if json.kind == JString: - case json.str - of "nan": - let b = NaN - return success T(b) - # dst = NaN # would fail some tests because range conversions would cause CT error - # in some cases; but this is not a hot-spot inside this branch and backend can optimize this. - of "inf": - let b = Inf - return success T(b) - of "-inf": - let b = -Inf - return success T(b) - else: - let err = newUnexpectedKindError(T, "'nan|inf|-inf'", json) - return failure(err) - else: - if json.kind == JFloat: - return success T(json.fnum) - else: - return success T(json.num) - -proc fromJson*( - _: type seq[byte], - json: JsonNode -): ?!seq[byte] = - expectJsonKind(seq[byte], JString, json) - hexToSeqByte(json.getStr).catch - -proc fromJson*[N: static[int], T: array[N, byte]]( - _: type T, - json: JsonNode -): ?!T = - expectJsonKind(T, JString, json) - T.fromHex(json.getStr).catch - -proc fromJson*[T: distinct]( - _: type T, - json: JsonNode -): ?!T = - success T(? T.distinctBase.fromJson(json)) - -proc fromJson*[N: static[int], T: StUint[N]]( - _: type T, - json: JsonNode -): ?!T = - expectJsonKind(T, JString, json) - catch parse(json.getStr, T) - -proc fromJson*[T]( - _: type Option[T], - json: JsonNode -): ?! Option[T] = - if json.isNil or json.kind == JNull: - return success(none T) - without val =? T.fromJson(json), error: - return failure(error) - success(val.some) proc fromJson*( _: type Cid, @@ -184,123 +19,6 @@ proc fromJson*( expectJsonKind(Cid, JString, json) Cid.init(json.str).mapFailure -proc fromJson*[T]( - _: type seq[T], - json: JsonNode -): ?! seq[T] = - expectJsonKind(seq[T], JArray, json) - var arr: seq[T] = @[] - for elem in json.elems: - arr.add(? T.fromJson(elem)) - success arr - -proc fromJson*[T: ref object or object]( - _: type T, - json: JsonNode -): ?!T = - expectJsonKind(T, JObject, json) - var res = when type(T) is ref: T.new() else: T.default - - # Leave this in, it's good for debugging: - # trace "deserializing object", to = $T, json - for name, value in fieldPairs(when type(T) is ref: res[] else: res): - if json{name} != nil: - without parsed =? type(value).fromJson(json{name}), e: - error "error deserializing field", - field = $T & "." & name, - json = json{name}, - error = e.msg - return failure(e) - value = parsed - success(res) - -proc fromJson*[T: object]( - _: type T, - bytes: seq[byte] -): ?!T = - let json = ?catch parseJson(string.fromBytes(bytes)) - T.fromJson(json) - -proc fromJson*[T: ref object]( - _: type T, - bytes: seq[byte] -): ?!T = - let json = ?catch parseJson(string.fromBytes(bytes)) - T.fromJson(json) - -func `%`*(s: string): JsonNode = newJString(s) - -func `%`*(n: uint): JsonNode = - if n > cast[uint](int.high): - newJString($n) - else: - newJInt(BiggestInt(n)) - -func `%`*(n: int): JsonNode = newJInt(n) - -func `%`*(n: BiggestUInt): JsonNode = - if n > cast[BiggestUInt](BiggestInt.high): - newJString($n) - else: - newJInt(BiggestInt(n)) - -func `%`*(n: BiggestInt): JsonNode = newJInt(n) - -func `%`*(n: float): JsonNode = - if n != n: newJString("nan") - elif n == Inf: newJString("inf") - elif n == -Inf: newJString("-inf") - else: newJFloat(n) - -func `%`*(b: bool): JsonNode = newJBool(b) - -func `%`*(keyVals: openArray[tuple[key: string, val: JsonNode]]): JsonNode = - if keyVals.len == 0: return newJArray() - let jObj = newJObject() - for key, val in items(keyVals): jObj.fields[key] = val - jObj - -template `%`*(j: JsonNode): JsonNode = j - -func `%`*[T](table: Table[string, T]|OrderedTable[string, T]): JsonNode = - let jObj = newJObject() - for k, v in table: jObj[k] = ? %v - jObj - -func `%`*[T](opt: Option[T]): JsonNode = - if opt.isSome: %(opt.get) else: newJNull() - -func `%`*[T: object](obj: T): JsonNode = - let jsonObj = newJObject() - for name, value in obj.fieldPairs: - when value.hasCustomPragma(serialize): - jsonObj[name] = %value - jsonObj - -func `%`*[T: ref object](obj: T): JsonNode = - let jsonObj = newJObject() - for name, value in obj[].fieldPairs: - when value.hasCustomPragma(serialize): - jsonObj[name] = %(value) - jsonObj - -proc `%`*(o: enum): JsonNode = % $o - -func `%`*(stint: StInt|StUint): JsonNode = %stint.toString - -func `%`*(cstr: cstring): JsonNode = % $cstr - -func `%`*(arr: openArray[byte]): JsonNode = % arr.to0xHex - -func `%`*[T](elements: openArray[T]): JsonNode = - let jObj = newJArray() - for elem in elements: jObj.add(%elem) - jObj - -func `%`*[T: distinct](id: T): JsonNode = - type baseType = T.distinctBase - % baseType(id) - func `%`*(cid: Cid): JsonNode = % $cid func `%`*(obj: PeerId): JsonNode = % $obj @@ -314,36 +32,3 @@ func `%`*(obj: AddressInfo): JsonNode = % $obj.address func `%`*(obj: MultiAddress): JsonNode = % $obj func `%`*(address: ethers.Address): JsonNode = % $address - -func toJson*[T](item: T): string = $(%item) - -proc toJsnImpl(x: NimNode): NimNode = - case x.kind - of nnkBracket: # array - if x.len == 0: return newCall(bindSym"newJArray") - result = newNimNode(nnkBracket) - for i in 0 ..< x.len: - result.add(toJsnImpl(x[i])) - result = newCall(bindSym("%", brOpen), result) - of nnkTableConstr: # object - if x.len == 0: return newCall(bindSym"newJObject") - result = newNimNode(nnkTableConstr) - for i in 0 ..< x.len: - x[i].expectKind nnkExprColonExpr - result.add newTree(nnkExprColonExpr, x[i][0], toJsnImpl(x[i][1])) - result = newCall(bindSym("%", brOpen), result) - of nnkCurly: # empty object - x.expectLen(0) - result = newCall(bindSym"newJObject") - of nnkNilLit: - result = newCall(bindSym"newJNull") - of nnkPar: - if x.len == 1: result = toJsnImpl(x[0]) - else: result = newCall(bindSym("%", brOpen), x) - else: - result = newCall(bindSym("%", brOpen), x) - -macro `%*`*(x: untyped): JsonNode = - ## Convert an expression to a JsonNode directly, without having to specify - ## `%` for every element. - result = toJsnImpl(x) diff --git a/tests/codex/sales/testslotqueue.nim b/tests/codex/sales/testslotqueue.nim index 2e30fb3a..ec4760f1 100644 --- a/tests/codex/sales/testslotqueue.nim +++ b/tests/codex/sales/testslotqueue.nim @@ -1,5 +1,4 @@ import std/sequtils -import pkg/chronicles import pkg/chronos import pkg/datastore import pkg/questionable diff --git a/tests/codex/slots/backends/testcircomcompat.nim b/tests/codex/slots/backends/testcircomcompat.nim index bc6240d3..08ac2d21 100644 --- a/tests/codex/slots/backends/testcircomcompat.nim +++ b/tests/codex/slots/backends/testcircomcompat.nim @@ -8,11 +8,11 @@ import ../../../asynctest import pkg/chronos import pkg/poseidon2 import pkg/datastore +import pkg/serde/json import pkg/codex/slots {.all.} import pkg/codex/slots/types {.all.} import pkg/codex/merkletree -import pkg/codex/utils/json import pkg/codex/codextypes import pkg/codex/manifest import pkg/codex/stores @@ -33,7 +33,7 @@ suite "Test Circom Compat Backend - control inputs": setup: let inputData = readFile("tests/circuits/fixtures/input.json") - inputJson = parseJson(inputData) + inputJson = !JsonNode.parse(inputData) proofInputs = Poseidon2Hash.jsonToProofInput(inputJson) circom = CircomCompat.init(r1cs, wasm, zkey) diff --git a/tests/codex/slots/sampler/testsampler.nim b/tests/codex/slots/sampler/testsampler.nim index ef04656f..2ed32011 100644 --- a/tests/codex/slots/sampler/testsampler.nim +++ b/tests/codex/slots/sampler/testsampler.nim @@ -35,7 +35,7 @@ suite "Test Sampler - control samples": setup: inputData = readFile("tests/circuits/fixtures/input.json") - inputJson = parseJson(inputData) + inputJson = !JsonNode.parse(inputData) proofInput = Poseidon2Hash.jsonToProofInput(inputJson) test "Should verify control samples": diff --git a/tests/codex/slots/sampler/testutils.nim b/tests/codex/slots/sampler/testutils.nim index 1e1bef96..ff277647 100644 --- a/tests/codex/slots/sampler/testutils.nim +++ b/tests/codex/slots/sampler/testutils.nim @@ -18,9 +18,9 @@ import pkg/codex/contracts/requests import pkg/codex/contracts import pkg/codex/merkletree import pkg/codex/stores/cachestore - import pkg/codex/slots/types import pkg/codex/slots/sampler/utils +import pkg/codex/utils/json import ../backends/helpers import ../../helpers @@ -38,7 +38,7 @@ asyncchecksuite "Test proof sampler utils": setup: inputData = readFile("tests/circuits/fixtures/input.json") - inputJson = parseJson(inputData) + inputJson = !JsonNode.parse(inputData) proofInput = Poseidon2Hash.jsonToProofInput(inputJson) test "Extract low bits": diff --git a/tests/codex/testlogutils.nim b/tests/codex/testlogutils.nim index 9f8e42ac..44e7533f 100644 --- a/tests/codex/testlogutils.nim +++ b/tests/codex/testlogutils.nim @@ -7,6 +7,7 @@ import pkg/codex/contracts/requests import pkg/codex/logutils import pkg/codex/purchasing/purchaseid import pkg/codex/units +import pkg/codex/utils/json import pkg/libp2p/cid import pkg/libp2p/multiaddress import pkg/questionable @@ -61,8 +62,8 @@ checksuite "Test logging output": outputLines.contains(toFind) template loggedJson(prop, expected): auto = - let json = $ parseJson(outputJson){prop} - json == expected + let jsonVal = !JsonNode.parse(outputJson) + $ jsonVal{prop} == expected template log(val) = testlines.trace "test", val diff --git a/tests/codex/testutils.nim b/tests/codex/testutils.nim index 937a6e86..6b4b2366 100644 --- a/tests/codex/testutils.nim +++ b/tests/codex/testutils.nim @@ -1,4 +1,3 @@ -import ./utils/testjson import ./utils/testoptionalcast import ./utils/testkeyutils import ./utils/testasyncstatemachine diff --git a/tests/codex/utils/testjson.nim b/tests/codex/utils/testjson.nim deleted file mode 100644 index 5ba7fff3..00000000 --- a/tests/codex/utils/testjson.nim +++ /dev/null @@ -1,345 +0,0 @@ -import std/math -import std/options -import std/strformat -import std/strutils -import std/unittest -import pkg/stew/byteutils -import pkg/stint -import pkg/codex/contracts/requests -from pkg/codex/rest/json import RestPurchase -import pkg/codex/logutils -import pkg/codex/utils/json as utilsjson -import pkg/questionable -import pkg/questionable/results -import pkg/libp2p -import ../helpers - -checksuite "json serialization": - var request: StorageRequest - var requestJson: JsonNode - - func flatten(s: string): string = - s.replace(" ") - .replace("\n") - - setup: - request = StorageRequest( - client: Address.init("0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174").get(), - ask: StorageAsk( - slots: 4, - slotSize: (1 * 1024 * 1024 * 1024).u256, # 1 Gigabyte - duration: (10 * 60 * 60).u256, # 10 hours - collateral: 200.u256, - proofProbability: 4.u256, # require a proof roughly once every 4 periods - reward: 84.u256, - maxSlotLoss: 2 # 2 slots can be freed without data considered to be lost - ), - content: StorageContent( - cid: "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob", - merkleRoot: array[32, byte].fromHex("0xc066dd7e405de5a795ce1e765209cfaa6de6c74829c607d1a2fe53107107a298") - ), - expiry: 1691545330.u256, - nonce: Nonce array[32, byte].fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1") - ) - requestJson = """{ - "client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174", - "ask": { - "slots": 4, - "slotSize": "1073741824", - "duration": "36000", - "proofProbability": "4", - "reward": "84", - "collateral": "200", - "maxSlotLoss": 2 - }, - "content": { - "cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob", - "merkleRoot": "0xc066dd7e405de5a795ce1e765209cfaa6de6c74829c607d1a2fe53107107a298" - }, - "expiry": "1691545330", - "nonce": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1" - }""".parseJson - - test "serializes UInt256 to non-hex string representation": - check (% 100000.u256) == newJString("100000") - - test "serializes sequence to an array": - let json = % @[1, 2, 3] - let expected = "[1,2,3]" - check $json == expected - - test "serializes Option[T] when has a value": - let obj = %(some 1) - let expected = "1" - check $obj == expected - - test "serializes Option[T] when doesn't have a value": - let obj = %(none int) - let expected = "null" - check $obj == expected - - test "serializes uints int.high or smaller": - let largeUInt: uint = uint(int.high) - check %largeUInt == newJInt(BiggestInt(largeUInt)) - - test "serializes large uints": - let largeUInt: uint = uint(int.high) + 1'u - check %largeUInt == newJString($largeUInt) - - - test "serializes Inf float": - check %Inf == newJString("inf") - - test "serializes -Inf float": - check %(-Inf) == newJString("-inf") - - test "deserializes NaN float": - check %NaN == newJString("nan") - - test "can construct json objects with %*": - type MyObj = object - mystring {.serialize.}: string - myint {.serialize.}: int - myoption {.serialize.}: ?bool - - let myobj = MyObj(mystring: "abc", myint: 123, myoption: some true) - let mystuint = 100000.u256 - - let json = %*{ - "myobj": myobj, - "mystuint": mystuint - } - - let expected = """{ - "myobj": { - "mystring": "abc", - "myint": 123, - "myoption": true - }, - "mystuint": "100000" - }""".flatten - - check $json == expected - - test "only serializes marked fields": - type MyObj = object - mystring {.serialize.}: string - myint {.serialize.}: int - mybool: bool - - let obj = % MyObj(mystring: "abc", myint: 1, mybool: true) - - let expected = """{ - "mystring": "abc", - "myint": 1 - }""".flatten - - check $obj == expected - - test "serializes ref objects": - type MyRef = ref object - mystring {.serialize.}: string - myint {.serialize.}: int - - let obj = % MyRef(mystring: "abc", myint: 1) - - let expected = """{ - "mystring": "abc", - "myint": 1 - }""".flatten - - check $obj == expected - - test "serializes RestPurchase": - let request = % RestPurchase( - request: some request, - requestId: RequestId.fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"), - error: some "error", - state: "state" - ) - let expected = """{ - "requestId": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1", - "request": { - "client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174", - "ask": { - "slots": 4, - "slotSize": "1073741824", - "duration": "36000", - "proofProbability": "4", - "reward": "84", - "collateral": "200", - "maxSlotLoss": 2 - }, - "content": { - "cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob" - }, - "expiry": "1691545330" - }, - "state": "state", - "error": "error" - }""".flatten - check $request == expected - - test "serializes StorageRequest": - let expected = """{ - "client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174", - "ask": { - "slots": 4, - "slotSize": "1073741824", - "duration": "36000", - "proofProbability": "4", - "reward": "84", - "collateral": "200", - "maxSlotLoss": 2 - }, - "content": { - "cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob" - }, - "expiry": "1691545330" - }""".flatten - check request.toJson == expected - - test "deserialize enum": - let json = newJString($CidVersion.CIDv1) - check !CidVersion.fromJson(json) == CIDv1 - - test "deserializes UInt256 from non-hex string representation": - let json = newJString("100000") - check !UInt256.fromJson(json) == 100000.u256 - - test "deserializes Option[T] when has a value": - let json = newJInt(1) - check (!fromJson(?int, json) == some 1) - - test "deserializes Option[T] when doesn't have a value": - let json = newJNull() - check !fromJson(?int, json) == none int - - test "deserializes float": - let json = newJFloat(1.234) - check !float.fromJson(json) == 1.234 - - test "deserializes Inf float": - let json = newJString("inf") - check !float.fromJson(json) == Inf - - test "deserializes -Inf float": - let json = newJString("-inf") - check !float.fromJson(json) == -Inf - - test "deserializes NaN float": - let json = newJString("nan") - check float.fromJson(json).get.isNaN - - test "deserializes array to sequence": - let expected = @[1, 2, 3] - let json = "[1,2,3]".parseJson - check !seq[int].fromJson(json) == expected - - test "deserializes uints int.high or smaller": - let largeUInt: uint = uint(int.high) - let json = newJInt(BiggestInt(largeUInt)) - check !uint.fromJson(json) == largeUInt - - test "deserializes large uints": - let largeUInt: uint = uint(int.high) + 1'u - let json = newJString($BiggestUInt(largeUInt)) - check !uint.fromJson(json) == largeUInt - - test "can deserialize json objects": - type MyObj = object - mystring: string - myint: int - myoption: ?bool - - let expected = MyObj(mystring: "abc", myint: 123, myoption: some true) - - let json = parseJson("""{ - "mystring": "abc", - "myint": 123, - "myoption": true - }""") - check !MyObj.fromJson(json) == expected - - test "ignores serialize pragma when deserializing": - type MyObj = object - mystring {.serialize.}: string - mybool: bool - - let expected = MyObj(mystring: "abc", mybool: true) - - let json = parseJson("""{ - "mystring": "abc", - "mybool": true - }""") - - check !MyObj.fromJson(json) == expected - - test "deserializes objects with extra fields": - type MyObj = object - mystring: string - mybool: bool - - let expected = MyObj(mystring: "abc", mybool: true) - - let json = """{ - "mystring": "abc", - "mybool": true, - "extra": "extra" - }""".parseJson - check !MyObj.fromJson(json) == expected - - test "deserializes objects with less fields": - type MyObj = object - mystring: string - mybool: bool - - let expected = MyObj(mystring: "abc", mybool: false) - - let json = """{ - "mystring": "abc" - }""".parseJson - check !MyObj.fromJson(json) == expected - - test "deserializes ref objects": - type MyRef = ref object - mystring: string - myint: int - - let expected = MyRef(mystring: "abc", myint: 1) - - let json = """{ - "mystring": "abc", - "myint": 1 - }""".parseJson - - let deserialized = !MyRef.fromJson(json) - check deserialized.mystring == expected.mystring - check deserialized.myint == expected.myint - - test "deserializes Cid": - let - jCid = newJString("zdj7Wakya26ggQWkvMdHYFcPgZ7Qh2HdMooQDDFDHkk4uHS14") - cid = "zdj7Wakya26ggQWkvMdHYFcPgZ7Qh2HdMooQDDFDHkk4uHS14" - - check: - !Cid.fromJson(jCid) == !Cid.init(cid).mapFailure - - test "deserializes StorageRequest": - check !StorageRequest.fromJson(requestJson) == request - - test "deserializes RestPurchase": - let json = """{ - "requestId": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1", - "state": "state", - "error": "error" - }""".parseJson - json["request"] = requestJson - - let expected = RestPurchase( - requestId: RequestId.fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"), - state: "state", - error: some "error", - request: some request - ) - check !RestPurchase.fromJson(json) == expected diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index a0919feb..972b4708 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -19,9 +19,9 @@ type CodexClient* = ref object proc new*(_: type CodexClient, baseurl: string): CodexClient = CodexClient(http: newHttpClient(), baseurl: baseurl) -proc info*(client: CodexClient): JsonNode = +proc info*(client: CodexClient): ?!JsonNode = let url = client.baseurl & "/debug/info" - client.http.getContent(url).parseJson() + JsonNode.parse( client.http.getContent(url) ) proc setLogLevel*(client: CodexClient, level: string) = let url = client.baseurl & "/debug/chronicles/loglevel?level=" & level @@ -52,8 +52,7 @@ proc list*(client: CodexClient): ?!seq[RestContent] = if response.status != "200 OK": return failure(response.status) - let json = ? parseJson(response.body).catch - seq[RestContent].fromJson(json) + seq[RestContent].fromJson(response.body) proc space*(client: CodexClient): ?!RestRepoStore = let url = client.baseurl & "/space" @@ -62,8 +61,7 @@ proc space*(client: CodexClient): ?!RestRepoStore = if response.status != "200 OK": return failure(response.status) - let json = ? parseJson(response.body).catch - RestRepoStore.fromJson(json) + RestRepoStore.fromJson(response.body) proc requestStorageRaw*( client: CodexClient, @@ -116,8 +114,7 @@ proc getPurchase*(client: CodexClient, purchaseId: PurchaseId): ?!RestPurchase = let url = client.baseurl & "/storage/purchases/" & purchaseId.toHex try: let body = client.http.getContent(url) - let json = ? parseJson(body).catch - return RestPurchase.fromJson(json) + return RestPurchase.fromJson(body) except CatchableError as e: return failure e.msg @@ -125,16 +122,14 @@ proc getSalesAgent*(client: CodexClient, slotId: SlotId): ?!RestSalesAgent = let url = client.baseurl & "/sales/slots/" & slotId.toHex try: let body = client.http.getContent(url) - let json = ? parseJson(body).catch - return RestSalesAgent.fromJson(json) + return RestSalesAgent.fromJson(body) except CatchableError as e: return failure e.msg proc getSlots*(client: CodexClient): ?!seq[Slot] = let url = client.baseurl & "/sales/slots" let body = client.http.getContent(url) - let json = ? parseJson(body).catch - seq[Slot].fromJson(json) + seq[Slot].fromJson(body) proc postAvailability*( client: CodexClient, @@ -151,13 +146,13 @@ proc postAvailability*( } let response = client.http.post(url, $json) doAssert response.status == "200 OK", "expected 200 OK, got " & response.status & ", body: " & response.body - Availability.fromJson(response.body.parseJson) + Availability.fromJson(response.body) proc getAvailabilities*(client: CodexClient): ?!seq[Availability] = ## Call sales availability REST endpoint let url = client.baseurl & "/sales/availability" let body = client.http.getContent(url) - seq[Availability].fromJson(parseJson(body)) + seq[Availability].fromJson(body) proc close*(client: CodexClient) = client.http.close() diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index b692f5be..43ca2b8a 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -288,7 +288,11 @@ template multinodesuite*(name: string, body: untyped) = node: node ) if clients().len == 1: - bootstrap = CodexProcess(node).client.info()["spr"].getStr() + without ninfo =? CodexProcess(node).client.info(): + # raise CatchableError instead of Defect (with .get or !) so we + # can gracefully shutdown and prevent zombies + raiseMultiNodeSuiteError "Failed to get node info" + bootstrap = ninfo["spr"].getStr() if var providers =? nodeConfigs.providers: failAndTeardownOnError "failed to start provider nodes": diff --git a/tests/integration/testIntegration.nim b/tests/integration/testIntegration.nim index 88f27ffd..4d2bc15a 100644 --- a/tests/integration/testIntegration.nim +++ b/tests/integration/testIntegration.nim @@ -26,7 +26,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false: await ethProvider.advanceTime(1.u256) test "nodes can print their peer information": - check client1.info() != client2.info() + check !client1.info() != !client2.info() test "nodes can set chronicles log level": client1.setLogLevel("DEBUG;TRACE:codex") diff --git a/tests/integration/twonodes.nim b/tests/integration/twonodes.nim index b8024079..5e1825ef 100644 --- a/tests/integration/twonodes.nim +++ b/tests/integration/twonodes.nim @@ -52,7 +52,7 @@ template twonodessuite*(name: string, debug1, debug2: string, body) = node1 = startNode(node1Args, debug = debug1) node1.waitUntilStarted() - let bootstrap = client1.info()["spr"].getStr() + let bootstrap = (!client1.info()["spr"]).getStr() var node2Args = @[ "--api-port=8081", diff --git a/tests/testTaiko.nim b/tests/testTaiko.nim index 49ad3226..a799697b 100644 --- a/tests/testTaiko.nim +++ b/tests/testTaiko.nim @@ -29,7 +29,7 @@ suite "Taiko L2 Integration Tests": ]) node1.waitUntilStarted() - let bootstrap = node1.client.info()["spr"].getStr() + let bootstrap = (!node1.client.info())["spr"].getStr() node2 = startNode([ "--data-dir=" & createTempDir("", ""), diff --git a/vendor/nim-serde b/vendor/nim-serde new file mode 160000 index 00000000..a261c3d2 --- /dev/null +++ b/vendor/nim-serde @@ -0,0 +1 @@ +Subproject commit a261c3d214b87d9de0338d6cfd5d134da03d2dc3