From 8f65f578db700ccff1a7e6d61a52eef5a7ef731f Mon Sep 17 00:00:00 2001 From: Ben Bierens <39762930+benbierens@users.noreply.github.com> Date: Mon, 18 Mar 2024 14:41:25 +0100 Subject: [PATCH 1/4] Pulls in latest contracts. (#742) --- vendor/codex-contracts-eth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/codex-contracts-eth b/vendor/codex-contracts-eth index 118ee0b2..a58427e4 160000 --- a/vendor/codex-contracts-eth +++ b/vendor/codex-contracts-eth @@ -1 +1 @@ -Subproject commit 118ee0b22b2d12c8fbf3376cf201d203d0a7cf97 +Subproject commit a58427e496088b904aed070e92f1c479c45fd852 From b4de53f43667feb6fb8e021ffc1a3cf180a36ba2 Mon Sep 17 00:00:00 2001 From: Giuliano Mega Date: Mon, 18 Mar 2024 19:41:22 -0300 Subject: [PATCH 2/4] update path for libp2p (#743) --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index ad4e4792..1089a309 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,7 +5,7 @@ branch = master [submodule "vendor/nim-libp2p"] path = vendor/nim-libp2p - url = https://github.com/status-im/nim-libp2p.git + url = https://github.com/vacp2p/nim-libp2p.git ignore = untracked branch = master [submodule "vendor/nimcrypto"] From f567f4ec15dbdf0f208aea5370bf3ba6bd6c14a4 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Tue, 19 Mar 2024 14:25:13 +1100 Subject: [PATCH 3/4] refactor: use serde for json de/serialization instead of utils/json (#704) * json > nim-serde bump Should wait until serde is integrated into nim-ethers before making these changes as there will be less import exceptions required. * bump nim-serde * change func to proc due to chronicles side effects * import serde into utils/json, use as proxy import nim-serde into utils/json and use utils/json as a proxy for serde functions, including overloading `%` and `fromJson` for application types. * update tests to use serde * bump serde to latest * remove testjson -- no longer needed * bump serde in nimble * updates to reconcile rebase with master --- .gitmodules | 3 + codex.nimble | 1 + codex/logutils.nim | 15 +- codex/rest/json.nim | 4 +- codex/slots/builder/builder.nim | 2 +- codex/slots/sampler/sampler.nim | 2 +- codex/utils/json.nim | 319 +--------------- tests/codex/sales/testslotqueue.nim | 1 - .../codex/slots/backends/testcircomcompat.nim | 4 +- tests/codex/slots/sampler/testsampler.nim | 2 +- tests/codex/slots/sampler/testutils.nim | 4 +- tests/codex/testlogutils.nim | 5 +- tests/codex/testutils.nim | 1 - tests/codex/utils/testjson.nim | 345 ------------------ tests/integration/codexclient.nim | 23 +- tests/integration/multinodes.nim | 6 +- tests/integration/testIntegration.nim | 2 +- tests/integration/twonodes.nim | 2 +- tests/testTaiko.nim | 2 +- vendor/nim-serde | 1 + 20 files changed, 43 insertions(+), 701 deletions(-) delete mode 100644 tests/codex/utils/testjson.nim create mode 160000 vendor/nim-serde diff --git a/.gitmodules b/.gitmodules index 1089a309..c5162818 100644 --- a/.gitmodules +++ b/.gitmodules @@ -209,3 +209,6 @@ url = https://github.com/codex-storage/codex-storage-proofs-circuits.git ignore = untracked branch = master +[submodule "vendor/nim-serde"] + path = vendor/nim-serde + url = https://github.com/codex-storage/nim-serde.git diff --git a/codex.nimble b/codex.nimble index 42347710..7a99779f 100644 --- a/codex.nimble +++ b/codex.nimble @@ -22,6 +22,7 @@ requires "presto" requires "protobuf_serialization >= 0.2.0 & < 0.3.0" requires "questionable >= 0.10.13 & < 0.11.0" requires "secp256k1" +requires "serde >= 1.0.0 & < 2.0.0" requires "stew" requires "upraises >= 0.1.0 & < 0.2.0" requires "toml_serialization" diff --git a/codex/logutils.nim b/codex/logutils.nim index 6abc21a5..eb75e906 100644 --- a/codex/logutils.nim +++ b/codex/logutils.nim @@ -17,16 +17,16 @@ ## module, and specifying `formatIt`. If textlines log output and json log output ## need to be different, overload `formatIt` and specify a `LogFormat`. If json ## serialization is needed, it can be declared with a `%` proc. `logutils` -## imports and exports `utils/json` which handles the de/serialization, examples +## imports and exports `nim-serde` which handles the de/serialization, examples ## below. **Only `codex/logutils` needs to be imported.** ## ## Using `logutils` in the Codex codebase: ## - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils` ## - most of `chronicles` is exported by `logutils` -## - Instead of importing `std/json`, import `pkg/codex/utils/json` -## - `std/json` is exported by `utils/json` which is exported by `logutils` +## - Instead of importing `std/json`, import `pkg/serde/json` +## - `std/json` is exported by `serde` which is exported by `logutils` ## - Instead of importing `pkg/nim-json-serialization`, import -## `pkg/codex/utils/json` +## `pkg/serde/json` or use codex-specific overloads by importing `utils/json` ## - one of the goals is to remove the use of `nim-json-serialization` ## ## ```nim @@ -54,7 +54,7 @@ ## # chronicles json output ## {"lvl":"TRC","msg":"test","tid":14397405,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}} ## ``` -## In this case, `BlockAddress` is just an object, so `utils/json` can handle +## In this case, `BlockAddress` is just an object, so `nim-serde` can handle ## serializing it without issue (only fields annotated with `{.serialize.}` will ## serialize (aka opt-in serialization)). ## @@ -95,20 +95,19 @@ import pkg/chronicles except toJson, `%` from pkg/libp2p import Cid, MultiAddress, `$` import pkg/questionable import pkg/questionable/results +import ./utils/json except formatIt # TODO: remove exception? import pkg/stew/byteutils import pkg/stint import pkg/upraises -import ./utils/json - export byteutils export chronicles except toJson, formatIt, `%` export questionable export sequtils +export json except formatIt export strutils export sugar export upraises -export json export results func shortLog*(long: string, ellipses = "*", start = 3, stop = 6): string = diff --git a/codex/rest/json.nim b/codex/rest/json.nim index 181ea584..748a8b5b 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -104,11 +104,11 @@ proc init*(_: type RestNodeId, id: NodeId): RestNodeId = id: id ) -func `%`*(obj: StorageRequest | Slot): JsonNode = +proc `%`*(obj: StorageRequest | Slot): JsonNode = let jsonObj = newJObject() for k, v in obj.fieldPairs: jsonObj[k] = %v jsonObj["id"] = %(obj.id) return jsonObj -func `%`*(obj: RestNodeId): JsonNode = % $obj.id +proc `%`*(obj: RestNodeId): JsonNode = % $obj.id diff --git a/codex/slots/builder/builder.nim b/codex/slots/builder/builder.nim index 674fd28e..f999a514 100644 --- a/codex/slots/builder/builder.nim +++ b/codex/slots/builder/builder.nim @@ -15,11 +15,11 @@ import std/sugar import pkg/libp2p import pkg/chronos -import pkg/chronicles import pkg/questionable import pkg/questionable/results import pkg/constantine/math/io/io_fields +import ../../logutils import ../../utils import ../../stores import ../../manifest diff --git a/codex/slots/sampler/sampler.nim b/codex/slots/sampler/sampler.nim index aef3545d..d22121c2 100644 --- a/codex/slots/sampler/sampler.nim +++ b/codex/slots/sampler/sampler.nim @@ -9,12 +9,12 @@ import std/sugar -import pkg/chronicles import pkg/chronos import pkg/questionable import pkg/questionable/results import pkg/stew/arrayops +import ../../logutils import ../../market import ../../blocktype as bt import ../../merkletree diff --git a/codex/utils/json.nim b/codex/utils/json.nim index 080d4e89..4113b632 100644 --- a/codex/utils/json.nim +++ b/codex/utils/json.nim @@ -1,181 +1,16 @@ -import std/json except `%`, `%*` -import std/macros import std/options -import std/strutils -import std/strformat -import std/tables import std/typetraits from pkg/ethers import Address from pkg/libp2p import Cid, PeerId, SignedPeerRecord, MultiAddress, AddressInfo, init, `$` import pkg/contractabi import pkg/codexdht/discv5/node as dn -import pkg/stew/byteutils -import pkg/stint +import pkg/serde/json import pkg/questionable/results import ../errors -export json except `%`, `%*` +export json -type - SerializationError = object of CodexError - UnexpectedKindError = object of SerializationError - -template serialize* {.pragma.} - -proc newUnexpectedKindError( - expectedType: type, - expectedKinds: string, - json: JsonNode -): ref UnexpectedKindError = - let kind = if json.isNil: "nil" - else: $json.kind - newException(UnexpectedKindError, - &"deserialization to {$expectedType} failed: expected {expectedKinds} " & - &"but got {kind}") - -proc newUnexpectedKindError( - expectedType: type, - expectedKinds: set[JsonNodeKind], - json: JsonNode -): ref UnexpectedKindError = - newUnexpectedKindError(expectedType, $expectedKinds, json) - -proc newUnexpectedKindError( - expectedType: type, - expectedKind: JsonNodeKind, - json: JsonNode -): ref UnexpectedKindError = - newUnexpectedKindError(expectedType, {expectedKind}, json) - -template expectJsonKind( - expectedType: type, - expectedKinds: set[JsonNodeKind], - json: JsonNode -) = - if json.isNil or json.kind notin expectedKinds: - return failure(newUnexpectedKindError(expectedType, expectedKinds, json)) - -template expectJsonKind( - expectedType: type, - expectedKind: JsonNodeKind, - json: JsonNode -) = - expectJsonKind(expectedType, {expectedKind}, json) - -proc fromJson*( - T: type enum, - json: JsonNode -): ?!T = - expectJsonKind(string, JString, json) - catch parseEnum[T](json.str) - -proc fromJson*( - _: type string, - json: JsonNode -): ?!string = - if json.isNil: - let err = newException(ValueError, "'json' expected, but was nil") - return failure(err) - elif json.kind == JNull: - return success("null") - elif json.isNil or json.kind != JString: - return failure(newUnexpectedKindError(string, JString, json)) - catch json.getStr - -proc fromJson*( - _: type bool, - json: JsonNode -): ?!bool = - expectJsonKind(bool, JBool, json) - catch json.getBool - -proc fromJson*( - _: type int, - json: JsonNode -): ?!int = - expectJsonKind(int, JInt, json) - catch json.getInt - -proc fromJson*[T: SomeInteger]( - _: type T, - json: JsonNode -): ?!T = - when T is uint|uint64 or (not defined(js) and int.sizeof == 4): - expectJsonKind(T, {JInt, JString}, json) - case json.kind - of JString: - let x = parseBiggestUInt(json.str) - return success cast[T](x) - else: - return success T(json.num) - else: - expectJsonKind(T, {JInt}, json) - return success cast[T](json.num) - -proc fromJson*[T: SomeFloat]( - _: type T, - json: JsonNode -): ?!T = - expectJsonKind(T, {JInt, JFloat, JString}, json) - if json.kind == JString: - case json.str - of "nan": - let b = NaN - return success T(b) - # dst = NaN # would fail some tests because range conversions would cause CT error - # in some cases; but this is not a hot-spot inside this branch and backend can optimize this. - of "inf": - let b = Inf - return success T(b) - of "-inf": - let b = -Inf - return success T(b) - else: - let err = newUnexpectedKindError(T, "'nan|inf|-inf'", json) - return failure(err) - else: - if json.kind == JFloat: - return success T(json.fnum) - else: - return success T(json.num) - -proc fromJson*( - _: type seq[byte], - json: JsonNode -): ?!seq[byte] = - expectJsonKind(seq[byte], JString, json) - hexToSeqByte(json.getStr).catch - -proc fromJson*[N: static[int], T: array[N, byte]]( - _: type T, - json: JsonNode -): ?!T = - expectJsonKind(T, JString, json) - T.fromHex(json.getStr).catch - -proc fromJson*[T: distinct]( - _: type T, - json: JsonNode -): ?!T = - success T(? T.distinctBase.fromJson(json)) - -proc fromJson*[N: static[int], T: StUint[N]]( - _: type T, - json: JsonNode -): ?!T = - expectJsonKind(T, JString, json) - catch parse(json.getStr, T) - -proc fromJson*[T]( - _: type Option[T], - json: JsonNode -): ?! Option[T] = - if json.isNil or json.kind == JNull: - return success(none T) - without val =? T.fromJson(json), error: - return failure(error) - success(val.some) proc fromJson*( _: type Cid, @@ -184,123 +19,6 @@ proc fromJson*( expectJsonKind(Cid, JString, json) Cid.init(json.str).mapFailure -proc fromJson*[T]( - _: type seq[T], - json: JsonNode -): ?! seq[T] = - expectJsonKind(seq[T], JArray, json) - var arr: seq[T] = @[] - for elem in json.elems: - arr.add(? T.fromJson(elem)) - success arr - -proc fromJson*[T: ref object or object]( - _: type T, - json: JsonNode -): ?!T = - expectJsonKind(T, JObject, json) - var res = when type(T) is ref: T.new() else: T.default - - # Leave this in, it's good for debugging: - # trace "deserializing object", to = $T, json - for name, value in fieldPairs(when type(T) is ref: res[] else: res): - if json{name} != nil: - without parsed =? type(value).fromJson(json{name}), e: - error "error deserializing field", - field = $T & "." & name, - json = json{name}, - error = e.msg - return failure(e) - value = parsed - success(res) - -proc fromJson*[T: object]( - _: type T, - bytes: seq[byte] -): ?!T = - let json = ?catch parseJson(string.fromBytes(bytes)) - T.fromJson(json) - -proc fromJson*[T: ref object]( - _: type T, - bytes: seq[byte] -): ?!T = - let json = ?catch parseJson(string.fromBytes(bytes)) - T.fromJson(json) - -func `%`*(s: string): JsonNode = newJString(s) - -func `%`*(n: uint): JsonNode = - if n > cast[uint](int.high): - newJString($n) - else: - newJInt(BiggestInt(n)) - -func `%`*(n: int): JsonNode = newJInt(n) - -func `%`*(n: BiggestUInt): JsonNode = - if n > cast[BiggestUInt](BiggestInt.high): - newJString($n) - else: - newJInt(BiggestInt(n)) - -func `%`*(n: BiggestInt): JsonNode = newJInt(n) - -func `%`*(n: float): JsonNode = - if n != n: newJString("nan") - elif n == Inf: newJString("inf") - elif n == -Inf: newJString("-inf") - else: newJFloat(n) - -func `%`*(b: bool): JsonNode = newJBool(b) - -func `%`*(keyVals: openArray[tuple[key: string, val: JsonNode]]): JsonNode = - if keyVals.len == 0: return newJArray() - let jObj = newJObject() - for key, val in items(keyVals): jObj.fields[key] = val - jObj - -template `%`*(j: JsonNode): JsonNode = j - -func `%`*[T](table: Table[string, T]|OrderedTable[string, T]): JsonNode = - let jObj = newJObject() - for k, v in table: jObj[k] = ? %v - jObj - -func `%`*[T](opt: Option[T]): JsonNode = - if opt.isSome: %(opt.get) else: newJNull() - -func `%`*[T: object](obj: T): JsonNode = - let jsonObj = newJObject() - for name, value in obj.fieldPairs: - when value.hasCustomPragma(serialize): - jsonObj[name] = %value - jsonObj - -func `%`*[T: ref object](obj: T): JsonNode = - let jsonObj = newJObject() - for name, value in obj[].fieldPairs: - when value.hasCustomPragma(serialize): - jsonObj[name] = %(value) - jsonObj - -proc `%`*(o: enum): JsonNode = % $o - -func `%`*(stint: StInt|StUint): JsonNode = %stint.toString - -func `%`*(cstr: cstring): JsonNode = % $cstr - -func `%`*(arr: openArray[byte]): JsonNode = % arr.to0xHex - -func `%`*[T](elements: openArray[T]): JsonNode = - let jObj = newJArray() - for elem in elements: jObj.add(%elem) - jObj - -func `%`*[T: distinct](id: T): JsonNode = - type baseType = T.distinctBase - % baseType(id) - func `%`*(cid: Cid): JsonNode = % $cid func `%`*(obj: PeerId): JsonNode = % $obj @@ -314,36 +32,3 @@ func `%`*(obj: AddressInfo): JsonNode = % $obj.address func `%`*(obj: MultiAddress): JsonNode = % $obj func `%`*(address: ethers.Address): JsonNode = % $address - -func toJson*[T](item: T): string = $(%item) - -proc toJsnImpl(x: NimNode): NimNode = - case x.kind - of nnkBracket: # array - if x.len == 0: return newCall(bindSym"newJArray") - result = newNimNode(nnkBracket) - for i in 0 ..< x.len: - result.add(toJsnImpl(x[i])) - result = newCall(bindSym("%", brOpen), result) - of nnkTableConstr: # object - if x.len == 0: return newCall(bindSym"newJObject") - result = newNimNode(nnkTableConstr) - for i in 0 ..< x.len: - x[i].expectKind nnkExprColonExpr - result.add newTree(nnkExprColonExpr, x[i][0], toJsnImpl(x[i][1])) - result = newCall(bindSym("%", brOpen), result) - of nnkCurly: # empty object - x.expectLen(0) - result = newCall(bindSym"newJObject") - of nnkNilLit: - result = newCall(bindSym"newJNull") - of nnkPar: - if x.len == 1: result = toJsnImpl(x[0]) - else: result = newCall(bindSym("%", brOpen), x) - else: - result = newCall(bindSym("%", brOpen), x) - -macro `%*`*(x: untyped): JsonNode = - ## Convert an expression to a JsonNode directly, without having to specify - ## `%` for every element. - result = toJsnImpl(x) diff --git a/tests/codex/sales/testslotqueue.nim b/tests/codex/sales/testslotqueue.nim index 2e30fb3a..ec4760f1 100644 --- a/tests/codex/sales/testslotqueue.nim +++ b/tests/codex/sales/testslotqueue.nim @@ -1,5 +1,4 @@ import std/sequtils -import pkg/chronicles import pkg/chronos import pkg/datastore import pkg/questionable diff --git a/tests/codex/slots/backends/testcircomcompat.nim b/tests/codex/slots/backends/testcircomcompat.nim index bc6240d3..08ac2d21 100644 --- a/tests/codex/slots/backends/testcircomcompat.nim +++ b/tests/codex/slots/backends/testcircomcompat.nim @@ -8,11 +8,11 @@ import ../../../asynctest import pkg/chronos import pkg/poseidon2 import pkg/datastore +import pkg/serde/json import pkg/codex/slots {.all.} import pkg/codex/slots/types {.all.} import pkg/codex/merkletree -import pkg/codex/utils/json import pkg/codex/codextypes import pkg/codex/manifest import pkg/codex/stores @@ -33,7 +33,7 @@ suite "Test Circom Compat Backend - control inputs": setup: let inputData = readFile("tests/circuits/fixtures/input.json") - inputJson = parseJson(inputData) + inputJson = !JsonNode.parse(inputData) proofInputs = Poseidon2Hash.jsonToProofInput(inputJson) circom = CircomCompat.init(r1cs, wasm, zkey) diff --git a/tests/codex/slots/sampler/testsampler.nim b/tests/codex/slots/sampler/testsampler.nim index ef04656f..2ed32011 100644 --- a/tests/codex/slots/sampler/testsampler.nim +++ b/tests/codex/slots/sampler/testsampler.nim @@ -35,7 +35,7 @@ suite "Test Sampler - control samples": setup: inputData = readFile("tests/circuits/fixtures/input.json") - inputJson = parseJson(inputData) + inputJson = !JsonNode.parse(inputData) proofInput = Poseidon2Hash.jsonToProofInput(inputJson) test "Should verify control samples": diff --git a/tests/codex/slots/sampler/testutils.nim b/tests/codex/slots/sampler/testutils.nim index 1e1bef96..ff277647 100644 --- a/tests/codex/slots/sampler/testutils.nim +++ b/tests/codex/slots/sampler/testutils.nim @@ -18,9 +18,9 @@ import pkg/codex/contracts/requests import pkg/codex/contracts import pkg/codex/merkletree import pkg/codex/stores/cachestore - import pkg/codex/slots/types import pkg/codex/slots/sampler/utils +import pkg/codex/utils/json import ../backends/helpers import ../../helpers @@ -38,7 +38,7 @@ asyncchecksuite "Test proof sampler utils": setup: inputData = readFile("tests/circuits/fixtures/input.json") - inputJson = parseJson(inputData) + inputJson = !JsonNode.parse(inputData) proofInput = Poseidon2Hash.jsonToProofInput(inputJson) test "Extract low bits": diff --git a/tests/codex/testlogutils.nim b/tests/codex/testlogutils.nim index 9f8e42ac..44e7533f 100644 --- a/tests/codex/testlogutils.nim +++ b/tests/codex/testlogutils.nim @@ -7,6 +7,7 @@ import pkg/codex/contracts/requests import pkg/codex/logutils import pkg/codex/purchasing/purchaseid import pkg/codex/units +import pkg/codex/utils/json import pkg/libp2p/cid import pkg/libp2p/multiaddress import pkg/questionable @@ -61,8 +62,8 @@ checksuite "Test logging output": outputLines.contains(toFind) template loggedJson(prop, expected): auto = - let json = $ parseJson(outputJson){prop} - json == expected + let jsonVal = !JsonNode.parse(outputJson) + $ jsonVal{prop} == expected template log(val) = testlines.trace "test", val diff --git a/tests/codex/testutils.nim b/tests/codex/testutils.nim index 937a6e86..6b4b2366 100644 --- a/tests/codex/testutils.nim +++ b/tests/codex/testutils.nim @@ -1,4 +1,3 @@ -import ./utils/testjson import ./utils/testoptionalcast import ./utils/testkeyutils import ./utils/testasyncstatemachine diff --git a/tests/codex/utils/testjson.nim b/tests/codex/utils/testjson.nim deleted file mode 100644 index 5ba7fff3..00000000 --- a/tests/codex/utils/testjson.nim +++ /dev/null @@ -1,345 +0,0 @@ -import std/math -import std/options -import std/strformat -import std/strutils -import std/unittest -import pkg/stew/byteutils -import pkg/stint -import pkg/codex/contracts/requests -from pkg/codex/rest/json import RestPurchase -import pkg/codex/logutils -import pkg/codex/utils/json as utilsjson -import pkg/questionable -import pkg/questionable/results -import pkg/libp2p -import ../helpers - -checksuite "json serialization": - var request: StorageRequest - var requestJson: JsonNode - - func flatten(s: string): string = - s.replace(" ") - .replace("\n") - - setup: - request = StorageRequest( - client: Address.init("0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174").get(), - ask: StorageAsk( - slots: 4, - slotSize: (1 * 1024 * 1024 * 1024).u256, # 1 Gigabyte - duration: (10 * 60 * 60).u256, # 10 hours - collateral: 200.u256, - proofProbability: 4.u256, # require a proof roughly once every 4 periods - reward: 84.u256, - maxSlotLoss: 2 # 2 slots can be freed without data considered to be lost - ), - content: StorageContent( - cid: "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob", - merkleRoot: array[32, byte].fromHex("0xc066dd7e405de5a795ce1e765209cfaa6de6c74829c607d1a2fe53107107a298") - ), - expiry: 1691545330.u256, - nonce: Nonce array[32, byte].fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1") - ) - requestJson = """{ - "client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174", - "ask": { - "slots": 4, - "slotSize": "1073741824", - "duration": "36000", - "proofProbability": "4", - "reward": "84", - "collateral": "200", - "maxSlotLoss": 2 - }, - "content": { - "cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob", - "merkleRoot": "0xc066dd7e405de5a795ce1e765209cfaa6de6c74829c607d1a2fe53107107a298" - }, - "expiry": "1691545330", - "nonce": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1" - }""".parseJson - - test "serializes UInt256 to non-hex string representation": - check (% 100000.u256) == newJString("100000") - - test "serializes sequence to an array": - let json = % @[1, 2, 3] - let expected = "[1,2,3]" - check $json == expected - - test "serializes Option[T] when has a value": - let obj = %(some 1) - let expected = "1" - check $obj == expected - - test "serializes Option[T] when doesn't have a value": - let obj = %(none int) - let expected = "null" - check $obj == expected - - test "serializes uints int.high or smaller": - let largeUInt: uint = uint(int.high) - check %largeUInt == newJInt(BiggestInt(largeUInt)) - - test "serializes large uints": - let largeUInt: uint = uint(int.high) + 1'u - check %largeUInt == newJString($largeUInt) - - - test "serializes Inf float": - check %Inf == newJString("inf") - - test "serializes -Inf float": - check %(-Inf) == newJString("-inf") - - test "deserializes NaN float": - check %NaN == newJString("nan") - - test "can construct json objects with %*": - type MyObj = object - mystring {.serialize.}: string - myint {.serialize.}: int - myoption {.serialize.}: ?bool - - let myobj = MyObj(mystring: "abc", myint: 123, myoption: some true) - let mystuint = 100000.u256 - - let json = %*{ - "myobj": myobj, - "mystuint": mystuint - } - - let expected = """{ - "myobj": { - "mystring": "abc", - "myint": 123, - "myoption": true - }, - "mystuint": "100000" - }""".flatten - - check $json == expected - - test "only serializes marked fields": - type MyObj = object - mystring {.serialize.}: string - myint {.serialize.}: int - mybool: bool - - let obj = % MyObj(mystring: "abc", myint: 1, mybool: true) - - let expected = """{ - "mystring": "abc", - "myint": 1 - }""".flatten - - check $obj == expected - - test "serializes ref objects": - type MyRef = ref object - mystring {.serialize.}: string - myint {.serialize.}: int - - let obj = % MyRef(mystring: "abc", myint: 1) - - let expected = """{ - "mystring": "abc", - "myint": 1 - }""".flatten - - check $obj == expected - - test "serializes RestPurchase": - let request = % RestPurchase( - request: some request, - requestId: RequestId.fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"), - error: some "error", - state: "state" - ) - let expected = """{ - "requestId": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1", - "request": { - "client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174", - "ask": { - "slots": 4, - "slotSize": "1073741824", - "duration": "36000", - "proofProbability": "4", - "reward": "84", - "collateral": "200", - "maxSlotLoss": 2 - }, - "content": { - "cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob" - }, - "expiry": "1691545330" - }, - "state": "state", - "error": "error" - }""".flatten - check $request == expected - - test "serializes StorageRequest": - let expected = """{ - "client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174", - "ask": { - "slots": 4, - "slotSize": "1073741824", - "duration": "36000", - "proofProbability": "4", - "reward": "84", - "collateral": "200", - "maxSlotLoss": 2 - }, - "content": { - "cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob" - }, - "expiry": "1691545330" - }""".flatten - check request.toJson == expected - - test "deserialize enum": - let json = newJString($CidVersion.CIDv1) - check !CidVersion.fromJson(json) == CIDv1 - - test "deserializes UInt256 from non-hex string representation": - let json = newJString("100000") - check !UInt256.fromJson(json) == 100000.u256 - - test "deserializes Option[T] when has a value": - let json = newJInt(1) - check (!fromJson(?int, json) == some 1) - - test "deserializes Option[T] when doesn't have a value": - let json = newJNull() - check !fromJson(?int, json) == none int - - test "deserializes float": - let json = newJFloat(1.234) - check !float.fromJson(json) == 1.234 - - test "deserializes Inf float": - let json = newJString("inf") - check !float.fromJson(json) == Inf - - test "deserializes -Inf float": - let json = newJString("-inf") - check !float.fromJson(json) == -Inf - - test "deserializes NaN float": - let json = newJString("nan") - check float.fromJson(json).get.isNaN - - test "deserializes array to sequence": - let expected = @[1, 2, 3] - let json = "[1,2,3]".parseJson - check !seq[int].fromJson(json) == expected - - test "deserializes uints int.high or smaller": - let largeUInt: uint = uint(int.high) - let json = newJInt(BiggestInt(largeUInt)) - check !uint.fromJson(json) == largeUInt - - test "deserializes large uints": - let largeUInt: uint = uint(int.high) + 1'u - let json = newJString($BiggestUInt(largeUInt)) - check !uint.fromJson(json) == largeUInt - - test "can deserialize json objects": - type MyObj = object - mystring: string - myint: int - myoption: ?bool - - let expected = MyObj(mystring: "abc", myint: 123, myoption: some true) - - let json = parseJson("""{ - "mystring": "abc", - "myint": 123, - "myoption": true - }""") - check !MyObj.fromJson(json) == expected - - test "ignores serialize pragma when deserializing": - type MyObj = object - mystring {.serialize.}: string - mybool: bool - - let expected = MyObj(mystring: "abc", mybool: true) - - let json = parseJson("""{ - "mystring": "abc", - "mybool": true - }""") - - check !MyObj.fromJson(json) == expected - - test "deserializes objects with extra fields": - type MyObj = object - mystring: string - mybool: bool - - let expected = MyObj(mystring: "abc", mybool: true) - - let json = """{ - "mystring": "abc", - "mybool": true, - "extra": "extra" - }""".parseJson - check !MyObj.fromJson(json) == expected - - test "deserializes objects with less fields": - type MyObj = object - mystring: string - mybool: bool - - let expected = MyObj(mystring: "abc", mybool: false) - - let json = """{ - "mystring": "abc" - }""".parseJson - check !MyObj.fromJson(json) == expected - - test "deserializes ref objects": - type MyRef = ref object - mystring: string - myint: int - - let expected = MyRef(mystring: "abc", myint: 1) - - let json = """{ - "mystring": "abc", - "myint": 1 - }""".parseJson - - let deserialized = !MyRef.fromJson(json) - check deserialized.mystring == expected.mystring - check deserialized.myint == expected.myint - - test "deserializes Cid": - let - jCid = newJString("zdj7Wakya26ggQWkvMdHYFcPgZ7Qh2HdMooQDDFDHkk4uHS14") - cid = "zdj7Wakya26ggQWkvMdHYFcPgZ7Qh2HdMooQDDFDHkk4uHS14" - - check: - !Cid.fromJson(jCid) == !Cid.init(cid).mapFailure - - test "deserializes StorageRequest": - check !StorageRequest.fromJson(requestJson) == request - - test "deserializes RestPurchase": - let json = """{ - "requestId": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1", - "state": "state", - "error": "error" - }""".parseJson - json["request"] = requestJson - - let expected = RestPurchase( - requestId: RequestId.fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"), - state: "state", - error: some "error", - request: some request - ) - check !RestPurchase.fromJson(json) == expected diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index a0919feb..972b4708 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -19,9 +19,9 @@ type CodexClient* = ref object proc new*(_: type CodexClient, baseurl: string): CodexClient = CodexClient(http: newHttpClient(), baseurl: baseurl) -proc info*(client: CodexClient): JsonNode = +proc info*(client: CodexClient): ?!JsonNode = let url = client.baseurl & "/debug/info" - client.http.getContent(url).parseJson() + JsonNode.parse( client.http.getContent(url) ) proc setLogLevel*(client: CodexClient, level: string) = let url = client.baseurl & "/debug/chronicles/loglevel?level=" & level @@ -52,8 +52,7 @@ proc list*(client: CodexClient): ?!seq[RestContent] = if response.status != "200 OK": return failure(response.status) - let json = ? parseJson(response.body).catch - seq[RestContent].fromJson(json) + seq[RestContent].fromJson(response.body) proc space*(client: CodexClient): ?!RestRepoStore = let url = client.baseurl & "/space" @@ -62,8 +61,7 @@ proc space*(client: CodexClient): ?!RestRepoStore = if response.status != "200 OK": return failure(response.status) - let json = ? parseJson(response.body).catch - RestRepoStore.fromJson(json) + RestRepoStore.fromJson(response.body) proc requestStorageRaw*( client: CodexClient, @@ -116,8 +114,7 @@ proc getPurchase*(client: CodexClient, purchaseId: PurchaseId): ?!RestPurchase = let url = client.baseurl & "/storage/purchases/" & purchaseId.toHex try: let body = client.http.getContent(url) - let json = ? parseJson(body).catch - return RestPurchase.fromJson(json) + return RestPurchase.fromJson(body) except CatchableError as e: return failure e.msg @@ -125,16 +122,14 @@ proc getSalesAgent*(client: CodexClient, slotId: SlotId): ?!RestSalesAgent = let url = client.baseurl & "/sales/slots/" & slotId.toHex try: let body = client.http.getContent(url) - let json = ? parseJson(body).catch - return RestSalesAgent.fromJson(json) + return RestSalesAgent.fromJson(body) except CatchableError as e: return failure e.msg proc getSlots*(client: CodexClient): ?!seq[Slot] = let url = client.baseurl & "/sales/slots" let body = client.http.getContent(url) - let json = ? parseJson(body).catch - seq[Slot].fromJson(json) + seq[Slot].fromJson(body) proc postAvailability*( client: CodexClient, @@ -151,13 +146,13 @@ proc postAvailability*( } let response = client.http.post(url, $json) doAssert response.status == "200 OK", "expected 200 OK, got " & response.status & ", body: " & response.body - Availability.fromJson(response.body.parseJson) + Availability.fromJson(response.body) proc getAvailabilities*(client: CodexClient): ?!seq[Availability] = ## Call sales availability REST endpoint let url = client.baseurl & "/sales/availability" let body = client.http.getContent(url) - seq[Availability].fromJson(parseJson(body)) + seq[Availability].fromJson(body) proc close*(client: CodexClient) = client.http.close() diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index b692f5be..43ca2b8a 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -288,7 +288,11 @@ template multinodesuite*(name: string, body: untyped) = node: node ) if clients().len == 1: - bootstrap = CodexProcess(node).client.info()["spr"].getStr() + without ninfo =? CodexProcess(node).client.info(): + # raise CatchableError instead of Defect (with .get or !) so we + # can gracefully shutdown and prevent zombies + raiseMultiNodeSuiteError "Failed to get node info" + bootstrap = ninfo["spr"].getStr() if var providers =? nodeConfigs.providers: failAndTeardownOnError "failed to start provider nodes": diff --git a/tests/integration/testIntegration.nim b/tests/integration/testIntegration.nim index 88f27ffd..4d2bc15a 100644 --- a/tests/integration/testIntegration.nim +++ b/tests/integration/testIntegration.nim @@ -26,7 +26,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false: await ethProvider.advanceTime(1.u256) test "nodes can print their peer information": - check client1.info() != client2.info() + check !client1.info() != !client2.info() test "nodes can set chronicles log level": client1.setLogLevel("DEBUG;TRACE:codex") diff --git a/tests/integration/twonodes.nim b/tests/integration/twonodes.nim index b8024079..5e1825ef 100644 --- a/tests/integration/twonodes.nim +++ b/tests/integration/twonodes.nim @@ -52,7 +52,7 @@ template twonodessuite*(name: string, debug1, debug2: string, body) = node1 = startNode(node1Args, debug = debug1) node1.waitUntilStarted() - let bootstrap = client1.info()["spr"].getStr() + let bootstrap = (!client1.info()["spr"]).getStr() var node2Args = @[ "--api-port=8081", diff --git a/tests/testTaiko.nim b/tests/testTaiko.nim index 49ad3226..a799697b 100644 --- a/tests/testTaiko.nim +++ b/tests/testTaiko.nim @@ -29,7 +29,7 @@ suite "Taiko L2 Integration Tests": ]) node1.waitUntilStarted() - let bootstrap = node1.client.info()["spr"].getStr() + let bootstrap = (!node1.client.info())["spr"].getStr() node2 = startNode([ "--data-dir=" & createTempDir("", ""), diff --git a/vendor/nim-serde b/vendor/nim-serde new file mode 160000 index 00000000..a261c3d2 --- /dev/null +++ b/vendor/nim-serde @@ -0,0 +1 @@ +Subproject commit a261c3d214b87d9de0338d6cfd5d134da03d2dc3 From 916e4834d088c9e1837b0a30e8403095fd8b565e Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Tue, 19 Mar 2024 22:08:54 +1100 Subject: [PATCH 4/4] Remove codex_use_hardhat compilation flag (#741) This flag was originally used to change OnChainClock behavior when using hardhat as an ethereum chain source due to a very strange bug which would mark the timestamp of new blocks as one second off the timestamp they should have been. The issue has since been worked around in another manner, and thus this flag is no longer needed. --- .github/workflows/docker-dist-tests.yml | 2 +- build.nims | 6 +++--- codex/conf.nim | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-dist-tests.yml b/.github/workflows/docker-dist-tests.yml index 5ae2e30a..ae1e5e9e 100644 --- a/.github/workflows/docker-dist-tests.yml +++ b/.github/workflows/docker-dist-tests.yml @@ -24,7 +24,7 @@ jobs: name: Build and Push uses: ./.github/workflows/docker-reusable.yml with: - nimflags: '-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_proof_failures=true -d:codex_use_hardhat=false -d:codex_enable_log_counter=true' + nimflags: '-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_proof_failures=true -d:codex_enable_log_counter=true' nat_ip_auto: true tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }} tag_suffix: dist-tests diff --git a/build.nims b/build.nims index 11eec387..33d7b623 100644 --- a/build.nims +++ b/build.nims @@ -25,13 +25,13 @@ task codex, "build codex binary": buildBinary "codex", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" task testCodex, "Build & run Codex tests": - test "testCodex", params = "-d:codex_enable_proof_failures=true -d:codex_use_hardhat=true" + test "testCodex", params = "-d:codex_enable_proof_failures=true" task testContracts, "Build & run Codex Contract tests": test "testContracts" task testIntegration, "Run integration tests": - buildBinary "codex", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true -d:codex_use_hardhat=true" + buildBinary "codex", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true" test "testIntegration" task build, "build codex binary": @@ -75,7 +75,7 @@ task coverage, "generates code coverage report": if f.endswith(".nim"): nimSrcs.add " " & f.absolutePath.quoteShell() echo "======== Running Tests ======== " - test "coverage", srcDir = "tests/", params = " --nimcache:nimcache/coverage -d:release -d:codex_enable_proof_failures=true -d:codex_use_hardhat=true" + test "coverage", srcDir = "tests/", params = " --nimcache:nimcache/coverage -d:release -d:codex_enable_proof_failures=true" exec("rm nimcache/coverage/*.c") rmDir("coverage"); mkDir("coverage") echo " ======== Running LCOV ======== " diff --git a/codex/conf.nim b/codex/conf.nim index 7bf176da..8ec99041 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -59,7 +59,6 @@ proc defaultDataDir*(): string = const codex_enable_api_debug_peers* {.booldefine.} = false codex_enable_proof_failures* {.booldefine.} = false - codex_use_hardhat* {.booldefine.} = false codex_enable_log_counter* {.booldefine.} = false DefaultDataDir* = defaultDataDir()