refactor: use serde for json de/serialization instead of utils/json (#704)
* json > nim-serde bump Should wait until serde is integrated into nim-ethers before making these changes as there will be less import exceptions required. * bump nim-serde * change func to proc due to chronicles side effects * import serde into utils/json, use as proxy import nim-serde into utils/json and use utils/json as a proxy for serde functions, including overloading `%` and `fromJson` for application types. * update tests to use serde * bump serde to latest * remove testjson -- no longer needed * bump serde in nimble * updates to reconcile rebase with master
This commit is contained in:
Eric
GitHub
parent
b4de53f436
commit
f567f4ec15
|
|
@ -209,3 +209,6 @@
|
|||
url = https://github.com/codex-storage/codex-storage-proofs-circuits.git
|
||||
ignore = untracked
|
||||
branch = master
|
||||
[submodule "vendor/nim-serde"]
|
||||
path = vendor/nim-serde
|
||||
url = https://github.com/codex-storage/nim-serde.git
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ requires "presto"
|
|||
requires "protobuf_serialization >= 0.2.0 & < 0.3.0"
|
||||
requires "questionable >= 0.10.13 & < 0.11.0"
|
||||
requires "secp256k1"
|
||||
requires "serde >= 1.0.0 & < 2.0.0"
|
||||
requires "stew"
|
||||
requires "upraises >= 0.1.0 & < 0.2.0"
|
||||
requires "toml_serialization"
|
||||
|
|
|
|||
|
|
@ -17,16 +17,16 @@
|
|||
## module, and specifying `formatIt`. If textlines log output and json log output
|
||||
## need to be different, overload `formatIt` and specify a `LogFormat`. If json
|
||||
## serialization is needed, it can be declared with a `%` proc. `logutils`
|
||||
## imports and exports `utils/json` which handles the de/serialization, examples
|
||||
## imports and exports `nim-serde` which handles the de/serialization, examples
|
||||
## below. **Only `codex/logutils` needs to be imported.**
|
||||
##
|
||||
## Using `logutils` in the Codex codebase:
|
||||
## - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils`
|
||||
## - most of `chronicles` is exported by `logutils`
|
||||
## - Instead of importing `std/json`, import `pkg/codex/utils/json`
|
||||
## - `std/json` is exported by `utils/json` which is exported by `logutils`
|
||||
## - Instead of importing `std/json`, import `pkg/serde/json`
|
||||
## - `std/json` is exported by `serde` which is exported by `logutils`
|
||||
## - Instead of importing `pkg/nim-json-serialization`, import
|
||||
## `pkg/codex/utils/json`
|
||||
## `pkg/serde/json` or use codex-specific overloads by importing `utils/json`
|
||||
## - one of the goals is to remove the use of `nim-json-serialization`
|
||||
##
|
||||
## ```nim
|
||||
|
|
@ -54,7 +54,7 @@
|
|||
## # chronicles json output
|
||||
## {"lvl":"TRC","msg":"test","tid":14397405,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}}
|
||||
## ```
|
||||
## In this case, `BlockAddress` is just an object, so `utils/json` can handle
|
||||
## In this case, `BlockAddress` is just an object, so `nim-serde` can handle
|
||||
## serializing it without issue (only fields annotated with `{.serialize.}` will
|
||||
## serialize (aka opt-in serialization)).
|
||||
##
|
||||
|
|
@ -95,20 +95,19 @@ import pkg/chronicles except toJson, `%`
|
|||
from pkg/libp2p import Cid, MultiAddress, `$`
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import ./utils/json except formatIt # TODO: remove exception?
|
||||
import pkg/stew/byteutils
|
||||
import pkg/stint
|
||||
import pkg/upraises
|
||||
|
||||
import ./utils/json
|
||||
|
||||
export byteutils
|
||||
export chronicles except toJson, formatIt, `%`
|
||||
export questionable
|
||||
export sequtils
|
||||
export json except formatIt
|
||||
export strutils
|
||||
export sugar
|
||||
export upraises
|
||||
export json
|
||||
export results
|
||||
|
||||
func shortLog*(long: string, ellipses = "*", start = 3, stop = 6): string =
|
||||
|
|
|
|||
|
|
@ -104,11 +104,11 @@ proc init*(_: type RestNodeId, id: NodeId): RestNodeId =
|
|||
id: id
|
||||
)
|
||||
|
||||
func `%`*(obj: StorageRequest | Slot): JsonNode =
|
||||
proc `%`*(obj: StorageRequest | Slot): JsonNode =
|
||||
let jsonObj = newJObject()
|
||||
for k, v in obj.fieldPairs: jsonObj[k] = %v
|
||||
jsonObj["id"] = %(obj.id)
|
||||
|
||||
return jsonObj
|
||||
|
||||
func `%`*(obj: RestNodeId): JsonNode = % $obj.id
|
||||
proc `%`*(obj: RestNodeId): JsonNode = % $obj.id
|
||||
|
|
|
|||
|
|
@ -15,11 +15,11 @@ import std/sugar
|
|||
|
||||
import pkg/libp2p
|
||||
import pkg/chronos
|
||||
import pkg/chronicles
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/constantine/math/io/io_fields
|
||||
|
||||
import ../../logutils
|
||||
import ../../utils
|
||||
import ../../stores
|
||||
import ../../manifest
|
||||
|
|
|
|||
|
|
@ -9,12 +9,12 @@
|
|||
|
||||
import std/sugar
|
||||
|
||||
import pkg/chronicles
|
||||
import pkg/chronos
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/stew/arrayops
|
||||
|
||||
import ../../logutils
|
||||
import ../../market
|
||||
import ../../blocktype as bt
|
||||
import ../../merkletree
|
||||
|
|
|
|||
|
|
@ -1,181 +1,16 @@
|
|||
|
||||
import std/json except `%`, `%*`
|
||||
import std/macros
|
||||
import std/options
|
||||
import std/strutils
|
||||
import std/strformat
|
||||
import std/tables
|
||||
import std/typetraits
|
||||
from pkg/ethers import Address
|
||||
from pkg/libp2p import Cid, PeerId, SignedPeerRecord, MultiAddress, AddressInfo, init, `$`
|
||||
import pkg/contractabi
|
||||
import pkg/codexdht/discv5/node as dn
|
||||
import pkg/stew/byteutils
|
||||
import pkg/stint
|
||||
import pkg/serde/json
|
||||
import pkg/questionable/results
|
||||
import ../errors
|
||||
|
||||
export json except `%`, `%*`
|
||||
export json
|
||||
|
||||
type
|
||||
SerializationError = object of CodexError
|
||||
UnexpectedKindError = object of SerializationError
|
||||
|
||||
template serialize* {.pragma.}
|
||||
|
||||
proc newUnexpectedKindError(
|
||||
expectedType: type,
|
||||
expectedKinds: string,
|
||||
json: JsonNode
|
||||
): ref UnexpectedKindError =
|
||||
let kind = if json.isNil: "nil"
|
||||
else: $json.kind
|
||||
newException(UnexpectedKindError,
|
||||
&"deserialization to {$expectedType} failed: expected {expectedKinds} " &
|
||||
&"but got {kind}")
|
||||
|
||||
proc newUnexpectedKindError(
|
||||
expectedType: type,
|
||||
expectedKinds: set[JsonNodeKind],
|
||||
json: JsonNode
|
||||
): ref UnexpectedKindError =
|
||||
newUnexpectedKindError(expectedType, $expectedKinds, json)
|
||||
|
||||
proc newUnexpectedKindError(
|
||||
expectedType: type,
|
||||
expectedKind: JsonNodeKind,
|
||||
json: JsonNode
|
||||
): ref UnexpectedKindError =
|
||||
newUnexpectedKindError(expectedType, {expectedKind}, json)
|
||||
|
||||
template expectJsonKind(
|
||||
expectedType: type,
|
||||
expectedKinds: set[JsonNodeKind],
|
||||
json: JsonNode
|
||||
) =
|
||||
if json.isNil or json.kind notin expectedKinds:
|
||||
return failure(newUnexpectedKindError(expectedType, expectedKinds, json))
|
||||
|
||||
template expectJsonKind(
|
||||
expectedType: type,
|
||||
expectedKind: JsonNodeKind,
|
||||
json: JsonNode
|
||||
) =
|
||||
expectJsonKind(expectedType, {expectedKind}, json)
|
||||
|
||||
proc fromJson*(
|
||||
T: type enum,
|
||||
json: JsonNode
|
||||
): ?!T =
|
||||
expectJsonKind(string, JString, json)
|
||||
catch parseEnum[T](json.str)
|
||||
|
||||
proc fromJson*(
|
||||
_: type string,
|
||||
json: JsonNode
|
||||
): ?!string =
|
||||
if json.isNil:
|
||||
let err = newException(ValueError, "'json' expected, but was nil")
|
||||
return failure(err)
|
||||
elif json.kind == JNull:
|
||||
return success("null")
|
||||
elif json.isNil or json.kind != JString:
|
||||
return failure(newUnexpectedKindError(string, JString, json))
|
||||
catch json.getStr
|
||||
|
||||
proc fromJson*(
|
||||
_: type bool,
|
||||
json: JsonNode
|
||||
): ?!bool =
|
||||
expectJsonKind(bool, JBool, json)
|
||||
catch json.getBool
|
||||
|
||||
proc fromJson*(
|
||||
_: type int,
|
||||
json: JsonNode
|
||||
): ?!int =
|
||||
expectJsonKind(int, JInt, json)
|
||||
catch json.getInt
|
||||
|
||||
proc fromJson*[T: SomeInteger](
|
||||
_: type T,
|
||||
json: JsonNode
|
||||
): ?!T =
|
||||
when T is uint|uint64 or (not defined(js) and int.sizeof == 4):
|
||||
expectJsonKind(T, {JInt, JString}, json)
|
||||
case json.kind
|
||||
of JString:
|
||||
let x = parseBiggestUInt(json.str)
|
||||
return success cast[T](x)
|
||||
else:
|
||||
return success T(json.num)
|
||||
else:
|
||||
expectJsonKind(T, {JInt}, json)
|
||||
return success cast[T](json.num)
|
||||
|
||||
proc fromJson*[T: SomeFloat](
|
||||
_: type T,
|
||||
json: JsonNode
|
||||
): ?!T =
|
||||
expectJsonKind(T, {JInt, JFloat, JString}, json)
|
||||
if json.kind == JString:
|
||||
case json.str
|
||||
of "nan":
|
||||
let b = NaN
|
||||
return success T(b)
|
||||
# dst = NaN # would fail some tests because range conversions would cause CT error
|
||||
# in some cases; but this is not a hot-spot inside this branch and backend can optimize this.
|
||||
of "inf":
|
||||
let b = Inf
|
||||
return success T(b)
|
||||
of "-inf":
|
||||
let b = -Inf
|
||||
return success T(b)
|
||||
else:
|
||||
let err = newUnexpectedKindError(T, "'nan|inf|-inf'", json)
|
||||
return failure(err)
|
||||
else:
|
||||
if json.kind == JFloat:
|
||||
return success T(json.fnum)
|
||||
else:
|
||||
return success T(json.num)
|
||||
|
||||
proc fromJson*(
|
||||
_: type seq[byte],
|
||||
json: JsonNode
|
||||
): ?!seq[byte] =
|
||||
expectJsonKind(seq[byte], JString, json)
|
||||
hexToSeqByte(json.getStr).catch
|
||||
|
||||
proc fromJson*[N: static[int], T: array[N, byte]](
|
||||
_: type T,
|
||||
json: JsonNode
|
||||
): ?!T =
|
||||
expectJsonKind(T, JString, json)
|
||||
T.fromHex(json.getStr).catch
|
||||
|
||||
proc fromJson*[T: distinct](
|
||||
_: type T,
|
||||
json: JsonNode
|
||||
): ?!T =
|
||||
success T(? T.distinctBase.fromJson(json))
|
||||
|
||||
proc fromJson*[N: static[int], T: StUint[N]](
|
||||
_: type T,
|
||||
json: JsonNode
|
||||
): ?!T =
|
||||
expectJsonKind(T, JString, json)
|
||||
catch parse(json.getStr, T)
|
||||
|
||||
proc fromJson*[T](
|
||||
_: type Option[T],
|
||||
json: JsonNode
|
||||
): ?! Option[T] =
|
||||
if json.isNil or json.kind == JNull:
|
||||
return success(none T)
|
||||
without val =? T.fromJson(json), error:
|
||||
return failure(error)
|
||||
success(val.some)
|
||||
|
||||
proc fromJson*(
|
||||
_: type Cid,
|
||||
|
|
@ -184,123 +19,6 @@ proc fromJson*(
|
|||
expectJsonKind(Cid, JString, json)
|
||||
Cid.init(json.str).mapFailure
|
||||
|
||||
proc fromJson*[T](
|
||||
_: type seq[T],
|
||||
json: JsonNode
|
||||
): ?! seq[T] =
|
||||
expectJsonKind(seq[T], JArray, json)
|
||||
var arr: seq[T] = @[]
|
||||
for elem in json.elems:
|
||||
arr.add(? T.fromJson(elem))
|
||||
success arr
|
||||
|
||||
proc fromJson*[T: ref object or object](
|
||||
_: type T,
|
||||
json: JsonNode
|
||||
): ?!T =
|
||||
expectJsonKind(T, JObject, json)
|
||||
var res = when type(T) is ref: T.new() else: T.default
|
||||
|
||||
# Leave this in, it's good for debugging:
|
||||
# trace "deserializing object", to = $T, json
|
||||
for name, value in fieldPairs(when type(T) is ref: res[] else: res):
|
||||
if json{name} != nil:
|
||||
without parsed =? type(value).fromJson(json{name}), e:
|
||||
error "error deserializing field",
|
||||
field = $T & "." & name,
|
||||
json = json{name},
|
||||
error = e.msg
|
||||
return failure(e)
|
||||
value = parsed
|
||||
success(res)
|
||||
|
||||
proc fromJson*[T: object](
|
||||
_: type T,
|
||||
bytes: seq[byte]
|
||||
): ?!T =
|
||||
let json = ?catch parseJson(string.fromBytes(bytes))
|
||||
T.fromJson(json)
|
||||
|
||||
proc fromJson*[T: ref object](
|
||||
_: type T,
|
||||
bytes: seq[byte]
|
||||
): ?!T =
|
||||
let json = ?catch parseJson(string.fromBytes(bytes))
|
||||
T.fromJson(json)
|
||||
|
||||
func `%`*(s: string): JsonNode = newJString(s)
|
||||
|
||||
func `%`*(n: uint): JsonNode =
|
||||
if n > cast[uint](int.high):
|
||||
newJString($n)
|
||||
else:
|
||||
newJInt(BiggestInt(n))
|
||||
|
||||
func `%`*(n: int): JsonNode = newJInt(n)
|
||||
|
||||
func `%`*(n: BiggestUInt): JsonNode =
|
||||
if n > cast[BiggestUInt](BiggestInt.high):
|
||||
newJString($n)
|
||||
else:
|
||||
newJInt(BiggestInt(n))
|
||||
|
||||
func `%`*(n: BiggestInt): JsonNode = newJInt(n)
|
||||
|
||||
func `%`*(n: float): JsonNode =
|
||||
if n != n: newJString("nan")
|
||||
elif n == Inf: newJString("inf")
|
||||
elif n == -Inf: newJString("-inf")
|
||||
else: newJFloat(n)
|
||||
|
||||
func `%`*(b: bool): JsonNode = newJBool(b)
|
||||
|
||||
func `%`*(keyVals: openArray[tuple[key: string, val: JsonNode]]): JsonNode =
|
||||
if keyVals.len == 0: return newJArray()
|
||||
let jObj = newJObject()
|
||||
for key, val in items(keyVals): jObj.fields[key] = val
|
||||
jObj
|
||||
|
||||
template `%`*(j: JsonNode): JsonNode = j
|
||||
|
||||
func `%`*[T](table: Table[string, T]|OrderedTable[string, T]): JsonNode =
|
||||
let jObj = newJObject()
|
||||
for k, v in table: jObj[k] = ? %v
|
||||
jObj
|
||||
|
||||
func `%`*[T](opt: Option[T]): JsonNode =
|
||||
if opt.isSome: %(opt.get) else: newJNull()
|
||||
|
||||
func `%`*[T: object](obj: T): JsonNode =
|
||||
let jsonObj = newJObject()
|
||||
for name, value in obj.fieldPairs:
|
||||
when value.hasCustomPragma(serialize):
|
||||
jsonObj[name] = %value
|
||||
jsonObj
|
||||
|
||||
func `%`*[T: ref object](obj: T): JsonNode =
|
||||
let jsonObj = newJObject()
|
||||
for name, value in obj[].fieldPairs:
|
||||
when value.hasCustomPragma(serialize):
|
||||
jsonObj[name] = %(value)
|
||||
jsonObj
|
||||
|
||||
proc `%`*(o: enum): JsonNode = % $o
|
||||
|
||||
func `%`*(stint: StInt|StUint): JsonNode = %stint.toString
|
||||
|
||||
func `%`*(cstr: cstring): JsonNode = % $cstr
|
||||
|
||||
func `%`*(arr: openArray[byte]): JsonNode = % arr.to0xHex
|
||||
|
||||
func `%`*[T](elements: openArray[T]): JsonNode =
|
||||
let jObj = newJArray()
|
||||
for elem in elements: jObj.add(%elem)
|
||||
jObj
|
||||
|
||||
func `%`*[T: distinct](id: T): JsonNode =
|
||||
type baseType = T.distinctBase
|
||||
% baseType(id)
|
||||
|
||||
func `%`*(cid: Cid): JsonNode = % $cid
|
||||
|
||||
func `%`*(obj: PeerId): JsonNode = % $obj
|
||||
|
|
@ -314,36 +32,3 @@ func `%`*(obj: AddressInfo): JsonNode = % $obj.address
|
|||
func `%`*(obj: MultiAddress): JsonNode = % $obj
|
||||
|
||||
func `%`*(address: ethers.Address): JsonNode = % $address
|
||||
|
||||
func toJson*[T](item: T): string = $(%item)
|
||||
|
||||
proc toJsnImpl(x: NimNode): NimNode =
|
||||
case x.kind
|
||||
of nnkBracket: # array
|
||||
if x.len == 0: return newCall(bindSym"newJArray")
|
||||
result = newNimNode(nnkBracket)
|
||||
for i in 0 ..< x.len:
|
||||
result.add(toJsnImpl(x[i]))
|
||||
result = newCall(bindSym("%", brOpen), result)
|
||||
of nnkTableConstr: # object
|
||||
if x.len == 0: return newCall(bindSym"newJObject")
|
||||
result = newNimNode(nnkTableConstr)
|
||||
for i in 0 ..< x.len:
|
||||
x[i].expectKind nnkExprColonExpr
|
||||
result.add newTree(nnkExprColonExpr, x[i][0], toJsnImpl(x[i][1]))
|
||||
result = newCall(bindSym("%", brOpen), result)
|
||||
of nnkCurly: # empty object
|
||||
x.expectLen(0)
|
||||
result = newCall(bindSym"newJObject")
|
||||
of nnkNilLit:
|
||||
result = newCall(bindSym"newJNull")
|
||||
of nnkPar:
|
||||
if x.len == 1: result = toJsnImpl(x[0])
|
||||
else: result = newCall(bindSym("%", brOpen), x)
|
||||
else:
|
||||
result = newCall(bindSym("%", brOpen), x)
|
||||
|
||||
macro `%*`*(x: untyped): JsonNode =
|
||||
## Convert an expression to a JsonNode directly, without having to specify
|
||||
## `%` for every element.
|
||||
result = toJsnImpl(x)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import std/sequtils
|
||||
import pkg/chronicles
|
||||
import pkg/chronos
|
||||
import pkg/datastore
|
||||
import pkg/questionable
|
||||
|
|
|
|||
|
|
@ -8,11 +8,11 @@ import ../../../asynctest
|
|||
import pkg/chronos
|
||||
import pkg/poseidon2
|
||||
import pkg/datastore
|
||||
import pkg/serde/json
|
||||
|
||||
import pkg/codex/slots {.all.}
|
||||
import pkg/codex/slots/types {.all.}
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/utils/json
|
||||
import pkg/codex/codextypes
|
||||
import pkg/codex/manifest
|
||||
import pkg/codex/stores
|
||||
|
|
@ -33,7 +33,7 @@ suite "Test Circom Compat Backend - control inputs":
|
|||
setup:
|
||||
let
|
||||
inputData = readFile("tests/circuits/fixtures/input.json")
|
||||
inputJson = parseJson(inputData)
|
||||
inputJson = !JsonNode.parse(inputData)
|
||||
|
||||
proofInputs = Poseidon2Hash.jsonToProofInput(inputJson)
|
||||
circom = CircomCompat.init(r1cs, wasm, zkey)
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ suite "Test Sampler - control samples":
|
|||
|
||||
setup:
|
||||
inputData = readFile("tests/circuits/fixtures/input.json")
|
||||
inputJson = parseJson(inputData)
|
||||
inputJson = !JsonNode.parse(inputData)
|
||||
proofInput = Poseidon2Hash.jsonToProofInput(inputJson)
|
||||
|
||||
test "Should verify control samples":
|
||||
|
|
|
|||
|
|
@ -18,9 +18,9 @@ import pkg/codex/contracts/requests
|
|||
import pkg/codex/contracts
|
||||
import pkg/codex/merkletree
|
||||
import pkg/codex/stores/cachestore
|
||||
|
||||
import pkg/codex/slots/types
|
||||
import pkg/codex/slots/sampler/utils
|
||||
import pkg/codex/utils/json
|
||||
|
||||
import ../backends/helpers
|
||||
import ../../helpers
|
||||
|
|
@ -38,7 +38,7 @@ asyncchecksuite "Test proof sampler utils":
|
|||
|
||||
setup:
|
||||
inputData = readFile("tests/circuits/fixtures/input.json")
|
||||
inputJson = parseJson(inputData)
|
||||
inputJson = !JsonNode.parse(inputData)
|
||||
proofInput = Poseidon2Hash.jsonToProofInput(inputJson)
|
||||
|
||||
test "Extract low bits":
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import pkg/codex/contracts/requests
|
|||
import pkg/codex/logutils
|
||||
import pkg/codex/purchasing/purchaseid
|
||||
import pkg/codex/units
|
||||
import pkg/codex/utils/json
|
||||
import pkg/libp2p/cid
|
||||
import pkg/libp2p/multiaddress
|
||||
import pkg/questionable
|
||||
|
|
@ -61,8 +62,8 @@ checksuite "Test logging output":
|
|||
outputLines.contains(toFind)
|
||||
|
||||
template loggedJson(prop, expected): auto =
|
||||
let json = $ parseJson(outputJson){prop}
|
||||
json == expected
|
||||
let jsonVal = !JsonNode.parse(outputJson)
|
||||
$ jsonVal{prop} == expected
|
||||
|
||||
template log(val) =
|
||||
testlines.trace "test", val
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import ./utils/testjson
|
||||
import ./utils/testoptionalcast
|
||||
import ./utils/testkeyutils
|
||||
import ./utils/testasyncstatemachine
|
||||
|
|
|
|||
|
|
@ -1,345 +0,0 @@
|
|||
import std/math
|
||||
import std/options
|
||||
import std/strformat
|
||||
import std/strutils
|
||||
import std/unittest
|
||||
import pkg/stew/byteutils
|
||||
import pkg/stint
|
||||
import pkg/codex/contracts/requests
|
||||
from pkg/codex/rest/json import RestPurchase
|
||||
import pkg/codex/logutils
|
||||
import pkg/codex/utils/json as utilsjson
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/libp2p
|
||||
import ../helpers
|
||||
|
||||
checksuite "json serialization":
|
||||
var request: StorageRequest
|
||||
var requestJson: JsonNode
|
||||
|
||||
func flatten(s: string): string =
|
||||
s.replace(" ")
|
||||
.replace("\n")
|
||||
|
||||
setup:
|
||||
request = StorageRequest(
|
||||
client: Address.init("0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174").get(),
|
||||
ask: StorageAsk(
|
||||
slots: 4,
|
||||
slotSize: (1 * 1024 * 1024 * 1024).u256, # 1 Gigabyte
|
||||
duration: (10 * 60 * 60).u256, # 10 hours
|
||||
collateral: 200.u256,
|
||||
proofProbability: 4.u256, # require a proof roughly once every 4 periods
|
||||
reward: 84.u256,
|
||||
maxSlotLoss: 2 # 2 slots can be freed without data considered to be lost
|
||||
),
|
||||
content: StorageContent(
|
||||
cid: "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob",
|
||||
merkleRoot: array[32, byte].fromHex("0xc066dd7e405de5a795ce1e765209cfaa6de6c74829c607d1a2fe53107107a298")
|
||||
),
|
||||
expiry: 1691545330.u256,
|
||||
nonce: Nonce array[32, byte].fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1")
|
||||
)
|
||||
requestJson = """{
|
||||
"client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174",
|
||||
"ask": {
|
||||
"slots": 4,
|
||||
"slotSize": "1073741824",
|
||||
"duration": "36000",
|
||||
"proofProbability": "4",
|
||||
"reward": "84",
|
||||
"collateral": "200",
|
||||
"maxSlotLoss": 2
|
||||
},
|
||||
"content": {
|
||||
"cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob",
|
||||
"merkleRoot": "0xc066dd7e405de5a795ce1e765209cfaa6de6c74829c607d1a2fe53107107a298"
|
||||
},
|
||||
"expiry": "1691545330",
|
||||
"nonce": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"
|
||||
}""".parseJson
|
||||
|
||||
test "serializes UInt256 to non-hex string representation":
|
||||
check (% 100000.u256) == newJString("100000")
|
||||
|
||||
test "serializes sequence to an array":
|
||||
let json = % @[1, 2, 3]
|
||||
let expected = "[1,2,3]"
|
||||
check $json == expected
|
||||
|
||||
test "serializes Option[T] when has a value":
|
||||
let obj = %(some 1)
|
||||
let expected = "1"
|
||||
check $obj == expected
|
||||
|
||||
test "serializes Option[T] when doesn't have a value":
|
||||
let obj = %(none int)
|
||||
let expected = "null"
|
||||
check $obj == expected
|
||||
|
||||
test "serializes uints int.high or smaller":
|
||||
let largeUInt: uint = uint(int.high)
|
||||
check %largeUInt == newJInt(BiggestInt(largeUInt))
|
||||
|
||||
test "serializes large uints":
|
||||
let largeUInt: uint = uint(int.high) + 1'u
|
||||
check %largeUInt == newJString($largeUInt)
|
||||
|
||||
|
||||
test "serializes Inf float":
|
||||
check %Inf == newJString("inf")
|
||||
|
||||
test "serializes -Inf float":
|
||||
check %(-Inf) == newJString("-inf")
|
||||
|
||||
test "deserializes NaN float":
|
||||
check %NaN == newJString("nan")
|
||||
|
||||
test "can construct json objects with %*":
|
||||
type MyObj = object
|
||||
mystring {.serialize.}: string
|
||||
myint {.serialize.}: int
|
||||
myoption {.serialize.}: ?bool
|
||||
|
||||
let myobj = MyObj(mystring: "abc", myint: 123, myoption: some true)
|
||||
let mystuint = 100000.u256
|
||||
|
||||
let json = %*{
|
||||
"myobj": myobj,
|
||||
"mystuint": mystuint
|
||||
}
|
||||
|
||||
let expected = """{
|
||||
"myobj": {
|
||||
"mystring": "abc",
|
||||
"myint": 123,
|
||||
"myoption": true
|
||||
},
|
||||
"mystuint": "100000"
|
||||
}""".flatten
|
||||
|
||||
check $json == expected
|
||||
|
||||
test "only serializes marked fields":
|
||||
type MyObj = object
|
||||
mystring {.serialize.}: string
|
||||
myint {.serialize.}: int
|
||||
mybool: bool
|
||||
|
||||
let obj = % MyObj(mystring: "abc", myint: 1, mybool: true)
|
||||
|
||||
let expected = """{
|
||||
"mystring": "abc",
|
||||
"myint": 1
|
||||
}""".flatten
|
||||
|
||||
check $obj == expected
|
||||
|
||||
test "serializes ref objects":
|
||||
type MyRef = ref object
|
||||
mystring {.serialize.}: string
|
||||
myint {.serialize.}: int
|
||||
|
||||
let obj = % MyRef(mystring: "abc", myint: 1)
|
||||
|
||||
let expected = """{
|
||||
"mystring": "abc",
|
||||
"myint": 1
|
||||
}""".flatten
|
||||
|
||||
check $obj == expected
|
||||
|
||||
test "serializes RestPurchase":
|
||||
let request = % RestPurchase(
|
||||
request: some request,
|
||||
requestId: RequestId.fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"),
|
||||
error: some "error",
|
||||
state: "state"
|
||||
)
|
||||
let expected = """{
|
||||
"requestId": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1",
|
||||
"request": {
|
||||
"client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174",
|
||||
"ask": {
|
||||
"slots": 4,
|
||||
"slotSize": "1073741824",
|
||||
"duration": "36000",
|
||||
"proofProbability": "4",
|
||||
"reward": "84",
|
||||
"collateral": "200",
|
||||
"maxSlotLoss": 2
|
||||
},
|
||||
"content": {
|
||||
"cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob"
|
||||
},
|
||||
"expiry": "1691545330"
|
||||
},
|
||||
"state": "state",
|
||||
"error": "error"
|
||||
}""".flatten
|
||||
check $request == expected
|
||||
|
||||
test "serializes StorageRequest":
|
||||
let expected = """{
|
||||
"client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174",
|
||||
"ask": {
|
||||
"slots": 4,
|
||||
"slotSize": "1073741824",
|
||||
"duration": "36000",
|
||||
"proofProbability": "4",
|
||||
"reward": "84",
|
||||
"collateral": "200",
|
||||
"maxSlotLoss": 2
|
||||
},
|
||||
"content": {
|
||||
"cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob"
|
||||
},
|
||||
"expiry": "1691545330"
|
||||
}""".flatten
|
||||
check request.toJson == expected
|
||||
|
||||
test "deserialize enum":
|
||||
let json = newJString($CidVersion.CIDv1)
|
||||
check !CidVersion.fromJson(json) == CIDv1
|
||||
|
||||
test "deserializes UInt256 from non-hex string representation":
|
||||
let json = newJString("100000")
|
||||
check !UInt256.fromJson(json) == 100000.u256
|
||||
|
||||
test "deserializes Option[T] when has a value":
|
||||
let json = newJInt(1)
|
||||
check (!fromJson(?int, json) == some 1)
|
||||
|
||||
test "deserializes Option[T] when doesn't have a value":
|
||||
let json = newJNull()
|
||||
check !fromJson(?int, json) == none int
|
||||
|
||||
test "deserializes float":
|
||||
let json = newJFloat(1.234)
|
||||
check !float.fromJson(json) == 1.234
|
||||
|
||||
test "deserializes Inf float":
|
||||
let json = newJString("inf")
|
||||
check !float.fromJson(json) == Inf
|
||||
|
||||
test "deserializes -Inf float":
|
||||
let json = newJString("-inf")
|
||||
check !float.fromJson(json) == -Inf
|
||||
|
||||
test "deserializes NaN float":
|
||||
let json = newJString("nan")
|
||||
check float.fromJson(json).get.isNaN
|
||||
|
||||
test "deserializes array to sequence":
|
||||
let expected = @[1, 2, 3]
|
||||
let json = "[1,2,3]".parseJson
|
||||
check !seq[int].fromJson(json) == expected
|
||||
|
||||
test "deserializes uints int.high or smaller":
|
||||
let largeUInt: uint = uint(int.high)
|
||||
let json = newJInt(BiggestInt(largeUInt))
|
||||
check !uint.fromJson(json) == largeUInt
|
||||
|
||||
test "deserializes large uints":
|
||||
let largeUInt: uint = uint(int.high) + 1'u
|
||||
let json = newJString($BiggestUInt(largeUInt))
|
||||
check !uint.fromJson(json) == largeUInt
|
||||
|
||||
test "can deserialize json objects":
|
||||
type MyObj = object
|
||||
mystring: string
|
||||
myint: int
|
||||
myoption: ?bool
|
||||
|
||||
let expected = MyObj(mystring: "abc", myint: 123, myoption: some true)
|
||||
|
||||
let json = parseJson("""{
|
||||
"mystring": "abc",
|
||||
"myint": 123,
|
||||
"myoption": true
|
||||
}""")
|
||||
check !MyObj.fromJson(json) == expected
|
||||
|
||||
test "ignores serialize pragma when deserializing":
|
||||
type MyObj = object
|
||||
mystring {.serialize.}: string
|
||||
mybool: bool
|
||||
|
||||
let expected = MyObj(mystring: "abc", mybool: true)
|
||||
|
||||
let json = parseJson("""{
|
||||
"mystring": "abc",
|
||||
"mybool": true
|
||||
}""")
|
||||
|
||||
check !MyObj.fromJson(json) == expected
|
||||
|
||||
test "deserializes objects with extra fields":
|
||||
type MyObj = object
|
||||
mystring: string
|
||||
mybool: bool
|
||||
|
||||
let expected = MyObj(mystring: "abc", mybool: true)
|
||||
|
||||
let json = """{
|
||||
"mystring": "abc",
|
||||
"mybool": true,
|
||||
"extra": "extra"
|
||||
}""".parseJson
|
||||
check !MyObj.fromJson(json) == expected
|
||||
|
||||
test "deserializes objects with less fields":
|
||||
type MyObj = object
|
||||
mystring: string
|
||||
mybool: bool
|
||||
|
||||
let expected = MyObj(mystring: "abc", mybool: false)
|
||||
|
||||
let json = """{
|
||||
"mystring": "abc"
|
||||
}""".parseJson
|
||||
check !MyObj.fromJson(json) == expected
|
||||
|
||||
test "deserializes ref objects":
|
||||
type MyRef = ref object
|
||||
mystring: string
|
||||
myint: int
|
||||
|
||||
let expected = MyRef(mystring: "abc", myint: 1)
|
||||
|
||||
let json = """{
|
||||
"mystring": "abc",
|
||||
"myint": 1
|
||||
}""".parseJson
|
||||
|
||||
let deserialized = !MyRef.fromJson(json)
|
||||
check deserialized.mystring == expected.mystring
|
||||
check deserialized.myint == expected.myint
|
||||
|
||||
test "deserializes Cid":
|
||||
let
|
||||
jCid = newJString("zdj7Wakya26ggQWkvMdHYFcPgZ7Qh2HdMooQDDFDHkk4uHS14")
|
||||
cid = "zdj7Wakya26ggQWkvMdHYFcPgZ7Qh2HdMooQDDFDHkk4uHS14"
|
||||
|
||||
check:
|
||||
!Cid.fromJson(jCid) == !Cid.init(cid).mapFailure
|
||||
|
||||
test "deserializes StorageRequest":
|
||||
check !StorageRequest.fromJson(requestJson) == request
|
||||
|
||||
test "deserializes RestPurchase":
|
||||
let json = """{
|
||||
"requestId": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1",
|
||||
"state": "state",
|
||||
"error": "error"
|
||||
}""".parseJson
|
||||
json["request"] = requestJson
|
||||
|
||||
let expected = RestPurchase(
|
||||
requestId: RequestId.fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"),
|
||||
state: "state",
|
||||
error: some "error",
|
||||
request: some request
|
||||
)
|
||||
check !RestPurchase.fromJson(json) == expected
|
||||
|
|
@ -19,9 +19,9 @@ type CodexClient* = ref object
|
|||
proc new*(_: type CodexClient, baseurl: string): CodexClient =
|
||||
CodexClient(http: newHttpClient(), baseurl: baseurl)
|
||||
|
||||
proc info*(client: CodexClient): JsonNode =
|
||||
proc info*(client: CodexClient): ?!JsonNode =
|
||||
let url = client.baseurl & "/debug/info"
|
||||
client.http.getContent(url).parseJson()
|
||||
JsonNode.parse( client.http.getContent(url) )
|
||||
|
||||
proc setLogLevel*(client: CodexClient, level: string) =
|
||||
let url = client.baseurl & "/debug/chronicles/loglevel?level=" & level
|
||||
|
|
@ -52,8 +52,7 @@ proc list*(client: CodexClient): ?!seq[RestContent] =
|
|||
if response.status != "200 OK":
|
||||
return failure(response.status)
|
||||
|
||||
let json = ? parseJson(response.body).catch
|
||||
seq[RestContent].fromJson(json)
|
||||
seq[RestContent].fromJson(response.body)
|
||||
|
||||
proc space*(client: CodexClient): ?!RestRepoStore =
|
||||
let url = client.baseurl & "/space"
|
||||
|
|
@ -62,8 +61,7 @@ proc space*(client: CodexClient): ?!RestRepoStore =
|
|||
if response.status != "200 OK":
|
||||
return failure(response.status)
|
||||
|
||||
let json = ? parseJson(response.body).catch
|
||||
RestRepoStore.fromJson(json)
|
||||
RestRepoStore.fromJson(response.body)
|
||||
|
||||
proc requestStorageRaw*(
|
||||
client: CodexClient,
|
||||
|
|
@ -116,8 +114,7 @@ proc getPurchase*(client: CodexClient, purchaseId: PurchaseId): ?!RestPurchase =
|
|||
let url = client.baseurl & "/storage/purchases/" & purchaseId.toHex
|
||||
try:
|
||||
let body = client.http.getContent(url)
|
||||
let json = ? parseJson(body).catch
|
||||
return RestPurchase.fromJson(json)
|
||||
return RestPurchase.fromJson(body)
|
||||
except CatchableError as e:
|
||||
return failure e.msg
|
||||
|
||||
|
|
@ -125,16 +122,14 @@ proc getSalesAgent*(client: CodexClient, slotId: SlotId): ?!RestSalesAgent =
|
|||
let url = client.baseurl & "/sales/slots/" & slotId.toHex
|
||||
try:
|
||||
let body = client.http.getContent(url)
|
||||
let json = ? parseJson(body).catch
|
||||
return RestSalesAgent.fromJson(json)
|
||||
return RestSalesAgent.fromJson(body)
|
||||
except CatchableError as e:
|
||||
return failure e.msg
|
||||
|
||||
proc getSlots*(client: CodexClient): ?!seq[Slot] =
|
||||
let url = client.baseurl & "/sales/slots"
|
||||
let body = client.http.getContent(url)
|
||||
let json = ? parseJson(body).catch
|
||||
seq[Slot].fromJson(json)
|
||||
seq[Slot].fromJson(body)
|
||||
|
||||
proc postAvailability*(
|
||||
client: CodexClient,
|
||||
|
|
@ -151,13 +146,13 @@ proc postAvailability*(
|
|||
}
|
||||
let response = client.http.post(url, $json)
|
||||
doAssert response.status == "200 OK", "expected 200 OK, got " & response.status & ", body: " & response.body
|
||||
Availability.fromJson(response.body.parseJson)
|
||||
Availability.fromJson(response.body)
|
||||
|
||||
proc getAvailabilities*(client: CodexClient): ?!seq[Availability] =
|
||||
## Call sales availability REST endpoint
|
||||
let url = client.baseurl & "/sales/availability"
|
||||
let body = client.http.getContent(url)
|
||||
seq[Availability].fromJson(parseJson(body))
|
||||
seq[Availability].fromJson(body)
|
||||
|
||||
proc close*(client: CodexClient) =
|
||||
client.http.close()
|
||||
|
|
|
|||
|
|
@ -288,7 +288,11 @@ template multinodesuite*(name: string, body: untyped) =
|
|||
node: node
|
||||
)
|
||||
if clients().len == 1:
|
||||
bootstrap = CodexProcess(node).client.info()["spr"].getStr()
|
||||
without ninfo =? CodexProcess(node).client.info():
|
||||
# raise CatchableError instead of Defect (with .get or !) so we
|
||||
# can gracefully shutdown and prevent zombies
|
||||
raiseMultiNodeSuiteError "Failed to get node info"
|
||||
bootstrap = ninfo["spr"].getStr()
|
||||
|
||||
if var providers =? nodeConfigs.providers:
|
||||
failAndTeardownOnError "failed to start provider nodes":
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
|
|||
await ethProvider.advanceTime(1.u256)
|
||||
|
||||
test "nodes can print their peer information":
|
||||
check client1.info() != client2.info()
|
||||
check !client1.info() != !client2.info()
|
||||
|
||||
test "nodes can set chronicles log level":
|
||||
client1.setLogLevel("DEBUG;TRACE:codex")
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ template twonodessuite*(name: string, debug1, debug2: string, body) =
|
|||
node1 = startNode(node1Args, debug = debug1)
|
||||
node1.waitUntilStarted()
|
||||
|
||||
let bootstrap = client1.info()["spr"].getStr()
|
||||
let bootstrap = (!client1.info()["spr"]).getStr()
|
||||
|
||||
var node2Args = @[
|
||||
"--api-port=8081",
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ suite "Taiko L2 Integration Tests":
|
|||
])
|
||||
node1.waitUntilStarted()
|
||||
|
||||
let bootstrap = node1.client.info()["spr"].getStr()
|
||||
let bootstrap = (!node1.client.info())["spr"].getStr()
|
||||
|
||||
node2 = startNode([
|
||||
"--data-dir=" & createTempDir("", ""),
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit a261c3d214b87d9de0338d6cfd5d134da03d2dc3
|
||||
Loading…
Reference in New Issue