Improve integration testing client (CodexClient) and json serialization (#514)

* Improve integration testing client (CodexClient) and json serialization

The current client used for integration testing against the REST endpoints for Codex accepts and passes primitive types. This caused a hard to diagnose bug where a `uint` was not being deserialized correctly.

In addition, the json de/serializing done between the CodexClient and REST client was not easy to read and was not tested.

These changes bring non-primitive types to most of the CodexClient functions, allowing us to lean on the compiler to ensure we're providing correct typings. More importantly, a json de/serialization util was created as a drop-in replacement for the std/json lib, with the main two differences being that field serialization is opt-in (instead of opt-out as in the case of json_serialization) and serialization errors are captured and logged, making debugging serialization issues much easier.

* Update integration test to use nodes=2 and tolerance=1

* clean up
This commit is contained in:
Eric 2023-09-01 15:44:41 +10:00 committed by GitHub
parent fc3eac9dbc
commit 37b3d99c3d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 884 additions and 183 deletions

View File

@ -1,4 +1,5 @@
import std/hashes
import std/typetraits
import pkg/contractabi
import pkg/nimcrypto
import pkg/ethers/fields
@ -6,26 +7,27 @@ import pkg/questionable/results
import pkg/stew/byteutils
import pkg/json_serialization
import pkg/upraises
import ../utils/json
export contractabi
type
StorageRequest* = object
client*: Address
ask*: StorageAsk
content*: StorageContent
expiry*: UInt256
client* {.serialize.}: Address
ask* {.serialize.}: StorageAsk
content* {.serialize.}: StorageContent
expiry* {.serialize.}: UInt256
nonce*: Nonce
StorageAsk* = object
slots*: uint64
slotSize*: UInt256
duration*: UInt256
proofProbability*: UInt256
reward*: UInt256
collateral*: UInt256
maxSlotLoss*: uint64
slots* {.serialize.}: uint64
slotSize* {.serialize.}: UInt256
duration* {.serialize.}: UInt256
proofProbability* {.serialize.}: UInt256
reward* {.serialize.}: UInt256
collateral* {.serialize.}: UInt256
maxSlotLoss* {.serialize.}: uint64
StorageContent* = object
cid*: string
cid* {.serialize.}: string
erasure*: StorageErasure
por*: StoragePoR
StorageErasure* = object
@ -35,8 +37,8 @@ type
publicKey*: seq[byte]
name*: seq[byte]
Slot* = object
request*: StorageRequest
slotIndex*: UInt256
request* {.serialize.}: StorageRequest
slotIndex* {.serialize.}: UInt256
SlotId* = distinct array[32, byte]
RequestId* = distinct array[32, byte]
Nonce* = distinct array[32, byte]
@ -75,6 +77,10 @@ proc fromHex*(T: type SlotId, hex: string): T =
proc fromHex*(T: type Nonce, hex: string): T =
T array[32, byte].fromHex(hex)
proc fromHex*[T: distinct](_: type T, hex: string): T =
type baseType = T.distinctBase
T baseType.fromHex(hex)
func fromTuple(_: type StorageRequest, tupl: tuple): StorageRequest =
StorageRequest(
client: tupl[0],

View File

@ -254,7 +254,7 @@ proc requestStorage*(
## - Run the PoR setup on the erasure dataset
## - Call into the marketplace and purchasing contracts
##
trace "Received a request for storage!", cid, duration, nodes, tolerance, reward
trace "Received a request for storage!", cid, duration, nodes, tolerance, reward, proofProbability, collateral, expiry
without contracts =? self.contracts.client:
trace "Purchasing not available"

View File

@ -34,7 +34,7 @@ import pkg/codexdht/discv5/node as dn
import ../node
import ../blocktype
import ../conf
import ../contracts
import ../contracts except `%*`, `%` # imported from contracts/marketplace (exporting ethers)
import ../streams
import ./coders
@ -361,10 +361,15 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
let body = await request.getBody()
without availability =? Availability.fromJson(body), error:
without restAv =? RestAvailability.fromJson(body), error:
return RestApiResponse.error(Http400, error.msg)
let reservations = contracts.sales.context.reservations
# assign id to availability via init
let availability = Availability.init(restAv.size,
restAv.duration,
restAv.minPrice,
restAv.maxCollateral)
if not reservations.hasAvailable(availability.size.truncate(uint)):
return RestApiResponse.error(Http422, "Not enough storage quota")
@ -389,7 +394,12 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
without purchase =? contracts.purchasing.getPurchase(id):
return RestApiResponse.error(Http404)
let json = %purchase
let json = % RestPurchase(
state: purchase.state |? "none",
error: purchase.error.?msg,
request: purchase.request,
requestId: purchase.requestId
)
return RestApiResponse.response($json, contentType="application/json")

View File

@ -1,67 +1,33 @@
import std/json
import std/strutils
import pkg/stew/byteutils
import pkg/questionable
import pkg/questionable/results
import pkg/stew/byteutils
import ../sales
import ../purchasing
import ../utils/stintutils
import ../utils/json
export json
type
StorageRequestParams* = object
duration*: UInt256
proofProbability*: UInt256
reward*: UInt256
collateral*: UInt256
expiry*: ?UInt256
nodes*: ?uint
tolerance*: ?uint
duration* {.serialize.}: UInt256
proofProbability* {.serialize.}: UInt256
reward* {.serialize.}: UInt256
collateral* {.serialize.}: UInt256
expiry* {.serialize.}: ?UInt256
nodes* {.serialize.}: ?uint
tolerance* {.serialize.}: ?uint
proc fromJson*(
_: type Availability,
bytes: seq[byte]
): ?!Availability =
let json = ?catch parseJson(string.fromBytes(bytes))
let size = ?catch UInt256.fromDecimal(json["size"].getStr)
let duration = ?catch UInt256.fromDecimal(json["duration"].getStr)
let minPrice = ?catch UInt256.fromDecimal(json["minPrice"].getStr)
let maxCollateral = ?catch UInt256.fromDecimal(json["maxCollateral"].getStr)
success Availability.init(size, duration, minPrice, maxCollateral)
RestPurchase* = object
requestId* {.serialize.}: RequestId
request* {.serialize.}: ?StorageRequest
state* {.serialize.}: string
error* {.serialize.}: ?string
proc fromJson*(
_: type StorageRequestParams,
bytes: seq[byte]
): ?! StorageRequestParams =
let json = ?catch parseJson(string.fromBytes(bytes))
let duration = ?catch UInt256.fromDecimal(json["duration"].getStr)
let proofProbability = ?catch UInt256.fromDecimal(json["proofProbability"].getStr)
let reward = ?catch UInt256.fromDecimal(json["reward"].getStr)
let collateral = ?catch UInt256.fromDecimal(json["collateral"].getStr)
let expiry = UInt256.fromDecimal(json["expiry"].getStr).catch.option
let nodes = parseUInt(json["nodes"].getStr).catch.option
let tolerance = parseUInt(json["tolerance"].getStr).catch.option
success StorageRequestParams(
duration: duration,
proofProbability: proofProbability,
reward: reward,
collateral: collateral,
expiry: expiry,
nodes: nodes,
tolerance: tolerance
)
func `%`*(address: Address): JsonNode =
% $address
func `%`*(stint: StInt|StUint): JsonNode=
%(stint.toString)
func `%`*(arr: openArray[byte]): JsonNode =
%("0x" & arr.toHex)
func `%`*(id: RequestId | SlotId | Nonce | AvailabilityId): JsonNode =
% id.toArray
RestAvailability* = object
size* {.serialize.}: UInt256
duration* {.serialize.}: UInt256
minPrice* {.serialize.}: UInt256
maxCollateral* {.serialize.}: UInt256
func `%`*(obj: StorageRequest | Slot): JsonNode =
let jsonObj = newJObject()
@ -69,11 +35,3 @@ func `%`*(obj: StorageRequest | Slot): JsonNode =
jsonObj["id"] = %(obj.id)
return jsonObj
func `%`*(purchase: Purchase): JsonNode =
%*{
"state": purchase.state |? "none",
"error": purchase.error.?msg,
"request": purchase.request,
"requestId": purchase.requestId
}

View File

@ -19,6 +19,7 @@ import pkg/stew/byteutils
import pkg/nimcrypto
import pkg/questionable
import pkg/questionable/results
import ../utils/json
push: {.upraises: [].}
@ -34,11 +35,11 @@ logScope:
type
AvailabilityId* = distinct array[32, byte]
Availability* = object
id*: AvailabilityId
size*: UInt256
duration*: UInt256
minPrice*: UInt256
maxCollateral*: UInt256
id* {.serialize.}: AvailabilityId
size* {.serialize.}: UInt256
duration* {.serialize.}: UInt256
minPrice* {.serialize.}: UInt256
maxCollateral* {.serialize.}: UInt256
used*: bool
Reservations* = ref object
repo: RepoStore

339
codex/utils/json.nim Normal file
View File

@ -0,0 +1,339 @@
import std/json except `%`, `%*`
import std/macros
import std/options
import std/strutils
import std/strformat
import std/tables
import std/typetraits
import pkg/chronicles
from pkg/libp2p import Cid, init
import pkg/contractabi
import pkg/stew/byteutils
import pkg/stint
import pkg/questionable/results
import ../errors
export json except `%`, `%*`
logScope:
topics = "json serialization"
type
SerializationError = object of CodexError
UnexpectedKindError = object of SerializationError
template serialize* {.pragma.}
proc newUnexpectedKindError(
expectedType: type,
expectedKinds: string,
json: JsonNode
): ref UnexpectedKindError =
let kind = if json.isNil: "nil"
else: $json.kind
newException(UnexpectedKindError,
&"deserialization to {$expectedType} failed: expected {expectedKinds} " &
&"but got {kind}")
proc newUnexpectedKindError(
expectedType: type,
expectedKinds: set[JsonNodeKind],
json: JsonNode
): ref UnexpectedKindError =
newUnexpectedKindError(expectedType, $expectedKinds, json)
proc newUnexpectedKindError(
expectedType: type,
expectedKind: JsonNodeKind,
json: JsonNode
): ref UnexpectedKindError =
newUnexpectedKindError(expectedType, {expectedKind}, json)
template expectJsonKind(
expectedType: type,
expectedKinds: set[JsonNodeKind],
json: JsonNode
) =
if json.isNil or json.kind notin expectedKinds:
return failure(newUnexpectedKindError(expectedType, expectedKinds, json))
template expectJsonKind(
expectedType: type,
expectedKind: JsonNodeKind,
json: JsonNode
) =
expectJsonKind(expectedType, {expectedKind}, json)
proc fromJson*(
_: type string,
json: JsonNode
): ?!string =
if json.isNil:
let err = newException(ValueError, "'json' expected, but was nil")
return failure(err)
elif json.kind == JNull:
return success("null")
elif json.isNil or json.kind != JString:
return failure(newUnexpectedKindError(string, JString, json))
catch json.getStr
proc fromJson*(
_: type bool,
json: JsonNode
): ?!bool =
expectJsonKind(bool, JBool, json)
catch json.getBool
proc fromJson*(
_: type int,
json: JsonNode
): ?!int =
expectJsonKind(int, JInt, json)
catch json.getInt
proc fromJson*[T: SomeInteger](
_: type T,
json: JsonNode
): ?!T =
when T is uint|uint64 or (not defined(js) and int.sizeof == 4):
expectJsonKind(T, {JInt, JString}, json)
case json.kind
of JString:
let x = parseBiggestUInt(json.str)
return success cast[T](x)
else:
return success T(json.num)
else:
expectJsonKind(T, {JInt}, json)
return success cast[T](json.num)
proc fromJson*[T: SomeFloat](
_: type T,
json: JsonNode
): ?!T =
expectJsonKind(T, {JInt, JFloat, JString}, json)
if json.kind == JString:
case json.str
of "nan":
let b = NaN
return success T(b)
# dst = NaN # would fail some tests because range conversions would cause CT error
# in some cases; but this is not a hot-spot inside this branch and backend can optimize this.
of "inf":
let b = Inf
return success T(b)
of "-inf":
let b = -Inf
return success T(b)
else:
let err = newUnexpectedKindError(T, "'nan|inf|-inf'", json)
return failure(err)
else:
if json.kind == JFloat:
return success T(json.fnum)
else:
return success T(json.num)
proc fromJson*(
_: type seq[byte],
json: JsonNode
): ?!seq[byte] =
expectJsonKind(seq[byte], JString, json)
hexToSeqByte(json.getStr).catch
proc fromJson*[N: static[int], T: array[N, byte]](
_: type T,
json: JsonNode
): ?!T =
expectJsonKind(T, JString, json)
T.fromHex(json.getStr).catch
proc fromJson*[T: distinct](
_: type T,
json: JsonNode
): ?!T =
success T(? T.distinctBase.fromJson(json))
proc fromJson*[N: static[int], T: StUint[N]](
_: type T,
json: JsonNode
): ?!T =
expectJsonKind(T, JString, json)
catch parse(json.getStr, T)
proc fromJson*[T](
_: type Option[T],
json: JsonNode
): ?! Option[T] =
if json.isNil or json.kind == JNull:
return success(none T)
without val =? T.fromJson(json), error:
return failure(error)
success(val.some)
proc fromJson*(
_: type Cid,
json: JsonNode
): ?!Cid =
expectJsonKind(Cid, JString, json)
Cid.init($json).mapFailure
proc fromJson*[T](
_: type seq[T],
json: JsonNode
): ?! seq[T] =
expectJsonKind(seq[T], JArray, json)
var arr: seq[T] = @[]
for elem in json.elems:
arr.add(? T.fromJson(elem))
success arr
proc fromJson*[T: object](
_: type T,
json: JsonNode
): ?!T =
expectJsonKind(T, JObject, json)
var res = T.default
# Leave this in, it's good for debugging:
# trace "deserializing object", to = $T, json
for name, value in fieldPairs(res):
if json{name} != nil:
without parsed =? type(value).fromJson(json{name}), e:
error "error deserializing field",
field = $T & "." & name,
json = json{name},
error = e.msg
return failure(e)
value = parsed
success(res)
proc fromJson*[T: ref object](
_: type T,
json: JsonNode
): ?!T =
expectJsonKind(T, JObject, json)
var res = T.new()
# Leave this in, it's good for debugging:
# trace "deserializing object", to = $T, json
for name, value in fieldPairs(res[]):
if json{name} != nil:
without parsed =? type(value).fromJson(json{name}), e:
error "error deserializing field",
field = $T & "." & name,
json = json{name},
error = e.msg
return failure(e)
value = parsed
success(res)
proc fromJson*[T: object](
_: type T,
bytes: seq[byte]
): ?!T =
let json = ?catch parseJson(string.fromBytes(bytes))
T.fromJson(json)
func `%`*(s: string): JsonNode = newJString(s)
func `%`*(n: uint): JsonNode =
if n > cast[uint](int.high):
newJString($n)
else:
newJInt(BiggestInt(n))
func `%`*(n: int): JsonNode = newJInt(n)
func `%`*(n: BiggestUInt): JsonNode =
if n > cast[BiggestUInt](BiggestInt.high):
newJString($n)
else:
newJInt(BiggestInt(n))
func `%`*(n: BiggestInt): JsonNode = newJInt(n)
func `%`*(n: float): JsonNode =
if n != n: newJString("nan")
elif n == Inf: newJString("inf")
elif n == -Inf: newJString("-inf")
else: newJFloat(n)
func `%`*(b: bool): JsonNode = newJBool(b)
func `%`*(keyVals: openArray[tuple[key: string, val: JsonNode]]): JsonNode =
if keyVals.len == 0: return newJArray()
let jObj = newJObject()
for key, val in items(keyVals): jObj.fields[key] = val
jObj
template `%`*(j: JsonNode): JsonNode = j
func `%`*[T](table: Table[string, T]|OrderedTable[string, T]): JsonNode =
let jObj = newJObject()
for k, v in table: jObj[k] = ? %v
jObj
func `%`*[T](opt: Option[T]): JsonNode =
if opt.isSome: %(opt.get) else: newJNull()
func `%`*[T: object](obj: T): JsonNode =
let jsonObj = newJObject()
for name, value in obj.fieldPairs:
when value.hasCustomPragma(serialize):
jsonObj[name] = %value
jsonObj
func `%`*[T: ref object](obj: T): JsonNode =
let jsonObj = newJObject()
for name, value in obj[].fieldPairs:
when value.hasCustomPragma(serialize):
jsonObj[name] = %(value)
jsonObj
proc `%`*(o: enum): JsonNode = % $o
func `%`*(stint: StInt|StUint): JsonNode = %stint.toString
func `%`*(cstr: cstring): JsonNode = % $cstr
func `%`*(arr: openArray[byte]): JsonNode = % arr.to0xHex
func `%`*[T](elements: openArray[T]): JsonNode =
let jObj = newJArray()
for elem in elements: jObj.add(%elem)
jObj
func `%`*[T: distinct](id: T): JsonNode =
type baseType = T.distinctBase
% baseType(id)
proc toJsnImpl(x: NimNode): NimNode =
case x.kind
of nnkBracket: # array
if x.len == 0: return newCall(bindSym"newJArray")
result = newNimNode(nnkBracket)
for i in 0 ..< x.len:
result.add(toJsnImpl(x[i]))
result = newCall(bindSym("%", brOpen), result)
of nnkTableConstr: # object
if x.len == 0: return newCall(bindSym"newJObject")
result = newNimNode(nnkTableConstr)
for i in 0 ..< x.len:
x[i].expectKind nnkExprColonExpr
result.add newTree(nnkExprColonExpr, x[i][0], toJsnImpl(x[i][1]))
result = newCall(bindSym("%", brOpen), result)
of nnkCurly: # empty object
x.expectLen(0)
result = newCall(bindSym"newJObject")
of nnkNilLit:
result = newCall(bindSym"newJNull")
of nnkPar:
if x.len == 1: result = toJsnImpl(x[0])
else: result = newCall(bindSym("%", brOpen), x)
else:
result = newCall(bindSym("%", brOpen), x)
macro `%*`*(x: untyped): JsonNode =
## Convert an expression to a JsonNode directly, without having to specify
## `%` for every element.
result = toJsnImpl(x)

View File

@ -1,3 +1,4 @@
import ./utils/testjson
import ./utils/testoptionalcast
import ./utils/testkeyutils
import ./utils/testasyncstatemachine

View File

@ -0,0 +1,347 @@
import std/math
import std/options
import std/strformat
import std/strutils
import std/unittest
import pkg/chronicles
import pkg/stew/byteutils
import pkg/stint
import pkg/codex/contracts/requests
from pkg/codex/rest/json import RestPurchase
import pkg/codex/utils/json as utilsjson
import pkg/questionable
import pkg/questionable/results
import ../examples
import ../helpers
checksuite "json serialization":
var request: StorageRequest
var requestJson: JsonNode
func flatten(s: string): string =
s.replace(" ")
.replace("\n")
setup:
request = StorageRequest(
client: Address.init("0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174").get(),
ask: StorageAsk(
slots: 4,
slotSize: (1 * 1024 * 1024 * 1024).u256, # 1 Gigabyte
duration: (10 * 60 * 60).u256, # 10 hours
collateral: 200.u256,
proofProbability: 4.u256, # require a proof roughly once every 4 periods
reward: 84.u256,
maxSlotLoss: 2 # 2 slots can be freed without data considered to be lost
),
content: StorageContent(
cid: "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob",
erasure: StorageErasure(
totalChunks: 12,
),
por: StoragePoR(
u: @(array[480, byte].fromHex("0xc066dd7e405de5a795ce1e765209cfaa6de6c74829c607d1a2fe53107107a2981baccff95e2fc2e38d08303e8ab59a1169cfc9bfbfa2294c6df065456056a0d0106f66d6fe48300222758fd6fd286a1ac9060d1a295e19f931a8d3ad2c47eb131bea267fe942d460fda96fd4bf663148cd90fbb1b670dd97aae70394248b75cfbb98f71e8c69f50381e0558884d5d9aa75147923b55386c66f75f63024b698eeb0ff994bfdb610eea1b7c75e87bdb54843071bc64fbaf93e5dc214e875bd95bd30f167b69df1de819a34cc71a3a0465f5c1d1b7e5b98de6017ff3e3c059536f974471fe62e0f224eba8f96352a8ee51befbf4c31c88ad0fc8ff4e9d9da174a455a1c30fd61ac977145d3677a08167d508fae207f9458a9b19d4ceec2be30506e2d70cc0362c2bcdb0f73d63fa5e79f9b2901bc870ac8b2a264d50e1862ea177eb587bcd16ceb7d66f96f198cadec3f644af4d3cbe478bc1665818401f89107053d1750047fb7cfc47938bec2cd006db9c176ce337e41160077e353f87ab319e5b9df92282916ef99334c067f6ca20c3d7cbc12b95180b7bba762993a4dbdf4242032da8865988183738d279918906c3357701d74e5d8f5142315ae8f6d0f93537abc3545118e953f983317657a9d8b86e4305ea49e10f80ea07dc7ea7321b32c")),
publicKey: @(array[96, byte].fromHex("0xb231b19de641f678d250623b2b76099ab4bbd67aac19dcf42ded946831e3366d2a20af0fd9e841197e7e64d7639da4518b76c353db480087e21d55b470f24a180d6d6c8265bf3895e2e4e4e54b8ca9334d62b22feeeed8e77e54bfbc8fae6b62")),
name: @(array[512, byte].fromHex("0x75b2ac401efd21e60e84a69288da6fff28c7badaae885e417f35055a4e10cb514855f68a0ae18bf42861426c9fc34af13df2f2d04dc68933af78bf3fc396953f301b95f6d6af54ec9fc871c292096e45b91e836063f128c2d1469adbee49bc9b7d62985a858801e4df2cb77eb41ee7b50a8a4e5afb5b585f9034a2808f81bd95b9a3fbdd2579331023f1816a1ecbe7a31e386721a72e3d0ff6087326fba8442dfd22d1182c85906d796e697231c2d7d4a888ae256c79a9019974a4c729d981f3e554f48895e27fe8f45da46bc48c35cc74ae5a31dfea8baa1334fa7f106cdc4ec54452f39c823fa0af97769217cc16c78eb7d0c494c26d2f286f09a507bd04cb15963270bffefb28258176d9e10b7aaad76cdd86e0fe49437eb83c1c0650cb5920e32dc54f3a21a70308b7312b47ce57ef72c2c19eba5027612128b747e80b88c912d7fc10177e67beda0ed5bb8fdfc268bfa5a5c700da953c56bcc79b9186da99ee19a6fa954f44bdcbc7c7f4d208fb750bad587d5513fbaccd511b9b6e0cd798120de87b9c0c410b3b85c75a8a0f469d9973a1ec4c86982cf4fe1a2be21a9206aaabb1ad2fafa628d5156d2ec99ee30fc0ddb9dca6a4cd3a7987227315ceeaa832909853cabaf33c976b59cf5ed9643781d92ab769c0d0aa3bcef40b41b4b3e6fc5a00c9dfbf794047d9cfb97d9d669d00520b6492760a08dba65b0fd7e6d0"))
)
),
expiry: 1691545330.u256,
nonce: Nonce array[32, byte].fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1")
)
requestJson = """{
"client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174",
"ask": {
"slots": 4,
"slotSize": "1073741824",
"duration": "36000",
"proofProbability": "4",
"reward": "84",
"collateral": "200",
"maxSlotLoss": 2
},
"content": {
"cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob",
"erasure": {
"totalChunks": 12
},
"por": {
"u": "0xc066dd7e405de5a795ce1e765209cfaa6de6c74829c607d1a2fe53107107a2981baccff95e2fc2e38d08303e8ab59a1169cfc9bfbfa2294c6df065456056a0d0106f66d6fe48300222758fd6fd286a1ac9060d1a295e19f931a8d3ad2c47eb131bea267fe942d460fda96fd4bf663148cd90fbb1b670dd97aae70394248b75cfbb98f71e8c69f50381e0558884d5d9aa75147923b55386c66f75f63024b698eeb0ff994bfdb610eea1b7c75e87bdb54843071bc64fbaf93e5dc214e875bd95bd30f167b69df1de819a34cc71a3a0465f5c1d1b7e5b98de6017ff3e3c059536f974471fe62e0f224eba8f96352a8ee51befbf4c31c88ad0fc8ff4e9d9da174a455a1c30fd61ac977145d3677a08167d508fae207f9458a9b19d4ceec2be30506e2d70cc0362c2bcdb0f73d63fa5e79f9b2901bc870ac8b2a264d50e1862ea177eb587bcd16ceb7d66f96f198cadec3f644af4d3cbe478bc1665818401f89107053d1750047fb7cfc47938bec2cd006db9c176ce337e41160077e353f87ab319e5b9df92282916ef99334c067f6ca20c3d7cbc12b95180b7bba762993a4dbdf4242032da8865988183738d279918906c3357701d74e5d8f5142315ae8f6d0f93537abc3545118e953f983317657a9d8b86e4305ea49e10f80ea07dc7ea7321b32c",
"publicKey": "0xb231b19de641f678d250623b2b76099ab4bbd67aac19dcf42ded946831e3366d2a20af0fd9e841197e7e64d7639da4518b76c353db480087e21d55b470f24a180d6d6c8265bf3895e2e4e4e54b8ca9334d62b22feeeed8e77e54bfbc8fae6b62",
"name": "0x75b2ac401efd21e60e84a69288da6fff28c7badaae885e417f35055a4e10cb514855f68a0ae18bf42861426c9fc34af13df2f2d04dc68933af78bf3fc396953f301b95f6d6af54ec9fc871c292096e45b91e836063f128c2d1469adbee49bc9b7d62985a858801e4df2cb77eb41ee7b50a8a4e5afb5b585f9034a2808f81bd95b9a3fbdd2579331023f1816a1ecbe7a31e386721a72e3d0ff6087326fba8442dfd22d1182c85906d796e697231c2d7d4a888ae256c79a9019974a4c729d981f3e554f48895e27fe8f45da46bc48c35cc74ae5a31dfea8baa1334fa7f106cdc4ec54452f39c823fa0af97769217cc16c78eb7d0c494c26d2f286f09a507bd04cb15963270bffefb28258176d9e10b7aaad76cdd86e0fe49437eb83c1c0650cb5920e32dc54f3a21a70308b7312b47ce57ef72c2c19eba5027612128b747e80b88c912d7fc10177e67beda0ed5bb8fdfc268bfa5a5c700da953c56bcc79b9186da99ee19a6fa954f44bdcbc7c7f4d208fb750bad587d5513fbaccd511b9b6e0cd798120de87b9c0c410b3b85c75a8a0f469d9973a1ec4c86982cf4fe1a2be21a9206aaabb1ad2fafa628d5156d2ec99ee30fc0ddb9dca6a4cd3a7987227315ceeaa832909853cabaf33c976b59cf5ed9643781d92ab769c0d0aa3bcef40b41b4b3e6fc5a00c9dfbf794047d9cfb97d9d669d00520b6492760a08dba65b0fd7e6d0"
}
},
"expiry": "1691545330",
"nonce": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"
}""".parseJson
test "serializes UInt256 to non-hex string representation":
check (% 100000.u256) == newJString("100000")
test "serializes sequence to an array":
let json = % @[1, 2, 3]
let expected = "[1,2,3]"
check $json == expected
test "serializes Option[T] when has a value":
let obj = %(some 1)
let expected = "1"
check $obj == expected
test "serializes Option[T] when doesn't have a value":
let obj = %(none int)
let expected = "null"
check $obj == expected
test "serializes uints int.high or smaller":
let largeUInt: uint = uint(int.high)
check %largeUInt == newJInt(BiggestInt(largeUInt))
test "serializes large uints":
let largeUInt: uint = uint(int.high) + 1'u
check %largeUInt == newJString($largeUInt)
test "serializes Inf float":
check %Inf == newJString("inf")
test "serializes -Inf float":
check %(-Inf) == newJString("-inf")
test "deserializes NaN float":
check %NaN == newJString("nan")
test "can construct json objects with %*":
type MyObj = object
mystring {.serialize.}: string
myint {.serialize.}: int
myoption {.serialize.}: ?bool
let myobj = MyObj(mystring: "abc", myint: 123, myoption: some true)
let mystuint = 100000.u256
let json = %*{
"myobj": myobj,
"mystuint": mystuint
}
let expected = """{
"myobj": {
"mystring": "abc",
"myint": 123,
"myoption": true
},
"mystuint": "100000"
}""".flatten
check $json == expected
test "only serializes marked fields":
type MyObj = object
mystring {.serialize.}: string
myint {.serialize.}: int
mybool: bool
let obj = % MyObj(mystring: "abc", myint: 1, mybool: true)
let expected = """{
"mystring": "abc",
"myint": 1
}""".flatten
check $obj == expected
test "serializes ref objects":
type MyRef = ref object
mystring {.serialize.}: string
myint {.serialize.}: int
let obj = % MyRef(mystring: "abc", myint: 1)
let expected = """{
"mystring": "abc",
"myint": 1
}""".flatten
check $obj == expected
test "serializes RestPurchase":
let request = % RestPurchase(
request: some request,
requestId: RequestId.fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"),
error: some "error",
state: "state"
)
let expected = """{
"requestId": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1",
"request": {
"client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174",
"ask": {
"slots": 4,
"slotSize": "1073741824",
"duration": "36000",
"proofProbability": "4",
"reward": "84",
"collateral": "200",
"maxSlotLoss": 2
},
"content": {
"cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob"
},
"expiry": "1691545330"
},
"state": "state",
"error": "error"
}""".flatten
check $request == expected
test "serializes StorageRequest":
let expected = """{
"client": "0xebcb2b4c2e3c9105b1a53cd128c5ed17c3195174",
"ask": {
"slots": 4,
"slotSize": "1073741824",
"duration": "36000",
"proofProbability": "4",
"reward": "84",
"collateral": "200",
"maxSlotLoss": 2
},
"content": {
"cid": "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob"
},
"expiry": "1691545330"
}""".flatten
check $(%request) == expected
test "deserializes UInt256 from non-hex string representation":
let json = newJString("100000")
check !UInt256.fromJson(json) == 100000.u256
test "deserializes Option[T] when has a value":
let json = newJInt(1)
check (!fromJson(?int, json) == some 1)
test "deserializes Option[T] when doesn't have a value":
let json = newJNull()
check !fromJson(?int, json) == none int
test "deserializes float":
let json = newJFloat(1.234)
check !float.fromJson(json) == 1.234
test "deserializes Inf float":
let json = newJString("inf")
check !float.fromJson(json) == Inf
test "deserializes -Inf float":
let json = newJString("-inf")
check !float.fromJson(json) == -Inf
test "deserializes NaN float":
let json = newJString("nan")
check float.fromJson(json).get.isNaN
test "deserializes array to sequence":
let expected = @[1, 2, 3]
let json = "[1,2,3]".parseJson
check !seq[int].fromJson(json) == expected
test "deserializes uints int.high or smaller":
let largeUInt: uint = uint(int.high)
let json = newJInt(BiggestInt(largeUInt))
check !uint.fromJson(json) == largeUInt
test "deserializes large uints":
let largeUInt: uint = uint(int.high) + 1'u
let json = newJString($BiggestUInt(largeUInt))
check !uint.fromJson(json) == largeUInt
test "can deserialize json objects":
type MyObj = object
mystring: string
myint: int
myoption: ?bool
let expected = MyObj(mystring: "abc", myint: 123, myoption: some true)
let json = parseJson("""{
"mystring": "abc",
"myint": 123,
"myoption": true
}""")
check !MyObj.fromJson(json) == expected
test "ignores serialize pragma when deserializing":
type MyObj = object
mystring {.serialize.}: string
mybool: bool
let expected = MyObj(mystring: "abc", mybool: true)
let json = parseJson("""{
"mystring": "abc",
"mybool": true
}""")
check !MyObj.fromJson(json) == expected
test "deserializes objects with extra fields":
type MyObj = object
mystring: string
mybool: bool
let expected = MyObj(mystring: "abc", mybool: true)
let json = """{
"mystring": "abc",
"mybool": true,
"extra": "extra"
}""".parseJson
check !MyObj.fromJson(json) == expected
test "deserializes objects with less fields":
type MyObj = object
mystring: string
mybool: bool
let expected = MyObj(mystring: "abc", mybool: false)
let json = """{
"mystring": "abc"
}""".parseJson
check !MyObj.fromJson(json) == expected
test "deserializes ref objects":
type MyRef = ref object
mystring: string
myint: int
let expected = MyRef(mystring: "abc", myint: 1)
let json = """{
"mystring": "abc",
"myint": 1
}""".parseJson
let deserialized = !MyRef.fromJson(json)
check deserialized.mystring == expected.mystring
check deserialized.myint == expected.myint
test "deserializes StorageRequest":
check !StorageRequest.fromJson(requestJson) == request
test "deserializes RestPurchase":
let json = """{
"requestId": "0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1",
"state": "state",
"error": "error"
}""".parseJson
json["request"] = requestJson
let expected = RestPurchase(
requestId: RequestId.fromHex("0xd4ebeadc44641c0a271153f6366f24ebb5e3aa64f9ee5e62794babc2e75950a1"),
state: "state",
error: some "error",
request: some request
)
check !RestPurchase.fromJson(json) == expected

View File

@ -1,8 +1,13 @@
import std/httpclient
import std/json
import std/strutils
from pkg/libp2p import Cid, `$`, init
import pkg/chronicles
import pkg/stint
import pkg/questionable/results
import pkg/codex/rest/json
import pkg/codex/purchasing
import pkg/codex/errors
import pkg/codex/sales/reservations
type CodexClient* = ref object
http: HttpClient
@ -21,38 +26,43 @@ proc setLogLevel*(client: CodexClient, level: string) =
let response = client.http.request(url, httpMethod=HttpPost, headers=headers)
assert response.status == "200 OK"
proc upload*(client: CodexClient, contents: string): string =
proc upload*(client: CodexClient, contents: string): ?!Cid =
let response = client.http.post(client.baseurl & "/upload", contents)
assert response.status == "200 OK"
response.body
Cid.init(response.body).mapFailure
proc requestStorage*(
client: CodexClient,
cid: string,
duration: uint64,
reward: uint64,
proofProbability: uint64,
cid: Cid,
duration: UInt256,
reward: UInt256,
proofProbability: UInt256,
expiry: UInt256,
collateral: uint64
): string =
collateral: UInt256,
nodes: uint = 1,
tolerance: uint = 0
): ?!PurchaseId =
## Call request storage REST endpoint
##
let url = client.baseurl & "/storage/request/" & cid
let url = client.baseurl & "/storage/request/" & $cid
let json = %*{
"duration": $duration,
"reward": $reward,
"proofProbability": $proofProbability,
"expiry": $expiry,
"collateral": $collateral,
"duration": duration,
"reward": reward,
"proofProbability": proofProbability,
"expiry": expiry,
"collateral": collateral,
"nodes": nodes,
"tolerance": tolerance
}
let response = client.http.post(url, $json)
assert response.status == "200 OK"
response.body
PurchaseId.fromHex(response.body).catch
proc getPurchase*(client: CodexClient, purchase: string): JsonNode =
let url = client.baseurl & "/storage/purchases/" & purchase
proc getPurchase*(client: CodexClient, purchaseId: PurchaseId): ?!RestPurchase =
let url = client.baseurl & "/storage/purchases/" & purchaseId.toHex
let body = client.http.getContent(url)
parseJson(body).catch |? nil
let json = ? parseJson(body).catch
RestPurchase.fromJson(json)
proc getSlots*(client: CodexClient): JsonNode =
let url = client.baseurl & "/sales/slots"
@ -61,27 +71,26 @@ proc getSlots*(client: CodexClient): JsonNode =
proc postAvailability*(
client: CodexClient,
size, duration, minPrice: uint64,
maxCollateral: uint64
): JsonNode =
size, duration, minPrice, maxCollateral: UInt256
): ?!Availability =
## Post sales availability endpoint
##
let url = client.baseurl & "/sales/availability"
let json = %*{
"size": $size,
"duration": $duration,
"minPrice": $minPrice,
"maxCollateral": $maxCollateral,
"size": size,
"duration": duration,
"minPrice": minPrice,
"maxCollateral": maxCollateral,
}
let response = client.http.post(url, $json)
assert response.status == "200 OK"
parseJson(response.body)
Availability.fromJson(response.body.parseJson)
proc getAvailabilities*(client: CodexClient): JsonNode =
proc getAvailabilities*(client: CodexClient): ?!seq[Availability] =
## Call sales availability REST endpoint
let url = client.baseurl & "/sales/availability"
let body = client.http.getContent(url)
parseJson(body)
seq[Availability].fromJson(parseJson(body))
proc close*(client: CodexClient) =
client.http.close()

View File

@ -1,4 +1,5 @@
import std/json
import std/options
from pkg/libp2p import `==`
import pkg/chronos
import pkg/stint
import pkg/ethers/erc20
@ -9,11 +10,16 @@ import ../contracts/deployment
import ../codex/helpers/eventually
import ./twonodes
# For debugging you can enable logging output with debugX = true
# You can also pass a string in same format like for the `--log-level` parameter
# to enable custom logging levels for specific topics like: debug2 = "INFO; TRACE: marketplace"
twonodessuite "Integration tests", debug1 = false, debug2 = false:
proc purchaseStateIs(client: CodexClient, id: PurchaseId, state: string): bool =
client.getPurchase(id).option.?state == some state
setup:
# Our Hardhat configuration does use automine, which means that time tracked by `provider.currentTime()` is not
# advanced until blocks are mined and that happens only when transaction is submitted.
@ -27,87 +33,105 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
client1.setLogLevel("DEBUG;TRACE:codex")
test "node accepts file uploads":
let cid1 = client1.upload("some file contents")
let cid2 = client1.upload("some other contents")
let cid1 = client1.upload("some file contents").get
let cid2 = client1.upload("some other contents").get
check cid1 != cid2
test "node handles new storage availability":
let availability1 = client1.postAvailability(size=1, duration=2, minPrice=3, maxCollateral=4)
let availability2 = client1.postAvailability(size=4, duration=5, minPrice=6, maxCollateral=7)
let availability1 = client1.postAvailability(size=1.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get
let availability2 = client1.postAvailability(size=4.u256, duration=5.u256, minPrice=6.u256, maxCollateral=7.u256).get
check availability1 != availability2
test "node lists storage that is for sale":
let availability = client1.postAvailability(size=1, duration=2, minPrice=3, maxCollateral=4)
check availability in client1.getAvailabilities()
let availability = client1.postAvailability(size=1.u256, duration=2.u256, minPrice=3.u256, maxCollateral=4.u256).get
check availability in client1.getAvailabilities().get
test "node handles storage request":
let expiry = (await provider.currentTime()) + 30
let cid = client1.upload("some file contents")
let id1 = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry, collateral=200)
let id2 = client1.requestStorage(cid, duration=4, reward=5, proofProbability=6, expiry=expiry, collateral=201)
let cid = client1.upload("some file contents").get
let id1 = client1.requestStorage(cid, duration=1.u256, reward=2.u256, proofProbability=3.u256, expiry=expiry, collateral=200.u256).get
let id2 = client1.requestStorage(cid, duration=4.u256, reward=5.u256, proofProbability=6.u256, expiry=expiry, collateral=201.u256).get
check id1 != id2
test "node retrieves purchase status":
let expiry = (await provider.currentTime()) + 30
let cid = client1.upload("some file contents")
let id = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry, collateral=200)
let purchase = client1.getPurchase(id)
check purchase{"request"}{"ask"}{"duration"} == %"1"
check purchase{"request"}{"ask"}{"reward"} == %"2"
check purchase{"request"}{"ask"}{"proofProbability"} == %"3"
let cid = client1.upload("some file contents").get
let id = client1.requestStorage(cid, duration=1.u256, reward=2.u256, proofProbability=3.u256, expiry=expiry, collateral=200.u256, nodes=2, tolerance=1).get
let request = client1.getPurchase(id).get.request.get
check request.ask.duration == 1.u256
check request.ask.reward == 2.u256
check request.ask.proofProbability == 3.u256
check request.expiry == expiry
check request.ask.collateral == 200.u256
check request.ask.slots == 3'u64
check request.ask.maxSlotLoss == 1'u64
test "node remembers purchase status after restart":
let expiry = (await provider.currentTime()) + 30
let cid = client1.upload("some file contents")
let id = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry, collateral=200)
check eventually client1.getPurchase(id){"state"}.getStr() == "submitted"
let cid = client1.upload("some file contents").get
let id = client1.requestStorage(cid,
duration=1.u256,
reward=2.u256,
proofProbability=3.u256,
expiry=expiry,
collateral=200.u256).get
check eventually client1.purchaseStateIs(id, "submitted")
node1.restart()
client1.restart()
check eventually (not isNil client1.getPurchase(id){"request"}{"ask"})
check client1.getPurchase(id){"request"}{"ask"}{"duration"} == %"1"
check client1.getPurchase(id){"request"}{"ask"}{"reward"} == %"2"
check eventually client1.purchaseStateIs(id, "submitted")
let request = client1.getPurchase(id).get.request.get
check request.ask.duration == 1.u256
check request.ask.reward == 2.u256
check request.ask.proofProbability == 3.u256
check request.expiry == expiry
check request.ask.collateral == 200.u256
check request.ask.slots == 1'u64
check request.ask.maxSlotLoss == 0'u64
test "nodes negotiate contracts on the marketplace":
let size: uint64 = 0xFFFFF
let size = 0xFFFFF.u256
# client 2 makes storage available
discard client2.postAvailability(size=size, duration=200, minPrice=300, maxCollateral=300)
discard client2.postAvailability(size=size, duration=200.u256, minPrice=300.u256, maxCollateral=300.u256)
# client 1 requests storage
let expiry = (await provider.currentTime()) + 30
let cid = client1.upload("some file contents")
let purchase = client1.requestStorage(cid, duration=100, reward=400, proofProbability=3, expiry=expiry, collateral=200)
let cid = client1.upload("some file contents").get
let id = client1.requestStorage(cid, duration=100.u256, reward=400.u256, proofProbability=3.u256, expiry=expiry, collateral=200.u256).get
check eventually client1.getPurchase(purchase){"state"} == %"started"
check client1.getPurchase(purchase){"error"} == newJNull()
let availabilities = client2.getAvailabilities()
check eventually client1.purchaseStateIs(id, "started")
let purchase = client1.getPurchase(id).get
check purchase.error == none string
let availabilities = client2.getAvailabilities().get
check availabilities.len == 1
let newSize = UInt256.fromDecimal(availabilities[0]{"size"}.getStr)
check newSize > 0 and newSize < size.u256
let newSize = availabilities[0].size
check newSize > 0 and newSize < size
test "node slots gets paid out":
let marketplace = Marketplace.new(Marketplace.address, provider.getSigner())
let tokenAddress = await marketplace.token()
let token = Erc20Token.new(tokenAddress, provider.getSigner())
let reward: uint64 = 400
let duration: uint64 = 100
let reward = 400.u256
let duration = 100.u256
# client 2 makes storage available
let startBalance = await token.balanceOf(account2)
discard client2.postAvailability(size=0xFFFFF, duration=200, minPrice=300, maxCollateral=300)
discard client2.postAvailability(size=0xFFFFF.u256, duration=200.u256, minPrice=300.u256, maxCollateral=300.u256).get
# client 1 requests storage
let expiry = (await provider.currentTime()) + 30
let cid = client1.upload("some file contents")
let purchase = client1.requestStorage(cid, duration=duration, reward=reward, proofProbability=3, expiry=expiry, collateral=200)
let cid = client1.upload("some file contents").get
let id = client1.requestStorage(cid, duration=duration, reward=reward, proofProbability=3.u256, expiry=expiry, collateral=200.u256).get
check eventually client1.getPurchase(purchase){"state"} == %"started"
check client1.getPurchase(purchase){"error"} == newJNull()
check eventually client1.purchaseStateIs(id, "started")
let purchase = client1.getPurchase(id).get
check purchase.error == none string
# Proving mechanism uses blockchain clock to do proving/collect/cleanup round
# hence we must use `advanceTime` over `sleepAsync` as Hardhat does mine new blocks
# only with new transaction
await provider.advanceTime(duration.u256)
await provider.advanceTime(duration)
check eventually (await token.balanceOf(account2)) - startBalance == duration.u256*reward.u256
check eventually (await token.balanceOf(account2)) - startBalance == duration*reward

View File

@ -2,7 +2,7 @@ import std/sequtils
import std/os
from std/times import getTime, toUnix
import pkg/chronicles
import codex/contracts/marketplace
import codex/contracts
import codex/periods
import ../contracts/time
import ../contracts/deployment
@ -19,6 +19,9 @@ twonodessuite "Proving integration test", debug1=false, debug2=false:
var marketplace: Marketplace
var period: uint64
proc purchaseStateIs(client: CodexClient, id: PurchaseId, state: string): bool =
client.getPurchase(id).option.?state == some state
setup:
marketplace = Marketplace.new(Marketplace.address, provider)
period = (await marketplace.config()).proofs.period.truncate(uint64)
@ -32,22 +35,22 @@ twonodessuite "Proving integration test", debug1=false, debug2=false:
duration: uint64 = 100 * period,
expiry: uint64 = 30) {.async.} =
discard client2.postAvailability(
size=0xFFFFF,
duration=duration,
minPrice=300,
maxCollateral=200
size=0xFFFFF.u256,
duration=duration.u256,
minPrice=300.u256,
maxCollateral=200.u256
)
let cid = client1.upload("some file contents")
let cid = client1.upload("some file contents").get
let expiry = (await provider.currentTime()) + expiry.u256
let purchase = client1.requestStorage(
let id = client1.requestStorage(
cid,
expiry=expiry,
duration=duration,
proofProbability=proofProbability,
collateral=100,
reward=400
)
check eventually client1.getPurchase(purchase){"state"} == %"started"
duration=duration.u256,
proofProbability=proofProbability.u256,
collateral=100.u256,
reward=400.u256
).get
check eventually client1.purchaseStateIs(id, "started")
proc advanceToNextPeriod {.async.} =
let periodicity = Periodicity(seconds: period.u256)
@ -105,6 +108,9 @@ multinodesuite "Simulate invalid proofs",
StartNodes.init(clients=1'u, providers=0'u, validators=1'u),
DebugNodes.init(client=false, provider=false, validator=false):
proc purchaseStateIs(client: CodexClient, id: PurchaseId, state: string): bool =
client.getPurchase(id).option.?state == some state
var marketplace: Marketplace
var period: uint64
var slotId: SlotId
@ -142,26 +148,26 @@ multinodesuite "Simulate invalid proofs",
let storageProvider = providers()[0].restClient
discard storageProvider.postAvailability(
size=0xFFFFF,
duration=duration,
minPrice=300,
maxCollateral=200
size=0xFFFFF.u256,
duration=duration.u256,
minPrice=300.u256,
maxCollateral=200.u256
)
let cid = client.upload("some file contents " & $ getTime().toUnix)
let cid = client.upload("some file contents " & $ getTime().toUnix).get
let expiry = (await provider.currentTime()) + expiry.u256
# avoid timing issues by filling the slot at the start of the next period
await advanceToNextPeriod()
let purchase = client.requestStorage(
let id = client.requestStorage(
cid,
expiry=expiry,
duration=duration,
proofProbability=proofProbability,
collateral=100,
reward=400
)
check eventually client.getPurchase(purchase){"state"} == %"started"
let requestId = RequestId.fromHex client.getPurchase(purchase){"requestId"}.getStr
slotId = slotId(requestId, 0.u256)
duration=duration.u256,
proofProbability=proofProbability.u256,
collateral=100.u256,
reward=400.u256
).get
check eventually client.purchaseStateIs(id, "started")
let purchase = client.getPurchase(id).get
slotId = slotId(purchase.requestId, 0.u256)
# TODO: these are very loose tests in that they are not testing EXACTLY how
# proofs were marked as missed by the validator. These tests should be