diff --git a/codex/clock.nim b/codex/clock.nim index f680ddec..b9ed4595 100644 --- a/codex/clock.nim +++ b/codex/clock.nim @@ -1,5 +1,8 @@ import pkg/chronos +import pkg/questionable +import pkg/questionable/results import pkg/stew/endians2 +import pkg/stint import pkg/upraises import pkg/stint @@ -11,6 +14,9 @@ type method now*(clock: Clock): SecondsSince1970 {.base, upraises: [].} = raiseAssert "not implemented" +method lastBlockTimestamp*(clock: Clock): Future[UInt256] {.base, async.} = + raiseAssert "not implemented" + method waitUntil*(clock: Clock, time: SecondsSince1970) {.base, async.} = raiseAssert "not implemented" diff --git a/codex/contracts/clock.nim b/codex/contracts/clock.nim index 23b678e9..315bbe78 100644 --- a/codex/contracts/clock.nim +++ b/codex/contracts/clock.nim @@ -6,7 +6,11 @@ import ../clock export clock +logScope: + topics = "contracts clock" + type + LastBlockUnknownError* = object of CatchableError OnChainClock* = ref object of Clock provider: Provider subscription: Subscription @@ -20,12 +24,14 @@ proc new*(_: type OnChainClock, provider: Provider): OnChainClock = method start*(clock: OnChainClock) {.async.} = if clock.started: return + trace "starting on chain clock" clock.started = true proc onBlock(blck: Block) {.upraises:[].} = let blockTime = initTime(blck.timestamp.truncate(int64), 0) let computerTime = getTime() clock.offset = blockTime - computerTime + trace "new block received, updated clock offset", blockTime, computerTime, offset = clock.offset clock.newBlock.fire() if latestBlock =? (await clock.provider.getBlock(BlockTag.latest)): @@ -36,6 +42,7 @@ method start*(clock: OnChainClock) {.async.} = method stop*(clock: OnChainClock) {.async.} = if not clock.started: return + trace "stopping on chain clock" clock.started = false await clock.subscription.unsubscribe() @@ -44,7 +51,15 @@ method now*(clock: OnChainClock): SecondsSince1970 = doAssert clock.started, "clock should be started before calling now()" toUnix(getTime() + clock.offset) +method lastBlockTimestamp*(clock: OnChainClock): Future[UInt256] {.async.} = + without blk =? await clock.provider.getBlock(BlockTag.latest): + raise newException(LastBlockUnknownError, "failed to get last block") + + return blk.timestamp + method waitUntil*(clock: OnChainClock, time: SecondsSince1970) {.async.} = - while (let difference = time - clock.now(); difference > 0): + trace "waiting until", time + while (let difference = time - (await clock.lastBlockTimestamp).truncate(int64); difference > 0): clock.newBlock.clear() discard await clock.newBlock.wait().withTimeout(chronos.seconds(difference)) + trace "waiting for time unblocked" diff --git a/codex/contracts/interactions/clientinteractions.nim b/codex/contracts/interactions/clientinteractions.nim index c30a22e4..e2354d85 100644 --- a/codex/contracts/interactions/clientinteractions.nim +++ b/codex/contracts/interactions/clientinteractions.nim @@ -7,7 +7,7 @@ import ../clock import ./interactions export purchasing -export chronicles +export chronicles except toJson type ClientInteractions* = ref object of ContractInteractions diff --git a/codex/contracts/interactions/hostinteractions.nim b/codex/contracts/interactions/hostinteractions.nim index bfb9ac9c..e9749df5 100644 --- a/codex/contracts/interactions/hostinteractions.nim +++ b/codex/contracts/interactions/hostinteractions.nim @@ -5,7 +5,7 @@ import ../../sales import ./interactions export sales -export chronicles +export chronicles except toJson type HostInteractions* = ref object of ContractInteractions diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index c2c7b4e6..6beaa923 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -51,6 +51,17 @@ method proofTimeout*(market: OnChainMarket): Future[UInt256] {.async.} = let config = await market.contract.config() return config.proofs.timeout +method proofDowntime*(market: OnChainMarket): Future[uint8] {.async.} = + let config = await market.contract.config() + return config.proofs.downtime + +method getPointer*(market: OnChainMarket, slotId: SlotId): Future[uint8] {.async.} = + return await market.contract.getPointer(slotId) + +method currentBlockchainTime*(market: OnChainMarket): Future[UInt256] {.async.} = + let provider = market.contract.provider + return (!await provider.getBlock(BlockTag.latest)).timestamp + method myRequests*(market: OnChainMarket): Future[seq[RequestId]] {.async.} = return await market.contract.myRequests diff --git a/codex/contracts/marketplace.nim b/codex/contracts/marketplace.nim index 47b1a3d2..f8951b7b 100644 --- a/codex/contracts/marketplace.nim +++ b/codex/contracts/marketplace.nim @@ -8,8 +8,8 @@ import ./requests import ./config export stint -export ethers -export erc20 +export ethers except `%`, `%*`, toJson +export erc20 except `%`, `%*`, toJson export config export requests @@ -64,6 +64,7 @@ proc isProofRequired*(marketplace: Marketplace, id: SlotId): bool {.contract, vi proc willProofBeRequired*(marketplace: Marketplace, id: SlotId): bool {.contract, view.} proc getChallenge*(marketplace: Marketplace, id: SlotId): array[32, byte] {.contract, view.} proc getPointer*(marketplace: Marketplace, id: SlotId): uint8 {.contract, view.} +proc inDowntime*(marketplace: Marketplace, id: SlotId): bool {.contract, view.} proc submitProof*(marketplace: Marketplace, id: SlotId, proof: seq[byte]) {.contract.} proc markProofAsMissing*(marketplace: Marketplace, id: SlotId, period: UInt256) {.contract.} diff --git a/codex/contracts/requests.nim b/codex/contracts/requests.nim index 1abfcd7e..60c3c577 100644 --- a/codex/contracts/requests.nim +++ b/codex/contracts/requests.nim @@ -75,6 +75,10 @@ proc fromHex*[T: distinct](_: type T, hex: string): T = type baseType = T.distinctBase T baseType.fromHex(hex) +proc toHex*[T: distinct](id: T): string = + type baseType = T.distinctBase + baseType(id).toHex + func fromTuple(_: type StorageRequest, tupl: tuple): StorageRequest = StorageRequest( client: tupl[0], diff --git a/codex/market.nim b/codex/market.nim index be0d06fc..2e8ac8a7 100644 --- a/codex/market.nim +++ b/codex/market.nim @@ -38,6 +38,17 @@ method periodicity*(market: Market): Future[Periodicity] {.base, async.} = method proofTimeout*(market: Market): Future[UInt256] {.base, async.} = raiseAssert("not implemented") +method proofDowntime*(market: Market): Future[uint8] {.base, async.} = + raiseAssert("not implemented") + +method getPointer*(market: Market, slotId: SlotId): Future[uint8] {.base, async.} = + raiseAssert("not implemented") + +proc inDowntime*(market: Market, slotId: SlotId): Future[bool] {.async.} = + let downtime = await market.proofDowntime + let pntr = await market.getPointer(slotId) + return pntr < downtime + method requestStorage*(market: Market, request: StorageRequest) {.base, async.} = raiseAssert("not implemented") diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 7eb6d578..d523ab53 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -16,10 +16,9 @@ import std/sequtils import pkg/questionable import pkg/questionable/results -import pkg/chronicles +import pkg/chronicles except toJson import pkg/chronos -import pkg/presto -import pkg/libp2p +import pkg/presto except toJson import pkg/metrics import pkg/stew/base10 import pkg/stew/byteutils @@ -32,8 +31,8 @@ import pkg/codexdht/discv5/spr as spr import ../node import ../blocktype import ../conf -import ../contracts except `%*`, `%` # imported from contracts/marketplace (exporting ethers) -import ../streams +import ../contracts +import ../streams/asyncstreamwrapper import ./coders import ./json @@ -197,6 +196,28 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = trace "Excepting processing request", exc = exc.msg return RestApiResponse.error(Http500) + router.api( + MethodGet, + "/api/codex/v1/sales/slots/{slotId}") do (slotId: SlotId) -> RestApiResponse: + ## Returns active slots for the host + + without contracts =? node.contracts.host: + return RestApiResponse.error(Http503, "Sales unavailable") + + without slotId =? slotId.tryGet.catch, error: + return RestApiResponse.error(Http400, error.msg) + + without agent =? await contracts.sales.activeSale(slotId): + return RestApiResponse.error(Http404, "Provider not filling slot") + + let restAgent = RestSalesAgent( + state: agent.state() |? "none", + slotIndex: agent.data.slotIndex, + requestId: agent.data.requestId + ) + + return RestApiResponse.response(restAgent.toJson, contentType="application/json") + router.api( MethodGet, "/api/codex/v1/sales/availability") do () -> RestApiResponse: diff --git a/codex/rest/coders.nim b/codex/rest/coders.nim index 66b41ee3..2fa4da49 100644 --- a/codex/rest/coders.nim +++ b/codex/rest/coders.nim @@ -84,7 +84,7 @@ proc decodeString*(_: type array[32, byte], except ValueError as e: err e.msg.cstring -proc decodeString*[T: PurchaseId | RequestId | Nonce](_: type T, +proc decodeString*[T: PurchaseId | RequestId | Nonce | SlotId](_: type T, value: string): Result[T, cstring] = array[32, byte].decodeString(value).map(id => T(id)) diff --git a/codex/rest/json.nim b/codex/rest/json.nim index da26af0a..d23966eb 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -34,6 +34,11 @@ type minPrice* {.serialize.}: UInt256 maxCollateral* {.serialize.}: UInt256 + RestSalesAgent* = object + state* {.serialize.}: string + requestId* {.serialize.}: RequestId + slotIndex* {.serialize.}: UInt256 + RestContent* = object cid* {.serialize.}: Cid manifest* {.serialize.}: Manifest diff --git a/codex/sales.nim b/codex/sales.nim index b68f9ccc..53b5860b 100644 --- a/codex/sales.nim +++ b/codex/sales.nim @@ -39,6 +39,7 @@ import ./utils/trackedfutures export stint export reservations +export salesagent logScope: topics = "sales marketplace" @@ -208,6 +209,14 @@ proc mySlots*(sales: Sales): Future[seq[Slot]] {.async.} = return slots +proc activeSale*(sales: Sales, slotId: SlotId): Future[?SalesAgent] {.async.} = + let market = sales.context.market + for agent in sales.agents: + if slotId(agent.data.requestId, agent.data.slotIndex) == slotId: + return some agent + + return none SalesAgent + proc load*(sales: Sales) {.async.} = let activeSlots = await sales.mySlots() diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index 19aff95d..ed414594 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -40,7 +40,7 @@ import ../contracts/requests import ../utils/json export requests -export chronicles +export chronicles except toJson logScope: topics = "sales reservations" diff --git a/codex/sales/salesagent.nim b/codex/sales/salesagent.nim index 3db520b4..ab12c8b0 100644 --- a/codex/sales/salesagent.nim +++ b/codex/sales/salesagent.nim @@ -59,6 +59,12 @@ proc retrieveRequestState*(agent: SalesAgent): Future[?RequestState] {.async.} = let market = agent.context.market return await market.requestState(data.requestId) +func state*(agent: SalesAgent): ?string = + debugEcho "[salesagent] getting state..." + proc description(state: State): string = + $state + agent.query(description) + proc subscribeCancellation(agent: SalesAgent) {.async.} = let data = agent.data let clock = agent.context.clock diff --git a/codex/sales/states/proving.nim b/codex/sales/states/proving.nim index 1c39b910..ef65bf52 100644 --- a/codex/sales/states/proving.nim +++ b/codex/sales/states/proving.nim @@ -1,5 +1,7 @@ import std/options import pkg/chronicles +import pkg/questionable +import pkg/questionable/results import ../../clock import ../../utils/exceptions import ../statemachine @@ -48,17 +50,20 @@ proc proveLoop( let slotId = slot.id logScope: - period = currentPeriod + # period = currentPeriod requestId = $request.id slotIndex slotId = $slot.id proc getCurrentPeriod(): Future[Period] {.async.} = let periodicity = await market.periodicity() - return periodicity.periodOf(clock.now().u256) + let blockchainNow = await clock.lastBlockTimestamp + return periodicity.periodOf(blockchainNow) + # return periodicity.periodOf(clock.now().u256) proc waitUntilPeriod(period: Period) {.async.} = let periodicity = await market.periodicity() + debug "waiting until time", time = periodicity.periodStart(period).truncate(int64) await clock.waitUntil(periodicity.periodStart(period).truncate(int64)) while true: diff --git a/codex/utils/asyncstatemachine.nim b/codex/utils/asyncstatemachine.nim index b01cba79..acfdde18 100644 --- a/codex/utils/asyncstatemachine.nim +++ b/codex/utils/asyncstatemachine.nim @@ -34,7 +34,8 @@ proc transition(_: type Event, previous, next: State): Event = return some next proc query*[T](machine: Machine, query: Query[T]): ?T = - if machine.state == nil: + if not machine.state.isNil: debugEcho "machine state: ", $machine.state + if machine.state.isNil: none T else: some query(machine.state) diff --git a/codex/utils/json.nim b/codex/utils/json.nim index 8ece70e0..810dcddb 100644 --- a/codex/utils/json.nim +++ b/codex/utils/json.nim @@ -306,6 +306,16 @@ func `%`*[T: distinct](id: T): JsonNode = func toJson*(obj: object): string = $(%obj) func toJson*(obj: ref object): string = $(%obj) +func toJson*[T: object](elements: openArray[T]): string = + let jObj = newJArray() + for elem in elements: jObj.add(%elem) + $jObj + +func toJson*[T: ref object](elements: openArray[T]): string = + let jObj = newJArray() + for elem in elements: jObj.add(%elem) + $jObj + proc toJsnImpl(x: NimNode): NimNode = case x.kind of nnkBracket: # array diff --git a/codex/validation.nim b/codex/validation.nim index a3f69296..31e0efed 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -2,6 +2,8 @@ import std/sets import std/sequtils import pkg/chronos import pkg/chronicles +import pkg/questionable +import pkg/questionable/results import ./market import ./clock @@ -34,11 +36,13 @@ proc new*( proc slots*(validation: Validation): seq[SlotId] = validation.slots.toSeq -proc getCurrentPeriod(validation: Validation): UInt256 = - return validation.periodicity.periodOf(validation.clock.now().u256) +proc getCurrentPeriod(validation: Validation): Future[UInt256] {.async.} = + let currentTime = await validation.clock.lastBlockTimestamp + return validation.periodicity.periodOf(currentTime) + # return validation.periodicity.periodOf(validation.clock.now().u256) proc waitUntilNextPeriod(validation: Validation) {.async.} = - let period = validation.getCurrentPeriod() + let period = await validation.getCurrentPeriod() let periodEnd = validation.periodicity.periodEnd(period) trace "Waiting until next period", currentPeriod = period await validation.clock.waitUntil(periodEnd.truncate(int64) + 1) @@ -66,13 +70,15 @@ proc markProofAsMissing(validation: Validation, slotId: SlotId, period: Period) {.async.} = logScope: - currentPeriod = validation.getCurrentPeriod() + currentPeriod = await validation.getCurrentPeriod() try: if await validation.market.canProofBeMarkedAsMissing(slotId, period): trace "Marking proof as missing", slotId = $slotId, periodProofMissed = period await validation.market.markProofAsMissing(slotId, period) - else: trace "Proof not missing", checkedPeriod = period + else: + let inDowntime = await validation.market.inDowntime(slotId) + trace "Proof not missing", checkedPeriod = period, inDowntime except CancelledError: raise except CatchableError as e: @@ -80,7 +86,7 @@ proc markProofAsMissing(validation: Validation, proc markProofsAsMissing(validation: Validation) {.async.} = for slotId in validation.slots: - let previousPeriod = validation.getCurrentPeriod() - 1 + let previousPeriod = (await validation.getCurrentPeriod()) - 1 await validation.markProofAsMissing(slotId, previousPeriod) proc run(validation: Validation) {.async.} = diff --git a/tests/codex/helpers/mockclock.nim b/tests/codex/helpers/mockclock.nim index ada449f9..8e19205a 100644 --- a/tests/codex/helpers/mockclock.nim +++ b/tests/codex/helpers/mockclock.nim @@ -35,6 +35,9 @@ proc advance*(clock: MockClock, seconds: int64) = method now*(clock: MockClock): SecondsSince1970 = clock.time +method lastBlockTimestamp*(clock: MockClock): Future[UInt256] {.base, async.} = + return clock.now.u256 + method waitUntil*(clock: MockClock, time: SecondsSince1970) {.async.} = if time > clock.now(): let future = newFuture[void]() diff --git a/tests/codex/helpers/mockmarket.nim b/tests/codex/helpers/mockmarket.nim index 867a3ef5..2f9c1848 100644 --- a/tests/codex/helpers/mockmarket.nim +++ b/tests/codex/helpers/mockmarket.nim @@ -110,6 +110,12 @@ method periodicity*(mock: MockMarket): Future[Periodicity] {.async.} = method proofTimeout*(market: MockMarket): Future[UInt256] {.async.} = return market.config.proofs.timeout +method proofDowntime*(market: MockMarket): Future[uint8] {.async.} = + return market.config.proofs.downtime + +method getPointer*(market: MockMarket, slotId: SlotId): Future[uint8] {.async.} = + return 0 # TODO + method requestStorage*(market: MockMarket, request: StorageRequest) {.async.} = market.requested.add(request) var subscriptions = market.subscriptions.onRequest diff --git a/tests/ethertest.nim b/tests/ethertest.nim index 3a76cd2e..0accad54 100644 --- a/tests/ethertest.nim +++ b/tests/ethertest.nim @@ -1,4 +1,4 @@ -import std/json +# import std/json import pkg/asynctest import pkg/ethers @@ -26,4 +26,4 @@ template ethersuite*(name, body) = body export asynctest -export ethers +export ethers except `%` diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index d8657119..285d365f 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -108,6 +108,18 @@ proc getPurchase*(client: CodexClient, purchaseId: PurchaseId): ?!RestPurchase = let json = ? parseJson(body).catch RestPurchase.fromJson(json) +proc getSalesAgent*(client: CodexClient, slotId: SlotId): ?!RestSalesAgent = + let url = client.baseurl & "/sales/slots/" & slotId.toHex + echo "getting sales agent for id, ", slotId.toHex + try: + let body = client.http.getContent(url) + echo "get sales agent body: ", body + let json = ? parseJson(body).catch + return RestSalesAgent.fromJson(json) + except CatchableError as e: + echo "[client.getSalesAgent] error getting agent: ", e.msg + return failure e.msg + proc getSlots*(client: CodexClient): ?!seq[Slot] = let url = client.baseurl & "/sales/slots" let body = client.http.getContent(url) @@ -147,5 +159,8 @@ proc restart*(client: CodexClient) = proc purchaseStateIs*(client: CodexClient, id: PurchaseId, state: string): bool = client.getPurchase(id).option.?state == some state +proc saleStateIs*(client: CodexClient, id: SlotId, state: string): bool = + client.getSalesAgent(id).option.?state == some state + proc requestId*(client: CodexClient, id: PurchaseId): ?RequestId = return client.getPurchase(id).option.?requestId diff --git a/tests/integration/marketplacesuite.nim b/tests/integration/marketplacesuite.nim index 7c3492c6..6068653c 100644 --- a/tests/integration/marketplacesuite.nim +++ b/tests/integration/marketplacesuite.nim @@ -2,6 +2,7 @@ import std/times import pkg/chronos import pkg/codex/contracts/marketplace as mp import pkg/codex/periods +import pkg/codex/utils/json import ./multinodes export mp @@ -13,14 +14,32 @@ template marketplacesuite*(name: string, startNodes: Nodes, body: untyped) = var marketplace {.inject, used.}: Marketplace var period: uint64 + var periodicity: Periodicity var token {.inject, used.}: Erc20Token + proc getCurrentPeriod(): Future[Period] {.async.} = + return periodicity.periodOf(await ethProvider.currentTime()) + proc advanceToNextPeriod() {.async.} = let periodicity = Periodicity(seconds: period.u256) - let currentPeriod = periodicity.periodOf(ethProvider.currentTime()) - let nextPeriod = periodicity.periodEnd(currentPeriod) - echo "advancing to next period start at ", nextPeriod + 1 - await ethProvider.advanceTimeTo(nextPeriod + 1) + let currentPeriod = periodicity.periodOf(await ethProvider.currentTime()) + let endOfPeriod = periodicity.periodEnd(currentPeriod) + echo "advancing to next period start at ", endOfPeriod + 1 + await ethProvider.advanceTimeTo(endOfPeriod + 1) + + proc timeUntil(period: Period): Future[times.Duration] {.async.} = + let currentPeriod = await getCurrentPeriod() + echo "[timeUntil] currentPeriod: ", currentPeriod + echo "[timeUntil] waiting until period: ", period + let endOfCurrPeriod = periodicity.periodEnd(currentPeriod) + let endOfLastPeriod = periodicity.periodEnd(period) + let endOfCurrPeriodTime = initTime(endOfCurrPeriod.truncate(int64), 0) + let endOfLastPeriodTime = initTime(endOfLastPeriod.truncate(int64), 0) + let r = endOfLastPeriodTime - endOfCurrPeriodTime + echo "[timeUntil] diff between end of current period and now: ", r + return r + # echo "advancing to next period start at ", endOfPeriod + 1 + # await ethProvider.advanceTimeTo(endOfPeriod + 1) proc periods(p: int): uint64 = p.uint64 * period @@ -88,6 +107,11 @@ template marketplacesuite*(name: string, startNodes: Nodes, body: untyped) = token = Erc20Token.new(tokenAddress, ethProvider.getSigner()) let config = await mp.config(marketplace) period = config.proofs.period.truncate(uint64) + periodicity = Periodicity(seconds: period.u256) + + + + discard await ethProvider.send("evm_setIntervalMining", @[%1000]) diff --git a/tests/integration/testproofs.nim b/tests/integration/testproofs.nim index dda4cbc8..f605afaf 100644 --- a/tests/integration/testproofs.nim +++ b/tests/integration/testproofs.nim @@ -1,6 +1,6 @@ # import std/sequtils # import std/os -# from std/times import getTime, toUnix +from std/times import inMilliseconds import pkg/chronicles import pkg/stew/byteutils # import pkg/codex/contracts @@ -242,136 +242,7 @@ logScope: -marketplacesuite "Simulate invalid proofs - 1 provider node", - Nodes( - # hardhat: HardhatConfig() - # .withLogFile(), - - clients: NodeConfig() - .nodes(1) - .debug() - .withLogFile() - .withLogTopics("node"), - - providers: NodeConfig() - .nodes(1) - .simulateProofFailuresFor(providerIdx=0, failEveryNProofs=1) - .debug() - .withLogFile() - .withLogTopics( - "marketplace", - "sales", - "reservations", - "node", - "JSONRPC-HTTP-CLIENT", - "JSONRPC-WS-CLIENT", - "ethers", - "restapi", - "clock" - ), - - validators: NodeConfig() - .nodes(1) - .withLogFile() - .withLogTopics("validator", "initial-proving", "proving", "clock") - ): - test "slot is freed after too many invalid proofs submitted": - let client0 = clients()[0].node.client - let totalProofs = 50 - - let data = byteutils.toHex(await exampleData()) - createAvailabilities(data.len, totalProofs.periods) - - let cid = client0.upload(data).get - - let purchaseId = await client0.requestStorage(cid, duration=totalProofs.periods) - let requestId = client0.requestId(purchaseId).get - - check eventually client0.purchaseStateIs(purchaseId, "started") - - var slotWasFreed = false - proc onSlotFreed(event: SlotFreed) {.gcsafe, upraises:[].} = - if event.requestId == requestId and - event.slotIndex == 0.u256: # assume only one slot, so index 0 - slotWasFreed = true - - let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) - - echo "waiting for proofs ", totalProofs - # check eventuallyP(slotWasFreed, totalProofs, 1.periods.int) - check eventually(slotWasFreed, totalProofs.periods.int * 1000) - echo "got to the end... freed: ", slotWasFreed - - echo "unsubscribing from slotFreed" - await subscription.unsubscribe() - echo "unsubscribed from slotFreed" - -marketplacesuite "Simulate invalid proofs - 1 provider node", - Nodes( - hardhat: HardhatConfig() - .withLogFile(), - - clients: NodeConfig() - .nodes(1) - .debug() - .withLogFile() - .withLogTopics("node"), - - providers: NodeConfig() - .nodes(1) - .simulateProofFailuresFor(providerIdx=0, failEveryNProofs=3) - .debug() - .withLogFile() - .withLogTopics( - "marketplace", - "sales", - "reservations", - "node", - "JSONRPC-HTTP-CLIENT", - "JSONRPC-WS-CLIENT", - "ethers", - "restapi", - "clock" - ), - - validators: NodeConfig() - .nodes(1) - .withLogFile() - .withLogTopics("validator", "initial-proving", "proving", "clock") - ): - test "slot is not freed when not enough invalid proofs submitted": - let client0 = clients()[0].node.client - let totalProofs = 25 - - let data = byteutils.toHex(await exampleData()) - createAvailabilities(data.len, totalProofs.periods) - - let cid = client0.upload(data).get - - let purchaseId = await client0.requestStorage(cid, duration=totalProofs.periods) - let requestId = client0.requestId(purchaseId).get - - check eventually client0.purchaseStateIs(purchaseId, "started") - - var slotWasFreed = false - proc onSlotFreed(event: SlotFreed) {.gcsafe, upraises:[].} = - if event.requestId == requestId and - event.slotIndex == 0.u256: # assume only one slot, so index 0 - slotWasFreed = true - - let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) - - echo "waiting for proofs: ", totalProofs - let freed = eventuallyP(slotWasFreed, totalProofs, 1.periods.int) - echo "got to the end... freed: ", freed - check not freed - # check not eventually(slotWasFreed, totalProofs.periods.int * 1000) - - echo "unsubscribing from slotFreed" - await subscription.unsubscribe() - echo "unsubscribed from slotFreed" - -# marketplacesuite "Simulate invalid proofs", +# marketplacesuite "Simulate invalid proofs - 1 provider node", # Nodes( # hardhat: HardhatConfig() # .withLogFile(), @@ -380,11 +251,11 @@ marketplacesuite "Simulate invalid proofs - 1 provider node", # .nodes(1) # .debug() # .withLogFile() -# .withLogTopics("node", "erasure"), +# .withLogTopics("node"), # providers: NodeConfig() -# .nodes(2) -# .simulateProofFailuresFor(providerIdx=0, failEveryNProofs=2) +# .nodes(1) +# .simulateProofFailuresFor(providerIdx=0, failEveryNProofs=1) # .debug() # .withLogFile() # .withLogTopics( @@ -395,64 +266,230 @@ marketplacesuite "Simulate invalid proofs - 1 provider node", # "JSONRPC-HTTP-CLIENT", # "JSONRPC-WS-CLIENT", # "ethers", -# "restapi" +# "restapi", +# "clock" # ), # validators: NodeConfig() # .nodes(1) # .withLogFile() -# .withLogTopics("validator", "initial-proving", "proving") +# .debug() +# .withLogTopics("validator", "initial-proving", "proving", "clock", "onchain", "ethers") # ): -# # TODO: these are very loose tests in that they are not testing EXACTLY how -# # proofs were marked as missed by the validator. These tests should be -# # tightened so that they are showing, as an integration test, that specific -# # proofs are being marked as missed by the validator. +# test "slot is freed after too many invalid proofs submitted": +# let client0 = clients()[0].node.client +# let totalProofs = 50 -# test "provider that submits invalid proofs is paid out less": -# let client0 = clients()[0].node.client -# let provider0 = providers()[0] -# let provider1 = providers()[1] -# let totalProofs = 25 +# let data = byteutils.toHex(await exampleData()) +# createAvailabilities(data.len, totalProofs.periods) -# let data = byteutils.toHex(await exampleData()) -# # createAvailabilities(data.len, totalProofs.periods) +# let cid = client0.upload(data).get -# discard provider0.node.client.postAvailability( -# size=data.len.u256, # should match 1 slot only -# duration=totalProofs.periods.u256, -# minPrice=300.u256, -# maxCollateral=200.u256 -# ) +# let purchaseId = await client0.requestStorage(cid, duration=totalProofs.periods) +# let requestId = client0.requestId(purchaseId).get -# let cid = client0.upload(data).get +# check eventually client0.purchaseStateIs(purchaseId, "started") -# let purchaseId = await client0.requestStorage( -# cid, -# duration=totalProofs.periods, -# # tolerance=1 -# ) +# var slotWasFreed = false +# proc onSlotFreed(event: SlotFreed) {.gcsafe, upraises:[].} = +# if event.requestId == requestId and +# event.slotIndex == 0.u256: # assume only one slot, so index 0 +# slotWasFreed = true -# await sleepAsync(1.seconds) # allow time for provider0 to fill a slot +# let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) -# # now add availability for provider1, which should allow provider1 to put -# # the remaining slot in its queue -# discard provider1.node.client.postAvailability( -# size=data.len.u256, # should match 1 slot only -# duration=totalProofs.periods.u256, -# minPrice=300.u256, -# maxCollateral=200.u256 -# ) +# echo "waiting for proofs ", totalProofs +# # check eventuallyP(slotWasFreed, totalProofs, 1.periods.int) +# check eventually(slotWasFreed, totalProofs.periods.int * 1000) +# echo "got to the end... freed: ", slotWasFreed -# check eventually client0.purchaseStateIs(purchaseId, "started") +# echo "unsubscribing from slotFreed" +# await subscription.unsubscribe() +# echo "unsubscribed from slotFreed" -# check eventuallyP( -# client0.purchaseStateIs(purchaseId, "finished"), -# totalProofs) +# marketplacesuite "Simulate invalid proofs - 1 provider node", +# Nodes( +# hardhat: HardhatConfig() +# .withLogFile(), -# check eventually ( -# (await token.balanceOf(!provider1.address)) > -# (await token.balanceOf(!provider0.address)) -# ) -# echo "provider1 balance: ", (await token.balanceOf(!provider1.address)) -# echo "provider0 balance: ", (await token.balanceOf(!provider0.address)) +# clients: NodeConfig() +# .nodes(1) +# .debug() +# .withLogFile() +# .withLogTopics("node"), + +# providers: NodeConfig() +# .nodes(1) +# .simulateProofFailuresFor(providerIdx=0, failEveryNProofs=3) +# .debug() +# .withLogFile() +# .withLogTopics( +# "marketplace", +# "sales", +# "reservations", +# "node", +# "JSONRPC-HTTP-CLIENT", +# "JSONRPC-WS-CLIENT", +# "ethers", +# "restapi", +# "clock" +# ), + +# validators: NodeConfig() +# .nodes(1) +# .withLogFile() +# .withLogTopics("validator", "initial-proving", "proving", "clock", "onchain", "ethers") +# ): + +# test "slot is not freed when not enough invalid proofs submitted": +# let client0 = clients()[0].node.client +# let totalProofs = 25 + +# let data = byteutils.toHex(await exampleData()) +# createAvailabilities(data.len, totalProofs.periods) + +# let cid = client0.upload(data).get + +# let purchaseId = await client0.requestStorage(cid, duration=totalProofs.periods) +# let requestId = client0.requestId(purchaseId).get + +# check eventually client0.purchaseStateIs(purchaseId, "started") + +# var slotWasFreed = false +# proc onSlotFreed(event: SlotFreed) {.gcsafe, upraises:[].} = +# if event.requestId == requestId and +# event.slotIndex == 0.u256: # assume only one slot, so index 0 +# slotWasFreed = true + +# let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) + +# echo "waiting for proofs: ", totalProofs +# # let freed = eventuallyP(slotWasFreed, totalProofs, 1.periods.int) +# # echo "got to the end... freed: ", freed +# # check not freed +# check not eventually(slotWasFreed, totalProofs.periods.int * 1000) + +# echo "unsubscribing from slotFreed" +# await subscription.unsubscribe() +# echo "unsubscribed from slotFreed" + +marketplacesuite "Simulate invalid proofs", + Nodes( + hardhat: HardhatConfig() + .withLogFile(), + + clients: NodeConfig() + .nodes(1) + .debug() + .withLogFile() + .withLogTopics("node", "erasure"), + + providers: NodeConfig() + .nodes(2) + .simulateProofFailuresFor(providerIdx=0, failEveryNProofs=2) + .debug() + .withLogFile() + .withLogTopics( + "marketplace", + "sales", + "reservations", + "node", + "JSONRPC-HTTP-CLIENT", + "JSONRPC-WS-CLIENT", + "ethers", + "restapi" + ), + + validators: NodeConfig() + .nodes(1) + .withLogFile() + .withLogTopics("validator", "initial-proving", "proving") + ): + + # TODO: these are very loose tests in that they are not testing EXACTLY how + # proofs were marked as missed by the validator. These tests should be + # tightened so that they are showing, as an integration test, that specific + # proofs are being marked as missed by the validator. + + test "provider that submits invalid proofs is paid out less": + let client0 = clients()[0].node.client + let provider0 = providers()[0] + let provider1 = providers()[1] + let totalProofs = 25 + + let data = byteutils.toHex(await exampleData()) + # createAvailabilities(data.len, totalProofs.periods) + + discard provider0.node.client.postAvailability( + size=data.len.u256, # should match 1 slot only + duration=totalProofs.periods.u256, + minPrice=300.u256, + maxCollateral=200.u256 + ) + + let cid = client0.upload(data).get + + let purchaseId = await client0.requestStorage( + cid, + duration=totalProofs.periods, + # tolerance=1 + ) + + without requestId =? client0.requestId(purchaseId): + fail() + + + var provider0slotIndex = none UInt256 + proc onSlotFilled(event: SlotFilled) {.upraises:[].} = + provider0slotIndex = some event.slotIndex + + let subscription = await marketplace.subscribe(SlotFilled, onSlotFilled) + + # wait til first slot is filled + check eventually provider0slotIndex.isSome + + + + # await sleepAsync(1.seconds) # allow time for provider0 to fill a slot + + # now add availability for provider1, which should allow provider1 to put + # the remaining slot in its queue + discard provider1.node.client.postAvailability( + size=data.len.u256, # should match 1 slot only + duration=totalProofs.periods.u256, + minPrice=300.u256, + maxCollateral=200.u256 + ) + let provider1slotIndex = if provider0slotIndex == some 0.u256: 1.u256 else: 0.u256 + echo "PROVIDER 0 SLOT INDEX: ", provider0slotIndex + echo "PROVIDER 1 SLOT INDEX: ", provider1slotIndex + let provider0slotId = slotId(requestId, !provider0slotIndex) + let provider1slotId = slotId(requestId, provider1slotIndex) + + # wait til second slot is filled + check eventually provider1.node.client.saleStateIs(provider1slotId, "SaleFilled") + + check eventually client0.purchaseStateIs(purchaseId, "started") + + let currentPeriod = await getCurrentPeriod() + let timeUtil = await timeUntil(currentPeriod + totalProofs.u256 + 1) + # check eventually( + # client0.purchaseStateIs(purchaseId, "finished"), + # timeUtil.inMilliseconds.int) + check eventually( + provider0.node.client.saleStateIs(provider0slotId, "SaleFinished"), + timeUtil.inMilliseconds.int) + + check eventually( + provider1.node.client.saleStateIs(provider1slotId, "SaleFinished"), + timeUtil.inMilliseconds.int) + + check eventually( + (await token.balanceOf(!provider1.address)) > + (await token.balanceOf(!provider0.address)) + ) + echo "provider1 balance: ", (await token.balanceOf(!provider1.address)) + echo "provider0 balance: ", (await token.balanceOf(!provider0.address)) + + await subscription.unsubscribe()