Adam Uhlíř 3b520fc0c6
fix: unknown state goes to payout when slot state is finished (#555)
Support logging to file

Log the entire config and fix build error

Process slot queue on reservation callback onMarkUnused

Add Reservation object, rename reserve > create

refactor Reservations api to include Reservation CRUD

All tests that use the Reservation module updated

- add requestId and slotIndex to Reservation (hopefully these will prove to be useful when we persist Reservations until request are completed, to add back bytes to Availability)
- add querying of all reservations, with accompanying tests
- change from find to findAvailabilities
- move onCleanUp from SalesContext to SalesAgent as it was getting overwritten for each slot processed
- remove sales agent AFTER deleting reservation, as this was causing some SIGSEGVs
- retrofit testsales and testslotqueue to match updated Reservations module API

Add deletion of inactive reservations on sales load

clean up

add exception message detail util

Apply to onStore errors as we are seeing undetailed errors in the dist tests logs

add missing file

change slotsize to reflect current implemenation

Fix slotSize to reduce by one block

Revert change to slotSize that reduces it by one block

Add additional test check for querying reservations/availabilities

filter past requests based on availability

Because availability filtering on push was removed, when availability is added and past storage request events are queried, those requests need to be filtered by availability before being added to the queue.

Revert "filter past requests based on availability"

This reverts commit 0c2362658b523e0de425794b1fa30ebd53bd30ae.

Add debugging for dist tests

Add cancel on error during filling state

When calling fillSlot, any transaction errors that occur (possibly during estimate gas) will cause that tx to be replaced with a cancellation transaction (a 0-valued tx to ourselves).

more debug logging

fix build

wait for fillSlot to be mined (so that a revert error can be extracted)

fix confirmation of fillSlot

add more debugging

moar debugging

debugging: change echo to trace

bump ethers to add chronicles

fix contracts tests

switch to earlier nim-ethers commit

bump json-rpc and nim-ethers

bump nim-ethers to prevent parsing newHeads log from crashing sales state machine

moar debugging

moar debugging

moar debugging

bump ethers to fix "key not found: data" error

ethers debug logging

bump ethers - better Transaction object deserialization

bump ethers to replay tx with past tx format

bump ethers to add serialization for PastTransaction

ethers bump: better logging, separate logic for revert reason string

fix build

ethers: update revert reason retreival

remove unneeded confirm

ethers: try replay without decrementing blockNumber

ethers: include type and chainId in replayed txs

ethers: add gas into and remove type from replayed txs

ensure gas is being serialized in Transaction

ethers: fix build error

bump ethers: rebased on top of cancel tx due to estimateGas error PR

Fix proving with latest ethers bump (tested on master)

Update multinode suite for better simulateFailedProofs enabling, add proofs test

Improve multinode suite for better debug options, including logging to file

There is a 503 "sales unavailable" error

improve multinode test suite, add logging to file
2023-12-06 15:39:03 +11:00

438 lines
15 KiB
Nim

import std/sequtils
import std/tables
import std/hashes
import std/sets
import std/sugar
import pkg/questionable
import pkg/codex/market
import pkg/codex/contracts/requests
import pkg/codex/contracts/config
import ../examples
export market
export tables
type
MockMarket* = ref object of Market
periodicity: Periodicity
activeRequests*: Table[Address, seq[RequestId]]
activeSlots*: Table[Address, seq[SlotId]]
requested*: seq[StorageRequest]
requestEnds*: Table[RequestId, SecondsSince1970]
requestState*: Table[RequestId, RequestState]
slotState*: Table[SlotId, SlotState]
fulfilled*: seq[Fulfillment]
filled*: seq[MockSlot]
freed*: seq[SlotId]
markedAsMissingProofs*: seq[SlotId]
canBeMarkedAsMissing: HashSet[SlotId]
withdrawn*: seq[RequestId]
proofsRequired: HashSet[SlotId]
proofsToBeRequired: HashSet[SlotId]
proofEnds: Table[SlotId, UInt256]
signer: Address
subscriptions: Subscriptions
config*: MarketplaceConfig
Fulfillment* = object
requestId*: RequestId
proof*: seq[byte]
host*: Address
MockSlot* = object
requestId*: RequestId
host*: Address
slotIndex*: UInt256
proof*: seq[byte]
Subscriptions = object
onRequest: seq[RequestSubscription]
onFulfillment: seq[FulfillmentSubscription]
onSlotFilled: seq[SlotFilledSubscription]
onSlotFreed: seq[SlotFreedSubscription]
onRequestCancelled: seq[RequestCancelledSubscription]
onRequestFailed: seq[RequestFailedSubscription]
onProofSubmitted: seq[ProofSubmittedSubscription]
RequestSubscription* = ref object of Subscription
market: MockMarket
callback: OnRequest
FulfillmentSubscription* = ref object of Subscription
market: MockMarket
requestId: ?RequestId
callback: OnFulfillment
SlotFilledSubscription* = ref object of Subscription
market: MockMarket
requestId: ?RequestId
slotIndex: ?UInt256
callback: OnSlotFilled
SlotFreedSubscription* = ref object of Subscription
market: MockMarket
callback: OnSlotFreed
RequestCancelledSubscription* = ref object of Subscription
market: MockMarket
requestId: ?RequestId
callback: OnRequestCancelled
RequestFailedSubscription* = ref object of Subscription
market: MockMarket
requestId: ?RequestId
callback: OnRequestCancelled
ProofSubmittedSubscription = ref object of Subscription
market: MockMarket
callback: OnProofSubmitted
proc hash*(address: Address): Hash =
hash(address.toArray)
proc hash*(requestId: RequestId): Hash =
hash(requestId.toArray)
proc new*(_: type MockMarket): MockMarket =
## Create a new mocked Market instance
##
let config = MarketplaceConfig(
collateral: CollateralConfig(
repairRewardPercentage: 10,
maxNumberOfSlashes: 5,
slashCriterion: 3,
slashPercentage: 10
),
proofs: ProofConfig(
period: 10.u256,
timeout: 5.u256,
downtime: 64.uint8
)
)
MockMarket(signer: Address.example, config: config)
method getSigner*(market: MockMarket): Future[Address] {.async.} =
return market.signer
method periodicity*(mock: MockMarket): Future[Periodicity] {.async.} =
return Periodicity(seconds: mock.config.proofs.period)
method proofTimeout*(market: MockMarket): Future[UInt256] {.async.} =
return market.config.proofs.timeout
method requestStorage*(market: MockMarket, request: StorageRequest) {.async.} =
market.requested.add(request)
var subscriptions = market.subscriptions.onRequest
for subscription in subscriptions:
subscription.callback(request.id,
request.ask,
request.expiry)
method myRequests*(market: MockMarket): Future[seq[RequestId]] {.async.} =
return market.activeRequests[market.signer]
method mySlots*(market: MockMarket): Future[seq[SlotId]] {.async.} =
return market.activeSlots[market.signer]
method getRequest(market: MockMarket,
id: RequestId): Future[?StorageRequest] {.async.} =
for request in market.requested:
if request.id == id:
return some request
return none StorageRequest
method getActiveSlot*(
market: MockMarket,
slotId: SlotId): Future[?Slot] {.async.} =
for slot in market.filled:
if slotId(slot.requestId, slot.slotIndex) == slotId and
request =? await market.getRequest(slot.requestId):
return some Slot(request: request, slotIndex: slot.slotIndex)
return none Slot
method requestState*(market: MockMarket,
requestId: RequestId): Future[?RequestState] {.async.} =
return market.requestState.?[requestId]
method slotState*(market: MockMarket,
slotId: SlotId): Future[SlotState] {.async.} =
if not market.slotState.hasKey(slotId):
return SlotState.Free
return market.slotState[slotId]
method getRequestEnd*(market: MockMarket,
id: RequestId): Future[SecondsSince1970] {.async.} =
return market.requestEnds[id]
method getHost*(market: MockMarket,
requestId: RequestId,
slotIndex: UInt256): Future[?Address] {.async.} =
for slot in market.filled:
if slot.requestId == requestId and slot.slotIndex == slotIndex:
return some slot.host
return none Address
proc emitSlotFilled*(market: MockMarket,
requestId: RequestId,
slotIndex: UInt256) =
var subscriptions = market.subscriptions.onSlotFilled
for subscription in subscriptions:
let requestMatches =
subscription.requestId.isNone or
subscription.requestId == some requestId
let slotMatches =
subscription.slotIndex.isNone or
subscription.slotIndex == some slotIndex
if requestMatches and slotMatches:
subscription.callback(requestId, slotIndex)
proc emitSlotFreed*(market: MockMarket,
requestId: RequestId,
slotIndex: UInt256) =
var subscriptions = market.subscriptions.onSlotFreed
for subscription in subscriptions:
subscription.callback(requestId, slotIndex)
proc emitRequestCancelled*(market: MockMarket, requestId: RequestId) =
var subscriptions = market.subscriptions.onRequestCancelled
for subscription in subscriptions:
if subscription.requestId == requestId.some or
subscription.requestId.isNone:
subscription.callback(requestId)
proc emitRequestFulfilled*(market: MockMarket, requestId: RequestId) =
var subscriptions = market.subscriptions.onFulfillment
for subscription in subscriptions:
if subscription.requestId == requestId.some or
subscription.requestId.isNone:
subscription.callback(requestId)
proc emitRequestFailed*(market: MockMarket, requestId: RequestId) =
var subscriptions = market.subscriptions.onRequestFailed
for subscription in subscriptions:
if subscription.requestId == requestId.some or
subscription.requestId.isNone:
subscription.callback(requestId)
proc cancelTransaction(market: Market, nonce: UInt256) {.async.} =
discard
proc fillSlot*(market: MockMarket,
requestId: RequestId,
slotIndex: UInt256,
proof: seq[byte],
host: Address) =
let slot = MockSlot(
requestId: requestId,
slotIndex: slotIndex,
proof: proof,
host: host
)
market.filled.add(slot)
market.slotState[slotId(slot.requestId, slot.slotIndex)] = SlotState.Filled
market.emitSlotFilled(requestId, slotIndex)
method fillSlot*(market: MockMarket,
requestId: RequestId,
slotIndex: UInt256,
proof: seq[byte],
collateral: UInt256) {.async.} =
market.fillSlot(requestId, slotIndex, proof, market.signer)
method freeSlot*(market: MockMarket, slotId: SlotId) {.async.} =
market.freed.add(slotId)
for s in market.filled:
if slotId(s.requestId, s.slotIndex) == slotId:
market.emitSlotFreed(s.requestId, s.slotIndex)
break
market.slotState[slotId] = SlotState.Free
method withdrawFunds*(market: MockMarket,
requestId: RequestId) {.async.} =
market.withdrawn.add(requestId)
market.emitRequestCancelled(requestId)
proc setProofRequired*(mock: MockMarket, id: SlotId, required: bool) =
if required:
mock.proofsRequired.incl(id)
else:
mock.proofsRequired.excl(id)
method isProofRequired*(mock: MockMarket,
id: SlotId): Future[bool] {.async.} =
return mock.proofsRequired.contains(id)
proc setProofToBeRequired*(mock: MockMarket, id: SlotId, required: bool) =
if required:
mock.proofsToBeRequired.incl(id)
else:
mock.proofsToBeRequired.excl(id)
method willProofBeRequired*(mock: MockMarket,
id: SlotId): Future[bool] {.async.} =
return mock.proofsToBeRequired.contains(id)
proc setProofEnd*(mock: MockMarket, id: SlotId, proofEnd: UInt256) =
mock.proofEnds[id] = proofEnd
method submitProof*(mock: MockMarket, id: SlotId, proof: seq[byte]) {.async.} =
for subscription in mock.subscriptions.onProofSubmitted:
subscription.callback(id, proof)
method markProofAsMissing*(market: MockMarket,
id: SlotId,
period: Period) {.async.} =
market.markedAsMissingProofs.add(id)
proc setCanProofBeMarkedAsMissing*(mock: MockMarket, id: SlotId, required: bool) =
if required:
mock.canBeMarkedAsMissing.incl(id)
else:
mock.canBeMarkedAsMissing.excl(id)
method canProofBeMarkedAsMissing*(market: MockMarket,
id: SlotId,
period: Period): Future[bool] {.async.} =
return market.canBeMarkedAsMissing.contains(id)
method subscribeRequests*(market: MockMarket,
callback: OnRequest):
Future[Subscription] {.async.} =
let subscription = RequestSubscription(
market: market,
callback: callback
)
market.subscriptions.onRequest.add(subscription)
return subscription
method subscribeFulfillment*(market: MockMarket,
callback: OnFulfillment):
Future[Subscription] {.async.} =
let subscription = FulfillmentSubscription(
market: market,
requestId: none RequestId,
callback: callback
)
market.subscriptions.onFulfillment.add(subscription)
return subscription
method subscribeFulfillment*(market: MockMarket,
requestId: RequestId,
callback: OnFulfillment):
Future[Subscription] {.async.} =
let subscription = FulfillmentSubscription(
market: market,
requestId: some requestId,
callback: callback
)
market.subscriptions.onFulfillment.add(subscription)
return subscription
method subscribeSlotFilled*(market: MockMarket,
callback: OnSlotFilled):
Future[Subscription] {.async.} =
let subscription = SlotFilledSubscription(market: market, callback: callback)
market.subscriptions.onSlotFilled.add(subscription)
return subscription
method subscribeSlotFilled*(market: MockMarket,
requestId: RequestId,
slotIndex: UInt256,
callback: OnSlotFilled):
Future[Subscription] {.async.} =
let subscription = SlotFilledSubscription(
market: market,
requestId: some requestId,
slotIndex: some slotIndex,
callback: callback
)
market.subscriptions.onSlotFilled.add(subscription)
return subscription
method subscribeSlotFreed*(market: MockMarket,
callback: OnSlotFreed):
Future[Subscription] {.async.} =
let subscription = SlotFreedSubscription(market: market, callback: callback)
market.subscriptions.onSlotFreed.add(subscription)
return subscription
method subscribeRequestCancelled*(market: MockMarket,
callback: OnRequestCancelled):
Future[Subscription] {.async.} =
let subscription = RequestCancelledSubscription(
market: market,
requestId: none RequestId,
callback: callback
)
market.subscriptions.onRequestCancelled.add(subscription)
return subscription
method subscribeRequestCancelled*(market: MockMarket,
requestId: RequestId,
callback: OnRequestCancelled):
Future[Subscription] {.async.} =
let subscription = RequestCancelledSubscription(
market: market,
requestId: some requestId,
callback: callback
)
market.subscriptions.onRequestCancelled.add(subscription)
return subscription
method subscribeRequestFailed*(market: MockMarket,
callback: OnRequestFailed):
Future[Subscription] {.async.} =
let subscription = RequestFailedSubscription(
market: market,
requestId: none RequestId,
callback: callback
)
market.subscriptions.onRequestFailed.add(subscription)
return subscription
method subscribeRequestFailed*(market: MockMarket,
requestId: RequestId,
callback: OnRequestFailed):
Future[Subscription] {.async.} =
let subscription = RequestFailedSubscription(
market: market,
requestId: some requestId,
callback: callback
)
market.subscriptions.onRequestFailed.add(subscription)
return subscription
method subscribeProofSubmission*(mock: MockMarket,
callback: OnProofSubmitted):
Future[Subscription] {.async.} =
let subscription = ProofSubmittedSubscription(
market: mock,
callback: callback
)
mock.subscriptions.onProofSubmitted.add(subscription)
return subscription
method queryPastStorageRequests*(market: MockMarket,
blocksAgo: int):
Future[seq[PastStorageRequest]] {.async.} =
# MockMarket does not have the concept of blocks, so simply return all
# previous events
return market.requested.map(request =>
PastStorageRequest(requestId: request.id,
ask: request.ask,
expiry: request.expiry)
)
method unsubscribe*(subscription: RequestSubscription) {.async.} =
subscription.market.subscriptions.onRequest.keepItIf(it != subscription)
method unsubscribe*(subscription: FulfillmentSubscription) {.async.} =
subscription.market.subscriptions.onFulfillment.keepItIf(it != subscription)
method unsubscribe*(subscription: SlotFilledSubscription) {.async.} =
subscription.market.subscriptions.onSlotFilled.keepItIf(it != subscription)
method unsubscribe*(subscription: SlotFreedSubscription) {.async.} =
subscription.market.subscriptions.onSlotFreed.keepItIf(it != subscription)
method unsubscribe*(subscription: RequestCancelledSubscription) {.async.} =
subscription.market.subscriptions.onRequestCancelled.keepItIf(it != subscription)
method unsubscribe*(subscription: RequestFailedSubscription) {.async.} =
subscription.market.subscriptions.onRequestFailed.keepItIf(it != subscription)
method unsubscribe*(subscription: ProofSubmittedSubscription) {.async.} =
subscription.market.subscriptions.onProofSubmitted.keepItIf(it != subscription)