fix(statemachine): do not raise from state.run (#1115)

* fix(statemachine): do not raise from state.run

* fix rebase

* fix exception handling in SaleProvingSimulated.prove

- re-raise CancelledError
- don't return State on CatchableError
- expect the Proofs_InvalidProof custom error instead of checking a string

* asyncSpawn salesagent.onCancelled

This was swallowing a KeyError in one of the tests (fixed in the previous commit)

* remove error handling states in asyncstatemachine

* revert unneeded changes

* formatting

* PR feedback, logging updates

codex/purchasing/states/cancelled.nim

@@ -1,25 +1,35 @@
 import pkg/metrics
 
 import ../../logutils
+import ../../utils/exceptions
 import ../statemachine
-import ./errorhandling
+import ./error
 
 declareCounter(codex_purchases_cancelled, "codex purchases cancelled")
 
 logScope:
   topics = "marketplace purchases cancelled"
 
-type PurchaseCancelled* = ref object of ErrorHandlingState
+type PurchaseCancelled* = ref object of PurchaseState
 
 method `$`*(state: PurchaseCancelled): string =
   "cancelled"
 
-method run*(state: PurchaseCancelled, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: PurchaseCancelled, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   codex_purchases_cancelled.inc()
   let purchase = Purchase(machine)
 
-  warn "Request cancelled, withdrawing remaining funds", requestId = purchase.requestId
+  try:
-  await purchase.market.withdrawFunds(purchase.requestId)
+    warn "Request cancelled, withdrawing remaining funds",
+      requestId = purchase.requestId
+    await purchase.market.withdrawFunds(purchase.requestId)
 
-  let error = newException(Timeout, "Purchase cancelled due to timeout")
+    let error = newException(Timeout, "Purchase cancelled due to timeout")
-  purchase.future.fail(error)
+    purchase.future.fail(error)
+  except CancelledError as e:
+    trace "PurchaseCancelled.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during PurchaseCancelled.run", error = e.msgDetail
+    return some State(PurchaseErrored(error: e))

codex/purchasing/states/error.nim

@@ -14,7 +14,9 @@ type PurchaseErrored* = ref object of PurchaseState
 method `$`*(state: PurchaseErrored): string =
   "errored"
 
-method run*(state: PurchaseErrored, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: PurchaseErrored, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   codex_purchases_error.inc()
   let purchase = Purchase(machine)
 

codex/purchasing/states/errorhandling.nim

@@ -1,8 +0,0 @@
-import pkg/questionable
-import ../statemachine
-import ./error
-
-type ErrorHandlingState* = ref object of PurchaseState
-
-method onError*(state: ErrorHandlingState, error: ref CatchableError): ?State =
-  some State(PurchaseErrored(error: error))

codex/purchasing/states/failed.nim

@@ -1,6 +1,7 @@
 import pkg/metrics
 import ../statemachine
 import ../../logutils
+import ../../utils/exceptions
 import ./error
 
 declareCounter(codex_purchases_failed, "codex purchases failed")
@@ -10,11 +11,20 @@ type PurchaseFailed* = ref object of PurchaseState
 method `$`*(state: PurchaseFailed): string =
   "failed"
 
-method run*(state: PurchaseFailed, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: PurchaseFailed, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   codex_purchases_failed.inc()
   let purchase = Purchase(machine)
-  warn "Request failed, withdrawing remaining funds", requestId = purchase.requestId
+
-  await purchase.market.withdrawFunds(purchase.requestId)
+  try:
+    warn "Request failed, withdrawing remaining funds", requestId = purchase.requestId
+    await purchase.market.withdrawFunds(purchase.requestId)
+  except CancelledError as e:
+    trace "PurchaseFailed.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during PurchaseFailed.run", error = e.msgDetail
+    return some State(PurchaseErrored(error: e))
 
   let error = newException(PurchaseError, "Purchase failed")
   return some State(PurchaseErrored(error: error))

codex/purchasing/states/finished.nim

@@ -1,7 +1,9 @@
 import pkg/metrics
 
 import ../statemachine
+import ../../utils/exceptions
 import ../../logutils
+import ./error
 
 declareCounter(codex_purchases_finished, "codex purchases finished")
 
@@ -13,10 +15,19 @@ type PurchaseFinished* = ref object of PurchaseState
 method `$`*(state: PurchaseFinished): string =
   "finished"
 
-method run*(state: PurchaseFinished, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: PurchaseFinished, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   codex_purchases_finished.inc()
   let purchase = Purchase(machine)
-  info "Purchase finished, withdrawing remaining funds", requestId = purchase.requestId
+  try:
-  await purchase.market.withdrawFunds(purchase.requestId)
+    info "Purchase finished, withdrawing remaining funds",
+      requestId = purchase.requestId
+    await purchase.market.withdrawFunds(purchase.requestId)
 
-  purchase.future.complete()
+    purchase.future.complete()
+  except CancelledError as e:
+    trace "PurchaseFinished.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during PurchaseFinished.run", error = e.msgDetail
+    return some State(PurchaseErrored(error: e))

codex/purchasing/states/pending.nim

@@ -1,18 +1,28 @@
 import pkg/metrics
+import ../../logutils
+import ../../utils/exceptions
 import ../statemachine
-import ./errorhandling
 import ./submitted
+import ./error
 
 declareCounter(codex_purchases_pending, "codex purchases pending")
 
-type PurchasePending* = ref object of ErrorHandlingState
+type PurchasePending* = ref object of PurchaseState
 
 method `$`*(state: PurchasePending): string =
   "pending"
 
-method run*(state: PurchasePending, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: PurchasePending, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   codex_purchases_pending.inc()
   let purchase = Purchase(machine)
-  let request = !purchase.request
+  try:
-  await purchase.market.requestStorage(request)
+    let request = !purchase.request
-  return some State(PurchaseSubmitted())
+    await purchase.market.requestStorage(request)
+    return some State(PurchaseSubmitted())
+  except CancelledError as e:
+    trace "PurchasePending.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during PurchasePending.run", error = e.msgDetail
+    return some State(PurchaseErrored(error: e))

codex/purchasing/states/started.nim

@@ -1,22 +1,25 @@
 import pkg/metrics
 
 import ../../logutils
+import ../../utils/exceptions
 import ../statemachine
-import ./errorhandling
 import ./finished
 import ./failed
+import ./error
 
 declareCounter(codex_purchases_started, "codex purchases started")
 
 logScope:
   topics = "marketplace purchases started"
 
-type PurchaseStarted* = ref object of ErrorHandlingState
+type PurchaseStarted* = ref object of PurchaseState
 
 method `$`*(state: PurchaseStarted): string =
   "started"
 
-method run*(state: PurchaseStarted, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: PurchaseStarted, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   codex_purchases_started.inc()
   let purchase = Purchase(machine)
 
@@ -28,15 +31,24 @@ method run*(state: PurchaseStarted, machine: Machine): Future[?State] {.async.}
   proc callback(_: RequestId) =
     failed.complete()
 
-  let subscription = await market.subscribeRequestFailed(purchase.requestId, callback)
+  var ended: Future[void]
+  try:
+    let subscription = await market.subscribeRequestFailed(purchase.requestId, callback)
 
-  # Ensure that we're past the request end by waiting an additional second
+    # Ensure that we're past the request end by waiting an additional second
-  let ended = clock.waitUntil((await market.getRequestEnd(purchase.requestId)) + 1)
+    ended = clock.waitUntil((await market.getRequestEnd(purchase.requestId)) + 1)
-  let fut = await one(ended, failed)
+    let fut = await one(ended, failed)
-  await subscription.unsubscribe()
+    await subscription.unsubscribe()
-  if fut.id == failed.id:
+    if fut.id == failed.id:
+      ended.cancelSoon()
+      return some State(PurchaseFailed())
+    else:
+      failed.cancelSoon()
+      return some State(PurchaseFinished())
+  except CancelledError as e:
     ended.cancelSoon()
-    return some State(PurchaseFailed())
-  else:
     failed.cancelSoon()
-    return some State(PurchaseFinished())
+    trace "PurchaseStarted.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during PurchaseStarted.run", error = e.msgDetail
+    return some State(PurchaseErrored(error: e))

codex/purchasing/states/submitted.nim

@@ -1,22 +1,25 @@
 import pkg/metrics
 
 import ../../logutils
+import ../../utils/exceptions
 import ../statemachine
-import ./errorhandling
 import ./started
 import ./cancelled
+import ./error
 
 logScope:
   topics = "marketplace purchases submitted"
 
 declareCounter(codex_purchases_submitted, "codex purchases submitted")
 
-type PurchaseSubmitted* = ref object of ErrorHandlingState
+type PurchaseSubmitted* = ref object of PurchaseState
 
 method `$`*(state: PurchaseSubmitted): string =
   "submitted"
 
-method run*(state: PurchaseSubmitted, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: PurchaseSubmitted, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   codex_purchases_submitted.inc()
   let purchase = Purchase(machine)
   let request = !purchase.request
@@ -44,5 +47,10 @@ method run*(state: PurchaseSubmitted, machine: Machine): Future[?State] {.async.
     await wait().withTimeout()
   except Timeout:
     return some State(PurchaseCancelled())
+  except CancelledError as e:
+    trace "PurchaseSubmitted.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during PurchaseSubmitted.run", error = e.msgDetail
+    return some State(PurchaseErrored(error: e))
 
   return some State(PurchaseStarted())

codex/purchasing/states/unknown.nim

@@ -1,34 +1,44 @@
 import pkg/metrics
+import ../../utils/exceptions
+import ../../logutils
 import ../statemachine
-import ./errorhandling
 import ./submitted
 import ./started
 import ./cancelled
 import ./finished
 import ./failed
+import ./error
 
 declareCounter(codex_purchases_unknown, "codex purchases unknown")
 
-type PurchaseUnknown* = ref object of ErrorHandlingState
+type PurchaseUnknown* = ref object of PurchaseState
 
 method `$`*(state: PurchaseUnknown): string =
   "unknown"
 
-method run*(state: PurchaseUnknown, machine: Machine): Future[?State] {.async.} =
+method run*(
-  codex_purchases_unknown.inc()
+    state: PurchaseUnknown, machine: Machine
-  let purchase = Purchase(machine)
+): Future[?State] {.async: (raises: []).} =
-  if (request =? await purchase.market.getRequest(purchase.requestId)) and
+  try:
-      (requestState =? await purchase.market.requestState(purchase.requestId)):
+    codex_purchases_unknown.inc()
-    purchase.request = some request
+    let purchase = Purchase(machine)
+    if (request =? await purchase.market.getRequest(purchase.requestId)) and
+        (requestState =? await purchase.market.requestState(purchase.requestId)):
+      purchase.request = some request
 
-    case requestState
+      case requestState
-    of RequestState.New:
+      of RequestState.New:
-      return some State(PurchaseSubmitted())
+        return some State(PurchaseSubmitted())
-    of RequestState.Started:
+      of RequestState.Started:
-      return some State(PurchaseStarted())
+        return some State(PurchaseStarted())
-    of RequestState.Cancelled:
+      of RequestState.Cancelled:
-      return some State(PurchaseCancelled())
+        return some State(PurchaseCancelled())
-    of RequestState.Finished:
+      of RequestState.Finished:
-      return some State(PurchaseFinished())
+        return some State(PurchaseFinished())
-    of RequestState.Failed:
+      of RequestState.Failed:
-      return some State(PurchaseFailed())
+        return some State(PurchaseFailed())
+  except CancelledError as e:
+    trace "PurchaseUnknown.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during PurchaseUnknown.run", error = e.msgDetail
+    return some State(PurchaseErrored(error: e))

codex/sales/salesagent.nim

@@ -6,6 +6,7 @@ import pkg/upraises
 import ../contracts/requests
 import ../errors
 import ../logutils
+import ../utils/exceptions
 import ./statemachine
 import ./salescontext
 import ./salesdata
@@ -68,41 +69,48 @@ proc subscribeCancellation(agent: SalesAgent) {.async.} =
   let data = agent.data
   let clock = agent.context.clock
 
-  proc onCancelled() {.async.} =
+  proc onCancelled() {.async: (raises: []).} =
     without request =? data.request:
       return
 
-    let market = agent.context.market
+    try:
-    let expiry = await market.requestExpiresAt(data.requestId)
+      let market = agent.context.market
+      let expiry = await market.requestExpiresAt(data.requestId)
 
-    while true:
+      while true:
-      let deadline = max(clock.now, expiry) + 1
+        let deadline = max(clock.now, expiry) + 1
-      trace "Waiting for request to be cancelled", now = clock.now, expiry = deadline
+        trace "Waiting for request to be cancelled", now = clock.now, expiry = deadline
-      await clock.waitUntil(deadline)
+        await clock.waitUntil(deadline)
 
-      without state =? await agent.retrieveRequestState():
+        without state =? await agent.retrieveRequestState():
-        error "Uknown request", requestId = data.requestId
+          error "Unknown request", requestId = data.requestId
-        return
+          return
 
-      case state
+        case state
-      of New:
+        of New:
-        discard
+          discard
-      of RequestState.Cancelled:
+        of RequestState.Cancelled:
-        agent.schedule(cancelledEvent(request))
+          agent.schedule(cancelledEvent(request))
-        break
+          break
-      of RequestState.Started, RequestState.Finished, RequestState.Failed:
+        of RequestState.Started, RequestState.Finished, RequestState.Failed:
-        break
+          break
 
-      debug "The request is not yet canceled, even though it should be. Waiting for some more time.",
+        debug "The request is not yet canceled, even though it should be. Waiting for some more time.",
-        currentState = state, now = clock.now
+          currentState = state, now = clock.now
+    except CancelledError:
+      trace "Waiting for expiry to lapse was cancelled", requestId = data.requestId
+    except CatchableError as e:
+      error "Error while waiting for expiry to lapse", error = e.msgDetail
 
   data.cancelled = onCancelled()
+  asyncSpawn data.cancelled
 
 method onFulfilled*(
     agent: SalesAgent, requestId: RequestId
 ) {.base, gcsafe, upraises: [].} =
-  if agent.data.requestId == requestId and not agent.data.cancelled.isNil:
+  let cancelled = agent.data.cancelled
-    agent.data.cancelled.cancelSoon()
+  if agent.data.requestId == requestId and not cancelled.isNil and not cancelled.finished:
+    cancelled.cancelSoon()
 
 method onFailed*(
     agent: SalesAgent, requestId: RequestId

codex/sales/states/cancelled.nim

@@ -1,17 +1,20 @@
 import ../../logutils
+import ../../utils/exceptions
 import ../salesagent
 import ../statemachine
-import ./errorhandling
+import ./errored
 
 logScope:
   topics = "marketplace sales cancelled"
 
-type SaleCancelled* = ref object of ErrorHandlingState
+type SaleCancelled* = ref object of SaleState
 
 method `$`*(state: SaleCancelled): string =
   "SaleCancelled"
 
-method run*(state: SaleCancelled, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: SaleCancelled, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   let agent = SalesAgent(machine)
   let data = agent.data
   let market = agent.context.market
@@ -19,21 +22,27 @@ method run*(state: SaleCancelled, machine: Machine): Future[?State] {.async.} =
   without request =? data.request:
     raiseAssert "no sale request"
 
-  let slot = Slot(request: request, slotIndex: data.slotIndex)
+  try:
-  debug "Collecting collateral and partial payout",
+    let slot = Slot(request: request, slotIndex: data.slotIndex)
-    requestId = data.requestId, slotIndex = data.slotIndex
+    debug "Collecting collateral and partial payout",
-  let currentCollateral = await market.currentCollateral(slot.id)
+      requestId = data.requestId, slotIndex = data.slotIndex
-  await market.freeSlot(slot.id)
+    let currentCollateral = await market.currentCollateral(slot.id)
+    await market.freeSlot(slot.id)
 
-  if onClear =? agent.context.onClear and request =? data.request:
+    if onClear =? agent.context.onClear and request =? data.request:
-    onClear(request, data.slotIndex)
+      onClear(request, data.slotIndex)
 
-  if onCleanUp =? agent.onCleanUp:
+    if onCleanUp =? agent.onCleanUp:
-    await onCleanUp(
+      await onCleanUp(
-      returnBytes = true,
+        returnBytes = true,
-      reprocessSlot = false,
+        reprocessSlot = false,
-      returnedCollateral = some currentCollateral,
+        returnedCollateral = some currentCollateral,
-    )
+      )
 
-  warn "Sale cancelled due to timeout",
+    warn "Sale cancelled due to timeout",
-    requestId = data.requestId, slotIndex = data.slotIndex
+      requestId = data.requestId, slotIndex = data.slotIndex
+  except CancelledError as e:
+    trace "SaleCancelled.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during SaleCancelled.run", error = e.msgDetail
+    return some State(SaleErrored(error: e))

codex/sales/states/downloading.nim

@@ -4,16 +4,16 @@ import pkg/questionable/results
 import ../../blocktype as bt
 import ../../logutils
 import ../../market
+import ../../utils/exceptions
 import ../salesagent
 import ../statemachine
-import ./errorhandling
 import ./cancelled
 import ./failed
 import ./filled
 import ./initialproving
 import ./errored
 
-type SaleDownloading* = ref object of ErrorHandlingState
+type SaleDownloading* = ref object of SaleState
 
 logScope:
   topics = "marketplace sales downloading"
@@ -32,7 +32,9 @@ method onSlotFilled*(
 ): ?State =
   return some State(SaleFilled())
 
-method run*(state: SaleDownloading, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: SaleDownloading, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   let agent = SalesAgent(machine)
   let data = agent.data
   let context = agent.context
@@ -64,9 +66,15 @@ method run*(state: SaleDownloading, machine: Machine): Future[?State] {.async.}
     trace "Releasing batch of bytes written to disk", bytes
     return await reservations.release(reservation.id, reservation.availabilityId, bytes)
 
-  trace "Starting download"
+  try:
-  if err =? (await onStore(request, data.slotIndex, onBlocks)).errorOption:
+    trace "Starting download"
-    return some State(SaleErrored(error: err, reprocessSlot: false))
+    if err =? (await onStore(request, data.slotIndex, onBlocks)).errorOption:
+      return some State(SaleErrored(error: err, reprocessSlot: false))
 
-  trace "Download complete"
+    trace "Download complete"
-  return some State(SaleInitialProving())
+    return some State(SaleInitialProving())
+  except CancelledError as e:
+    trace "SaleDownloading.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during SaleDownloading.run", error = e.msgDetail
+    return some State(SaleErrored(error: e))

codex/sales/states/errored.nim

@@ -17,10 +17,9 @@ type SaleErrored* = ref object of SaleState
 method `$`*(state: SaleErrored): string =
   "SaleErrored"
 
-method onError*(state: SaleState, err: ref CatchableError): ?State {.upraises: [].} =
+method run*(
-  error "error during SaleErrored run", error = err.msg
+    state: SaleErrored, machine: Machine
-
+): Future[?State] {.async: (raises: []).} =
-method run*(state: SaleErrored, machine: Machine): Future[?State] {.async.} =
   let agent = SalesAgent(machine)
   let data = agent.data
   let context = agent.context
@@ -30,8 +29,13 @@ method run*(state: SaleErrored, machine: Machine): Future[?State] {.async.} =
     requestId = data.requestId,
     slotIndex = data.slotIndex
 
-  if onClear =? context.onClear and request =? data.request:
+  try:
-    onClear(request, data.slotIndex)
+    if onClear =? context.onClear and request =? data.request:
+      onClear(request, data.slotIndex)
 
-  if onCleanUp =? agent.onCleanUp:
+    if onCleanUp =? agent.onCleanUp:
-    await onCleanUp(returnBytes = true, reprocessSlot = state.reprocessSlot)
+      await onCleanUp(returnBytes = true, reprocessSlot = state.reprocessSlot)
+  except CancelledError as e:
+    trace "SaleErrored.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during SaleErrored.run", error = e.msgDetail

codex/sales/states/errorhandling.nim

@@ -1,8 +0,0 @@
-import pkg/questionable
-import ../statemachine
-import ./errored
-
-type ErrorHandlingState* = ref object of SaleState
-
-method onError*(state: ErrorHandlingState, error: ref CatchableError): ?State =
-  some State(SaleErrored(error: error))

codex/sales/states/failed.nim

@@ -1,30 +1,39 @@
 import ../../logutils
+import ../../utils/exceptions
+import ../../utils/exceptions
 import ../salesagent
 import ../statemachine
-import ./errorhandling
 import ./errored
 
 logScope:
   topics = "marketplace sales failed"
 
 type
-  SaleFailed* = ref object of ErrorHandlingState
+  SaleFailed* = ref object of SaleState
   SaleFailedError* = object of SaleError
 
 method `$`*(state: SaleFailed): string =
   "SaleFailed"
 
-method run*(state: SaleFailed, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: SaleFailed, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   let data = SalesAgent(machine).data
   let market = SalesAgent(machine).context.market
 
   without request =? data.request:
     raiseAssert "no sale request"
 
-  let slot = Slot(request: request, slotIndex: data.slotIndex)
+  try:
-  debug "Removing slot from mySlots",
+    let slot = Slot(request: request, slotIndex: data.slotIndex)
-    requestId = data.requestId, slotIndex = data.slotIndex
+    debug "Removing slot from mySlots",
-  await market.freeSlot(slot.id)
+      requestId = data.requestId, slotIndex = data.slotIndex
+    await market.freeSlot(slot.id)
 
-  let error = newException(SaleFailedError, "Sale failed")
+    let error = newException(SaleFailedError, "Sale failed")
-  return some State(SaleErrored(error: error))
+    return some State(SaleErrored(error: error))
+  except CancelledError as e:
+    trace "SaleFailed.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during SaleFailed.run", error = e.msgDetail
+    return some State(SaleErrored(error: e))

codex/sales/states/filled.nim

@@ -3,9 +3,9 @@ import pkg/questionable/results
 
 import ../../conf
 import ../../logutils
+import ../../utils/exceptions
 import ../statemachine
 import ../salesagent
-import ./errorhandling
 import ./errored
 import ./cancelled
 import ./failed
@@ -18,7 +18,7 @@ logScope:
   topics = "marketplace sales filled"
 
 type
-  SaleFilled* = ref object of ErrorHandlingState
+  SaleFilled* = ref object of SaleState
   HostMismatchError* = object of CatchableError
 
 method onCancelled*(state: SaleFilled, request: StorageRequest): ?State =
@@ -30,40 +30,48 @@ method onFailed*(state: SaleFilled, request: StorageRequest): ?State =
 method `$`*(state: SaleFilled): string =
   "SaleFilled"
 
-method run*(state: SaleFilled, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: SaleFilled, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   let agent = SalesAgent(machine)
   let data = agent.data
   let context = agent.context
-
   let market = context.market
-  let host = await market.getHost(data.requestId, data.slotIndex)
-  let me = await market.getSigner()
 
-  if host == me.some:
+  try:
-    info "Slot succesfully filled",
+    let host = await market.getHost(data.requestId, data.slotIndex)
-      requestId = data.requestId, slotIndex = data.slotIndex
+    let me = await market.getSigner()
 
-    without request =? data.request:
+    if host == me.some:
-      raiseAssert "no sale request"
+      info "Slot succesfully filled",
+        requestId = data.requestId, slotIndex = data.slotIndex
 
-    if onFilled =? agent.onFilled:
+      without request =? data.request:
-      onFilled(request, data.slotIndex)
+        raiseAssert "no sale request"
 
-    without onExpiryUpdate =? context.onExpiryUpdate:
+      if onFilled =? agent.onFilled:
-      raiseAssert "onExpiryUpdate callback not set"
+        onFilled(request, data.slotIndex)
 
-    let requestEnd = await market.getRequestEnd(data.requestId)
+      without onExpiryUpdate =? context.onExpiryUpdate:
-    if err =? (await onExpiryUpdate(request.content.cid, requestEnd)).errorOption:
+        raiseAssert "onExpiryUpdate callback not set"
-      return some State(SaleErrored(error: err))
 
-    when codex_enable_proof_failures:
+      let requestEnd = await market.getRequestEnd(data.requestId)
-      if context.simulateProofFailures > 0:
+      if err =? (await onExpiryUpdate(request.content.cid, requestEnd)).errorOption:
-        info "Proving with failure rate", rate = context.simulateProofFailures
+        return some State(SaleErrored(error: err))
-        return some State(
-          SaleProvingSimulated(failEveryNProofs: context.simulateProofFailures)
-        )
 
-    return some State(SaleProving())
+      when codex_enable_proof_failures:
-  else:
+        if context.simulateProofFailures > 0:
-    let error = newException(HostMismatchError, "Slot filled by other host")
+          info "Proving with failure rate", rate = context.simulateProofFailures
-    return some State(SaleErrored(error: error))
+          return some State(
+            SaleProvingSimulated(failEveryNProofs: context.simulateProofFailures)
+          )
+
+      return some State(SaleProving())
+    else:
+      let error = newException(HostMismatchError, "Slot filled by other host")
+      return some State(SaleErrored(error: error))
+  except CancelledError as e:
+    trace "SaleFilled.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during SaleFilled.run", error = e.msgDetail
+    return some State(SaleErrored(error: e))

codex/sales/states/filling.nim

@@ -1,9 +1,9 @@
 import pkg/stint
 import ../../logutils
 import ../../market
+import ../../utils/exceptions
 import ../statemachine
 import ../salesagent
-import ./errorhandling
 import ./filled
 import ./cancelled
 import ./failed
@@ -13,7 +13,7 @@ import ./errored
 logScope:
   topics = "marketplace sales filling"
 
-type SaleFilling* = ref object of ErrorHandlingState
+type SaleFilling* = ref object of SaleState
   proof*: Groth16Proof
 
 method `$`*(state: SaleFilling): string =
@@ -25,7 +25,9 @@ method onCancelled*(state: SaleFilling, request: StorageRequest): ?State =
 method onFailed*(state: SaleFilling, request: StorageRequest): ?State =
   return some State(SaleFailed())
 
-method run(state: SaleFilling, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: SaleFilling, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   let data = SalesAgent(machine).data
   let market = SalesAgent(machine).context.market
   without (request =? data.request):
@@ -35,28 +37,34 @@ method run(state: SaleFilling, machine: Machine): Future[?State] {.async.} =
     requestId = data.requestId
     slotIndex = data.slotIndex
 
-  let slotState = await market.slotState(slotId(data.requestId, data.slotIndex))
-  let requestedCollateral = request.ask.collateralPerSlot
-  var collateral: UInt256
-
-  if slotState == SlotState.Repair:
-    # When repairing the node gets "discount" on the collateral that it needs to
-    let repairRewardPercentage = (await market.repairRewardPercentage).u256
-    collateral =
-      requestedCollateral -
-      ((requestedCollateral * repairRewardPercentage)).div(100.u256)
-  else:
-    collateral = requestedCollateral
-
-  debug "Filling slot"
   try:
-    await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral)
+    let slotState = await market.slotState(slotId(data.requestId, data.slotIndex))
-  except MarketError as e:
+    let requestedCollateral = request.ask.collateralPerSlot
-    if e.msg.contains "Slot is not free":
+    var collateral: UInt256
-      debug "Slot is already filled, ignoring slot"
-      return some State(SaleIgnored(reprocessSlot: false, returnBytes: true))
-    else:
-      return some State(SaleErrored(error: e))
-  # other CatchableErrors are handled "automatically" by the ErrorHandlingState
 
-  return some State(SaleFilled())
+    if slotState == SlotState.Repair:
+      # When repairing the node gets "discount" on the collateral that it needs to
+      let repairRewardPercentage = (await market.repairRewardPercentage).u256
+      collateral =
+        requestedCollateral -
+        ((requestedCollateral * repairRewardPercentage)).div(100.u256)
+    else:
+      collateral = requestedCollateral
+
+    debug "Filling slot"
+    try:
+      await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral)
+    except MarketError as e:
+      if e.msg.contains "Slot is not free":
+        debug "Slot is already filled, ignoring slot"
+        return some State(SaleIgnored(reprocessSlot: false, returnBytes: true))
+      else:
+        return some State(SaleErrored(error: e))
+    # other CatchableErrors are handled "automatically" by the SaleState
+
+    return some State(SaleFilled())
+  except CancelledError as e:
+    trace "SaleFilling.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during SaleFilling.run", error = e.msgDetail
+    return some State(SaleErrored(error: e))

codex/sales/states/finished.nim

@@ -1,16 +1,17 @@
 import pkg/chronos
 
 import ../../logutils
+import ../../utils/exceptions
 import ../statemachine
 import ../salesagent
-import ./errorhandling
 import ./cancelled
 import ./failed
+import ./errored
 
 logScope:
   topics = "marketplace sales finished"
 
-type SaleFinished* = ref object of ErrorHandlingState
+type SaleFinished* = ref object of SaleState
   returnedCollateral*: ?UInt256
 
 method `$`*(state: SaleFinished): string =
@@ -22,7 +23,9 @@ method onCancelled*(state: SaleFinished, request: StorageRequest): ?State =
 method onFailed*(state: SaleFinished, request: StorageRequest): ?State =
   return some State(SaleFailed())
 
-method run*(state: SaleFinished, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: SaleFinished, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   let agent = SalesAgent(machine)
   let data = agent.data
 
@@ -32,5 +35,11 @@ method run*(state: SaleFinished, machine: Machine): Future[?State] {.async.} =
   info "Slot finished and paid out",
     requestId = data.requestId, slotIndex = data.slotIndex
 
-  if onCleanUp =? agent.onCleanUp:
+  try:
-    await onCleanUp(returnedCollateral = state.returnedCollateral)
+    if onCleanUp =? agent.onCleanUp:
+      await onCleanUp(returnedCollateral = state.returnedCollateral)
+  except CancelledError as e:
+    trace "SaleFilled.run onCleanUp was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during SaleFilled.run in onCleanUp callback", error = e.msgDetail
+    return some State(SaleErrored(error: e))

codex/sales/states/ignored.nim

@@ -1,9 +1,10 @@
 import pkg/chronos
 
 import ../../logutils
+import ../../utils/exceptions
 import ../statemachine
 import ../salesagent
-import ./errorhandling
+import ./errored
 
 logScope:
   topics = "marketplace sales ignored"
@@ -11,17 +12,25 @@ logScope:
 # Ignored slots could mean there was no availability or that the slot could
 # not be reserved.
 
-type SaleIgnored* = ref object of ErrorHandlingState
+type SaleIgnored* = ref object of SaleState
   reprocessSlot*: bool # readd slot to queue with `seen` flag
   returnBytes*: bool # return unreleased bytes from Reservation to Availability
 
 method `$`*(state: SaleIgnored): string =
   "SaleIgnored"
 
-method run*(state: SaleIgnored, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: SaleIgnored, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   let agent = SalesAgent(machine)
 
-  if onCleanUp =? agent.onCleanUp:
+  try:
-    await onCleanUp(
+    if onCleanUp =? agent.onCleanUp:
-      reprocessSlot = state.reprocessSlot, returnBytes = state.returnBytes
+      await onCleanUp(
-    )
+        reprocessSlot = state.reprocessSlot, returnBytes = state.returnBytes
+      )
+  except CancelledError as e:
+    trace "SaleIgnored.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during SaleIgnored.run in onCleanUp", error = e.msgDetail
+    return some State(SaleErrored(error: e))

codex/sales/states/initialproving.nim

@@ -1,9 +1,9 @@
 import pkg/questionable/results
 import ../../clock
 import ../../logutils
+import ../../utils/exceptions
 import ../statemachine
 import ../salesagent
-import ./errorhandling
 import ./filling
 import ./cancelled
 import ./errored
@@ -12,7 +12,7 @@ import ./failed
 logScope:
   topics = "marketplace sales initial-proving"
 
-type SaleInitialProving* = ref object of ErrorHandlingState
+type SaleInitialProving* = ref object of SaleState
 
 method `$`*(state: SaleInitialProving): string =
   "SaleInitialProving"
@@ -36,7 +36,9 @@ proc waitForStableChallenge(market: Market, clock: Clock, slotId: SlotId) {.asyn
   while (await market.getPointer(slotId)) > (256 - downtime):
     await clock.waitUntilNextPeriod(periodicity)
 
-method run*(state: SaleInitialProving, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: SaleInitialProving, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   let data = SalesAgent(machine).data
   let context = SalesAgent(machine).context
   let market = context.market
@@ -48,16 +50,22 @@ method run*(state: SaleInitialProving, machine: Machine): Future[?State] {.async
   without onProve =? context.onProve:
     raiseAssert "onProve callback not set"
 
-  debug "Waiting for a proof challenge that is valid for the entire period"
+  try:
-  let slot = Slot(request: request, slotIndex: data.slotIndex)
+    debug "Waiting for a proof challenge that is valid for the entire period"
-  await waitForStableChallenge(market, clock, slot.id)
+    let slot = Slot(request: request, slotIndex: data.slotIndex)
+    await waitForStableChallenge(market, clock, slot.id)
 
-  debug "Generating initial proof", requestId = data.requestId
+    debug "Generating initial proof", requestId = data.requestId
-  let challenge = await context.market.getChallenge(slot.id)
+    let challenge = await context.market.getChallenge(slot.id)
-  without proof =? (await onProve(slot, challenge)), err:
+    without proof =? (await onProve(slot, challenge)), err:
-    error "Failed to generate initial proof", error = err.msg
+      error "Failed to generate initial proof", error = err.msg
-    return some State(SaleErrored(error: err))
+      return some State(SaleErrored(error: err))
 
-  debug "Finished proof calculation", requestId = data.requestId
+    debug "Finished proof calculation", requestId = data.requestId
 
-  return some State(SaleFilling(proof: proof))
+    return some State(SaleFilling(proof: proof))
+  except CancelledError as e:
+    trace "SaleInitialProving.run onCleanUp was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during SaleInitialProving.run", error = e.msgDetail
+    return some State(SaleErrored(error: e))

codex/sales/states/payout.nim

@@ -1,16 +1,17 @@
 import ../../logutils
 import ../../market
+import ../../utils/exceptions
 import ../statemachine
 import ../salesagent
-import ./errorhandling
 import ./cancelled
 import ./failed
 import ./finished
+import ./errored
 
 logScope:
   topics = "marketplace sales payout"
 
-type SalePayout* = ref object of ErrorHandlingState
+type SalePayout* = ref object of SaleState
 
 method `$`*(state: SalePayout): string =
   "SalePayout"
@@ -21,17 +22,25 @@ method onCancelled*(state: SalePayout, request: StorageRequest): ?State =
 method onFailed*(state: SalePayout, request: StorageRequest): ?State =
   return some State(SaleFailed())
 
-method run*(state: SalePayout, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: SalePayout, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   let data = SalesAgent(machine).data
   let market = SalesAgent(machine).context.market
 
   without request =? data.request:
     raiseAssert "no sale request"
 
-  let slot = Slot(request: request, slotIndex: data.slotIndex)
+  try:
-  debug "Collecting finished slot's reward",
+    let slot = Slot(request: request, slotIndex: data.slotIndex)
-    requestId = data.requestId, slotIndex = data.slotIndex
+    debug "Collecting finished slot's reward",
-  let currentCollateral = await market.currentCollateral(slot.id)
+      requestId = data.requestId, slotIndex = data.slotIndex
-  await market.freeSlot(slot.id)
+    let currentCollateral = await market.currentCollateral(slot.id)
+    await market.freeSlot(slot.id)
 
-  return some State(SaleFinished(returnedCollateral: some currentCollateral))
+    return some State(SaleFinished(returnedCollateral: some currentCollateral))
+  except CancelledError as e:
+    trace "SalePayout.run onCleanUp was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during SalePayout.run", error = e.msgDetail
+    return some State(SaleErrored(error: e))

codex/sales/states/preparing.nim

@@ -4,9 +4,9 @@ import pkg/metrics
 
 import ../../logutils
 import ../../market
+import ../../utils/exceptions
 import ../salesagent
 import ../statemachine
-import ./errorhandling
 import ./cancelled
 import ./failed
 import ./filled
@@ -18,7 +18,7 @@ declareCounter(
   codex_reservations_availability_mismatch, "codex reservations availability_mismatch"
 )
 
-type SalePreparing* = ref object of ErrorHandlingState
+type SalePreparing* = ref object of SaleState
 
 logScope:
   topics = "marketplace sales preparing"
@@ -37,62 +37,70 @@ method onSlotFilled*(
 ): ?State =
   return some State(SaleFilled())
 
-method run*(state: SalePreparing, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: SalePreparing, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   let agent = SalesAgent(machine)
   let data = agent.data
   let context = agent.context
   let market = context.market
   let reservations = context.reservations
 
-  await agent.retrieveRequest()
+  try:
-  await agent.subscribe()
+    await agent.retrieveRequest()
+    await agent.subscribe()
 
-  without request =? data.request:
+    without request =? data.request:
-    raiseAssert "no sale request"
+      raiseAssert "no sale request"
 
-  let slotId = slotId(data.requestId, data.slotIndex)
+    let slotId = slotId(data.requestId, data.slotIndex)
-  let state = await market.slotState(slotId)
+    let state = await market.slotState(slotId)
-  if state != SlotState.Free and state != SlotState.Repair:
+    if state != SlotState.Free and state != SlotState.Repair:
-    return some State(SaleIgnored(reprocessSlot: false, returnBytes: false))
+      return some State(SaleIgnored(reprocessSlot: false, returnBytes: false))
 
-  # TODO: Once implemented, check to ensure the host is allowed to fill the slot,
+    # TODO: Once implemented, check to ensure the host is allowed to fill the slot,
-  # due to the [sliding window mechanism](https://github.com/codex-storage/codex-research/blob/master/design/marketplace.md#dispersal)
+    # due to the [sliding window mechanism](https://github.com/codex-storage/codex-research/blob/master/design/marketplace.md#dispersal)
 
-  logScope:
+    logScope:
-    slotIndex = data.slotIndex
+      slotIndex = data.slotIndex
-    slotSize = request.ask.slotSize
+      slotSize = request.ask.slotSize
-    duration = request.ask.duration
+      duration = request.ask.duration
-    pricePerBytePerSecond = request.ask.pricePerBytePerSecond
+      pricePerBytePerSecond = request.ask.pricePerBytePerSecond
-    collateralPerByte = request.ask.collateralPerByte
+      collateralPerByte = request.ask.collateralPerByte
 
-  without availability =?
+    without availability =?
-    await reservations.findAvailability(
+      await reservations.findAvailability(
-      request.ask.slotSize, request.ask.duration, request.ask.pricePerBytePerSecond,
+        request.ask.slotSize, request.ask.duration, request.ask.pricePerBytePerSecond,
-      request.ask.collateralPerByte,
+        request.ask.collateralPerByte,
-    ):
+      ):
-    debug "No availability found for request, ignoring"
+      debug "No availability found for request, ignoring"
 
-    return some State(SaleIgnored(reprocessSlot: true))
-
-  info "Availability found for request, creating reservation"
-
-  without reservation =?
-    await reservations.createReservation(
-      availability.id, request.ask.slotSize, request.id, data.slotIndex,
-      request.ask.collateralPerByte,
-    ), error:
-    trace "Creation of reservation failed"
-    # Race condition:
-    # reservations.findAvailability (line 64) is no guarantee. You can never know for certain that the reservation can be created until after you have it.
-    # Should createReservation fail because there's no space, we proceed to SaleIgnored.
-    if error of BytesOutOfBoundsError:
-      # Lets monitor how often this happen and if it is often we can make it more inteligent to handle it
-      codex_reservations_availability_mismatch.inc()
       return some State(SaleIgnored(reprocessSlot: true))
 
-    return some State(SaleErrored(error: error))
+    info "Availability found for request, creating reservation"
 
-  trace "Reservation created succesfully"
+    without reservation =?
+      await reservations.createReservation(
+        availability.id, request.ask.slotSize, request.id, data.slotIndex,
+        request.ask.collateralPerByte,
+      ), error:
+      trace "Creation of reservation failed"
+      # Race condition:
+      # reservations.findAvailability (line 64) is no guarantee. You can never know for certain that the reservation can be created until after you have it.
+      # Should createReservation fail because there's no space, we proceed to SaleIgnored.
+      if error of BytesOutOfBoundsError:
+        # Lets monitor how often this happen and if it is often we can make it more inteligent to handle it
+        codex_reservations_availability_mismatch.inc()
+        return some State(SaleIgnored(reprocessSlot: true))
 
-  data.reservation = some reservation
+      return some State(SaleErrored(error: error))
-  return some State(SaleSlotReserving())
+
+    trace "Reservation created successfully"
+
+    data.reservation = some reservation
+    return some State(SaleSlotReserving())
+  except CancelledError as e:
+    trace "SalePreparing.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during SalePreparing.run", error = e.msgDetail
+    return some State(SaleErrored(error: e))

codex/sales/states/proving.nim

@@ -6,7 +6,6 @@ import ../../utils/exceptions
 import ../statemachine
 import ../salesagent
 import ../salescontext
-import ./errorhandling
 import ./cancelled
 import ./failed
 import ./errored
@@ -18,7 +17,7 @@ logScope:
 type
   SlotFreedError* = object of CatchableError
   SlotNotFilledError* = object of CatchableError
-  SaleProving* = ref object of ErrorHandlingState
+  SaleProving* = ref object of SaleState
     loop: Future[void]
 
 method prove*(
@@ -113,7 +112,9 @@ method onFailed*(state: SaleProving, request: StorageRequest): ?State =
   # state change
   return some State(SaleFailed())
 
-method run*(state: SaleProving, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: SaleProving, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   let data = SalesAgent(machine).data
   let context = SalesAgent(machine).context
 
@@ -129,27 +130,37 @@ method run*(state: SaleProving, machine: Machine): Future[?State] {.async.} =
   without clock =? context.clock:
     raiseAssert("clock not set")
 
-  debug "Start proving", requestId = data.requestId, slotIndex = data.slotIndex
   try:
-    let loop = state.proveLoop(market, clock, request, data.slotIndex, onProve)
+    debug "Start proving", requestId = data.requestId, slotIndex = data.slotIndex
-    state.loop = loop
+    try:
-    await loop
+      let loop = state.proveLoop(market, clock, request, data.slotIndex, onProve)
-  except CancelledError:
+      state.loop = loop
-    discard
+      await loop
+    except CancelledError as e:
+      trace "proving loop cancelled"
+      discard
+    except CatchableError as e:
+      error "Proving failed",
+        msg = e.msg, typ = $(type e), stack = e.getStackTrace(), error = e.msgDetail
+      return some State(SaleErrored(error: e))
+    finally:
+      # Cleanup of the proving loop
+      debug "Stopping proving.", requestId = data.requestId, slotIndex = data.slotIndex
+
+      if not state.loop.isNil:
+        if not state.loop.finished:
+          try:
+            await state.loop.cancelAndWait()
+          except CancelledError:
+            discard
+          except CatchableError as e:
+            error "Error during cancellation of proving loop", msg = e.msg
+
+        state.loop = nil
+
+    return some State(SalePayout())
+  except CancelledError as e:
+    trace "SaleProving.run onCleanUp was cancelled", error = e.msgDetail
   except CatchableError as e:
-    error "Proving failed", msg = e.msg
+    error "Error during SaleProving.run", error = e.msgDetail
     return some State(SaleErrored(error: e))
-  finally:
-    # Cleanup of the proving loop
-    debug "Stopping proving.", requestId = data.requestId, slotIndex = data.slotIndex
-
-    if not state.loop.isNil:
-      if not state.loop.finished:
-        try:
-          await state.loop.cancelAndWait()
-        except CatchableError as e:
-          error "Error during cancellation of proving loop", msg = e.msg
-
-      state.loop = nil
-
-  return some State(SalePayout())

codex/sales/states/provingsimulated.nim

@@ -4,12 +4,14 @@ when codex_enable_proof_failures:
   import pkg/stint
   import pkg/ethers
 
+  import ../../contracts/marketplace
   import ../../contracts/requests
   import ../../logutils
   import ../../market
   import ../../utils/exceptions
   import ../salescontext
   import ./proving
+  import ./errored
 
   logScope:
     topics = "marketplace sales simulated-proving"
@@ -29,22 +31,27 @@ when codex_enable_proof_failures:
       market: Market,
       currentPeriod: Period,
   ) {.async.} =
-    trace "Processing proving in simulated mode"
+    try:
-    state.proofCount += 1
+      trace "Processing proving in simulated mode"
-    if state.failEveryNProofs > 0 and state.proofCount mod state.failEveryNProofs == 0:
+      state.proofCount += 1
-      state.proofCount = 0
+      if state.failEveryNProofs > 0 and state.proofCount mod state.failEveryNProofs == 0:
+        state.proofCount = 0
 
-      try:
+        try:
-        warn "Submitting INVALID proof", period = currentPeriod, slotId = slot.id
+          warn "Submitting INVALID proof", period = currentPeriod, slotId = slot.id
-        await market.submitProof(slot.id, Groth16Proof.default)
+          await market.submitProof(slot.id, Groth16Proof.default)
-      except MarketError as e:
+        except Proofs_InvalidProof as e:
-        if not e.msg.contains("Invalid proof"):
+          discard # expected
+        except CancelledError as error:
+          raise error
+        except CatchableError as e:
           onSubmitProofError(e, currentPeriod, slot.id)
-      except CancelledError as error:
+      else:
-        raise error
+        await procCall SaleProving(state).prove(
-      except CatchableError as e:
+          slot, challenge, onProve, market, currentPeriod
-        onSubmitProofError(e, currentPeriod, slot.id)
+        )
-    else:
+    except CancelledError as e:
-      await procCall SaleProving(state).prove(
+      trace "Submitting INVALID proof cancelled", error = e.msgDetail
-        slot, challenge, onProve, market, currentPeriod
+      raise e
-      )
+    except CatchableError as e:
+      error "Submitting INVALID proof failed", error = e.msgDetail

codex/sales/states/slotreserving.nim

@@ -3,16 +3,16 @@ import pkg/metrics
 
 import ../../logutils
 import ../../market
+import ../../utils/exceptions
 import ../salesagent
 import ../statemachine
-import ./errorhandling
 import ./cancelled
 import ./failed
 import ./ignored
 import ./downloading
 import ./errored
 
-type SaleSlotReserving* = ref object of ErrorHandlingState
+type SaleSlotReserving* = ref object of SaleState
 
 logScope:
   topics = "marketplace sales reserving"
@@ -26,7 +26,9 @@ method onCancelled*(state: SaleSlotReserving, request: StorageRequest): ?State =
 method onFailed*(state: SaleSlotReserving, request: StorageRequest): ?State =
   return some State(SaleFailed())
 
-method run*(state: SaleSlotReserving, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: SaleSlotReserving, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   let agent = SalesAgent(machine)
   let data = agent.data
   let context = agent.context
@@ -36,23 +38,29 @@ method run*(state: SaleSlotReserving, machine: Machine): Future[?State] {.async.
     requestId = data.requestId
     slotIndex = data.slotIndex
 
-  let canReserve = await market.canReserveSlot(data.requestId, data.slotIndex)
+  try:
-  if canReserve:
+    let canReserve = await market.canReserveSlot(data.requestId, data.slotIndex)
-    try:
+    if canReserve:
-      trace "Reserving slot"
+      try:
-      await market.reserveSlot(data.requestId, data.slotIndex)
+        trace "Reserving slot"
-    except MarketError as e:
+        await market.reserveSlot(data.requestId, data.slotIndex)
-      if e.msg.contains "SlotReservations_ReservationNotAllowed":
+      except MarketError as e:
-        debug "Slot cannot be reserved, ignoring", error = e.msg
+        if e.msg.contains "SlotReservations_ReservationNotAllowed":
-        return some State(SaleIgnored(reprocessSlot: false, returnBytes: true))
+          debug "Slot cannot be reserved, ignoring", error = e.msg
-      else:
+          return some State(SaleIgnored(reprocessSlot: false, returnBytes: true))
-        return some State(SaleErrored(error: e))
+        else:
-    # other CatchableErrors are handled "automatically" by the ErrorHandlingState
+          return some State(SaleErrored(error: e))
+      # other CatchableErrors are handled "automatically" by the SaleState
 
-    trace "Slot successfully reserved"
+      trace "Slot successfully reserved"
-    return some State(SaleDownloading())
+      return some State(SaleDownloading())
-  else:
+    else:
-    # do not re-add this slot to the queue, and return bytes from Reservation to
+      # do not re-add this slot to the queue, and return bytes from Reservation to
-    # the Availability
+      # the Availability
-    debug "Slot cannot be reserved, ignoring"
+      debug "Slot cannot be reserved, ignoring"
-    return some State(SaleIgnored(reprocessSlot: false, returnBytes: true))
+      return some State(SaleIgnored(reprocessSlot: false, returnBytes: true))
+  except CancelledError as e:
+    trace "SaleSlotReserving.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during SaleSlotReserving.run", error = e.msgDetail
+    return some State(SaleErrored(error: e))

codex/sales/states/unknown.nim

@@ -1,4 +1,5 @@
 import ../../logutils
+import ../../utils/exceptions
 import ../statemachine
 import ../salesagent
 import ./filled
@@ -26,34 +27,42 @@ method onCancelled*(state: SaleUnknown, request: StorageRequest): ?State =
 method onFailed*(state: SaleUnknown, request: StorageRequest): ?State =
   return some State(SaleFailed())
 
-method run*(state: SaleUnknown, machine: Machine): Future[?State] {.async.} =
+method run*(
+    state: SaleUnknown, machine: Machine
+): Future[?State] {.async: (raises: []).} =
   let agent = SalesAgent(machine)
   let data = agent.data
   let market = agent.context.market
 
-  await agent.retrieveRequest()
+  try:
-  await agent.subscribe()
+    await agent.retrieveRequest()
+    await agent.subscribe()
 
-  let slotId = slotId(data.requestId, data.slotIndex)
+    let slotId = slotId(data.requestId, data.slotIndex)
-  let slotState = await market.slotState(slotId)
+    let slotState = await market.slotState(slotId)
 
-  case slotState
+    case slotState
-  of SlotState.Free:
+    of SlotState.Free:
-    let error =
+      let error =
-      newException(UnexpectedSlotError, "Slot state on chain should not be 'free'")
+        newException(UnexpectedSlotError, "Slot state on chain should not be 'free'")
-    return some State(SaleErrored(error: error))
+      return some State(SaleErrored(error: error))
-  of SlotState.Filled:
+    of SlotState.Filled:
-    return some State(SaleFilled())
+      return some State(SaleFilled())
-  of SlotState.Finished:
+    of SlotState.Finished:
-    return some State(SalePayout())
+      return some State(SalePayout())
-  of SlotState.Paid:
+    of SlotState.Paid:
-    return some State(SaleFinished())
+      return some State(SaleFinished())
-  of SlotState.Failed:
+    of SlotState.Failed:
-    return some State(SaleFailed())
+      return some State(SaleFailed())
-  of SlotState.Cancelled:
+    of SlotState.Cancelled:
-    return some State(SaleCancelled())
+      return some State(SaleCancelled())
-  of SlotState.Repair:
+    of SlotState.Repair:
-    let error = newException(
+      let error = newException(
-      SlotFreedError, "Slot was forcible freed and host was removed from its hosting"
+        SlotFreedError, "Slot was forcible freed and host was removed from its hosting"
-    )
+      )
-    return some State(SaleErrored(error: error))
+      return some State(SaleErrored(error: error))
+  except CancelledError as e:
+    trace "SaleUnknown.run was cancelled", error = e.msgDetail
+  except CatchableError as e:
+    error "Error during SaleUnknown.run", error = e.msgDetail
+    return some State(SaleErrored(error: e))

codex/utils/asyncstatemachine.nim

@@ -2,6 +2,7 @@ import pkg/questionable
 import pkg/chronos
 import ../logutils
 import ./trackedfutures
+import ./exceptions
 
 {.push raises: [].}
 
@@ -46,24 +47,14 @@ proc schedule*(machine: Machine, event: Event) =
   except AsyncQueueFullError:
     raiseAssert "unlimited queue is full?!"
 
-method run*(state: State, machine: Machine): Future[?State] {.base, async.} =
+method run*(
+    state: State, machine: Machine
+): Future[?State] {.base, async: (raises: []).} =
   discard
 
-method onError*(state: State, error: ref CatchableError): ?State {.base.} =
-  raise (ref Defect)(msg: "error in state machine: " & error.msg, parent: error)
-
-proc onError(machine: Machine, error: ref CatchableError): Event =
-  return proc(state: State): ?State =
-    state.onError(error)
-
 proc run(machine: Machine, state: State) {.async: (raises: []).} =
-  try:
+  if next =? await state.run(machine):
-    if next =? await state.run(machine):
+    machine.schedule(Event.transition(state, next))
-      machine.schedule(Event.transition(state, next))
-  except CancelledError:
-    discard # do not propagate
-  except CatchableError as e:
-    machine.schedule(machine.onError(e))
 
 proc scheduler(machine: Machine) {.async: (raises: []).} =
   var running: Future[void].Raising([])

tests/codex/sales/testsales.nim

@@ -36,6 +36,7 @@ asyncchecksuite "Sales - start":
   var repo: RepoStore
   var queue: SlotQueue
   var itemsProcessed: seq[SlotQueueItem]
+  var expiry: SecondsSince1970
 
   setup:
     request = StorageRequest(
@@ -76,7 +77,8 @@ asyncchecksuite "Sales - start":
     ): Future[?!Groth16Proof] {.async.} =
       return success(proof)
     itemsProcessed = @[]
-    request.expiry = (clock.now() + 42).u256
+    expiry = (clock.now() + 42)
+    request.expiry = expiry.u256
 
   teardown:
     await sales.stop()
@@ -97,6 +99,7 @@ asyncchecksuite "Sales - start":
     request.ask.slots = 2
     market.requested = @[request]
     market.requestState[request.id] = RequestState.New
+    market.requestExpiry[request.id] = expiry
 
     let slot0 =
       MockSlot(requestId: request.id, slotIndex: 0.u256, proof: proof, host: me)
@@ -430,23 +433,6 @@ asyncchecksuite "Sales":
     check eventually storingRequest == request
     check storingSlot < request.ask.slots.u256
 
-  test "handles errors during state run":
-    var saleFailed = false
-    sales.onProve = proc(
-        slot: Slot, challenge: ProofChallenge
-    ): Future[?!Groth16Proof] {.async.} =
-      # raise exception so machine.onError is called
-      raise newException(ValueError, "some error")
-
-    # onClear is called in SaleErrored.run
-    sales.onClear = proc(request: StorageRequest, idx: UInt256) =
-      saleFailed = true
-    createAvailability()
-    await market.requestStorage(request)
-    await allowRequestToStart()
-
-    check eventually saleFailed
-
   test "makes storage available again when data retrieval fails":
     let error = newException(IOError, "data retrieval failed")
     sales.onStore = proc(

tests/codex/sales/testsalesagent.nim

@@ -4,7 +4,6 @@ import pkg/codex/sales
 import pkg/codex/sales/salesagent
 import pkg/codex/sales/salescontext
 import pkg/codex/sales/statemachine
-import pkg/codex/sales/states/errorhandling
 
 import ../../asynctest
 import ../helpers/mockmarket
@@ -15,18 +14,12 @@ import ../examples
 var onCancelCalled = false
 var onFailedCalled = false
 var onSlotFilledCalled = false
-var onErrorCalled = false
 
-type
+type MockState = ref object of SaleState
-  MockState = ref object of SaleState
-  MockErrorState = ref object of ErrorHandlingState
 
 method `$`*(state: MockState): string =
   "MockState"
 
-method `$`*(state: MockErrorState): string =
-  "MockErrorState"
-
 method onCancelled*(state: MockState, request: StorageRequest): ?State =
   onCancelCalled = true
 
@@ -38,12 +31,6 @@ method onSlotFilled*(
 ): ?State =
   onSlotFilledCalled = true
 
-method onError*(state: MockErrorState, err: ref CatchableError): ?State =
-  onErrorCalled = true
-
-method run*(state: MockErrorState, machine: Machine): Future[?State] {.async.} =
-  raise newException(ValueError, "failure")
-
 asyncchecksuite "Sales agent":
   let request = StorageRequest.example
   var agent: SalesAgent
@@ -123,7 +110,9 @@ asyncchecksuite "Sales agent":
     agent.start(MockState.new())
     await agent.subscribe()
     agent.onFulfilled(request.id)
-    check eventually agent.data.cancelled.cancelled()
+    # Note: futures that are cancelled, and do not re-raise the CancelledError
+    # will have a state of completed, not cancelled.
+    check eventually agent.data.cancelled.completed()
 
   test "current state onFailed called when onFailed called":
     agent.start(MockState.new())
@@ -134,7 +123,3 @@ asyncchecksuite "Sales agent":
     agent.start(MockState.new())
     agent.onSlotFilled(request.id, slotIndex)
     check eventually onSlotFilledCalled
-
-  test "ErrorHandlingState.onError can be overridden at the state level":
-    agent.start(MockErrorState.new())
-    check eventually onErrorCalled

tests/codex/utils/testasyncstatemachine.nim

@@ -10,9 +10,8 @@ type
   State1 = ref object of State
   State2 = ref object of State
   State3 = ref object of State
-  State4 = ref object of State
 
-var runs, cancellations, errors = [0, 0, 0, 0]
+var runs, cancellations = [0, 0, 0, 0]
 
 method `$`(state: State1): string =
   "State1"
@@ -23,28 +22,20 @@ method `$`(state: State2): string =
 method `$`(state: State3): string =
   "State3"
 
-method `$`(state: State4): string =
+method run(state: State1, machine: Machine): Future[?State] {.async: (raises: []).} =
-  "State4"
-
-method run(state: State1, machine: Machine): Future[?State] {.async.} =
   inc runs[0]
   return some State(State2.new())
 
-method run(state: State2, machine: Machine): Future[?State] {.async.} =
+method run(state: State2, machine: Machine): Future[?State] {.async: (raises: []).} =
   inc runs[1]
   try:
     await sleepAsync(1.hours)
   except CancelledError:
     inc cancellations[1]
-    raise
 
-method run(state: State3, machine: Machine): Future[?State] {.async.} =
+method run(state: State3, machine: Machine): Future[?State] {.async: (raises: []).} =
   inc runs[2]
 
-method run(state: State4, machine: Machine): Future[?State] {.async.} =
-  inc runs[3]
-  raise newException(ValueError, "failed")
-
 method onMoveToNextStateEvent*(state: State): ?State {.base, upraises: [].} =
   discard
 
@@ -54,19 +45,6 @@ method onMoveToNextStateEvent(state: State2): ?State =
 method onMoveToNextStateEvent(state: State3): ?State =
   some State(State1.new())
 
-method onError(state: State1, error: ref CatchableError): ?State =
-  inc errors[0]
-
-method onError(state: State2, error: ref CatchableError): ?State =
-  inc errors[1]
-
-method onError(state: State3, error: ref CatchableError): ?State =
-  inc errors[2]
-
-method onError(state: State4, error: ref CatchableError): ?State =
-  inc errors[3]
-  some State(State2.new())
-
 asyncchecksuite "async state machines":
   var machine: Machine
 
@@ -76,7 +54,6 @@ asyncchecksuite "async state machines":
   setup:
     runs = [0, 0, 0, 0]
     cancellations = [0, 0, 0, 0]
-    errors = [0, 0, 0, 0]
     machine = Machine.new()
 
   test "should call run on start state":
@@ -112,16 +89,6 @@ asyncchecksuite "async state machines":
     check runs == [0, 1, 0, 0]
     check cancellations == [0, 1, 0, 0]
 
-  test "forwards errors to error handler":
-    machine.start(State4.new())
-    check eventually errors == [0, 0, 0, 1] and runs == [0, 1, 0, 1]
-
-  test "error handler ignores CancelledError":
-    machine.start(State2.new())
-    machine.schedule(moveToNextStateEvent)
-    check eventually cancellations == [0, 1, 0, 0]
-    check errors == [0, 0, 0, 0]
-
   test "queries properties of the current state":
     proc description(state: State): string =
       $state