[marketplace] add/remove proofs for contract state

Add or remove proof requirements when a request contract’s state changes. When a request sale has completed (for a slot), the host who purchased that slot now must provide regular proofs for the data they are contracted to hold. This is now enforced by adding the slotId to the HashSet of Ids for which to require proofs. When a request has been cancelled (not all slots were filled before the request expired), proofs no longer need to be provided and the slotId is removed from teh HashSet.

Add `isCancelled` and `isSlotCancelled` checks to query the contract state without relying the on the state context variable in the contract. Because contract state can only be updated in a transaction, and the client withdrawing funds is responsible for changing the contract state to “Cancelled”, the `isCancelled` and `isSlotCancelled` functions were introduced to check the state regardless of whether or not the client had already withdrawn their funds.

codex/contracts/interactions.nim

@@ -33,10 +33,11 @@ proc new*(_: type ContractInteractions,
   let market = OnChainMarket.new(contract)
   let proofs = OnChainProofs.new(contract)
   let clock = OnChainClock.new(signer.provider)
+  let proving = Proving.new(proofs, clock)
   some ContractInteractions(
     purchasing: Purchasing.new(market, clock),
-    sales: Sales.new(market, clock),
+    sales: Sales.new(market, clock, proving),
-    proving: Proving.new(proofs, clock),
+    proving: proving,
     clock: clock
   )
 

codex/contracts/proofs.nim

@@ -22,6 +22,14 @@ method periodicity*(proofs: OnChainProofs): Future[Periodicity] {.async.} =
   let period = await proofs.storage.proofPeriod()
   return Periodicity(seconds: period)
 
+method isSlotCancelled*(proofs: OnChainProofs,
+                        id: ContractId): Future[bool] {.async.} =
+  return await proofs.storage.isSlotCancelled(id)
+
+method isCancelled*(proofs: OnChainProofs,
+                    id: array[32, byte]): Future[bool] {.async.} =
+  return await proofs.storage.isCancelled(id)
+
 method isProofRequired*(proofs: OnChainProofs,
                         id: SlotId): Future[bool] {.async.} =
   return await proofs.storage.isProofRequired(id)

codex/contracts/storage.nim

@@ -45,6 +45,8 @@ proc proofTimeout*(storage: Storage): UInt256 {.contract, view.}
 
 proc proofEnd*(storage: Storage, id: SlotId): UInt256 {.contract, view.}
 proc missingProofs*(storage: Storage, id: SlotId): UInt256 {.contract, view.}
+proc isCancelled*(storage: Storage, id: Id): bool {.contract, view.}
+proc isSlotCancelled*(storage: Storage, id: Id): bool {.contract, view.}
 proc isProofRequired*(storage: Storage, id: SlotId): bool {.contract, view.}
 proc willProofBeRequired*(storage: Storage, id: SlotId): bool {.contract, view.}
 proc getChallenge*(storage: Storage, id: SlotId): array[32, byte] {.contract, view.}

codex/node.nim

@@ -343,9 +343,12 @@ proc start*(node: CodexNodeRef) {.async.} =
       if fetchRes.isErr:
         raise newException(CodexError, "Unable to retrieve blocks")
 
-    contracts.sales.onClear = proc(availability: Availability, request: StorageRequest) =
+    contracts.sales.onClear = proc(availability: Availability,
+                                   request: StorageRequest,
+                                   slotIndex: UInt256) =
       # TODO: remove data from local storage
       discard
+
     contracts.sales.onProve = proc(request: StorageRequest,
                                    slot: UInt256): Future[seq[byte]] {.async.} =
       # TODO: generate proof

codex/proving.nim

@@ -42,11 +42,19 @@ proc removeEndedContracts(proving: Proving) {.async.} =
       ended.incl(id)
   proving.slots.excl(ended)
 
+proc removeCancelledContracts(proving: Proving) {.async.} =
+  var cancelled: HashSet[ContractId]
+  for id in proving.contracts:
+    if (await proving.proofs.isSlotCancelled(id)):
+      cancelled.incl(id)
+  proving.contracts.excl(cancelled)
+
 proc run(proving: Proving) {.async.} =
   try:
     while true:
       let currentPeriod = await proving.getCurrentPeriod()
       await proving.removeEndedContracts()
+      await proving.removeCancelledContracts()
       for id in proving.slots:
         if (await proving.proofs.isProofRequired(id)) or
           (await proving.proofs.willProofBeRequired(id)):

codex/sales.nim

@@ -7,6 +7,8 @@ import pkg/chronicles
 import ./rng
 import ./market
 import ./clock
+import ./proving
+import ./contracts/requests
 
 ## Sales holds a list of available storage that it may sell.
 ##
@@ -32,12 +34,13 @@ type
   Sales* = ref object
     market: Market
     clock: Clock
-    subscription: ?Subscription
+    subscription: ?market.Subscription
     available*: seq[Availability]
     onStore: ?OnStore
     onProve: ?OnProve
     onClear: ?OnClear
     onSale: ?OnSale
+    proving: Proving
   Availability* = object
     id*: array[32, byte]
     size*: UInt256
@@ -50,7 +53,7 @@ type
     availability: Availability
     request: ?StorageRequest
     slotIndex: ?UInt256
-    subscription: ?Subscription
+    subscription: ?market.Subscription
     running: ?Future[void]
     waiting: ?Future[void]
     finished: bool
@@ -59,15 +62,21 @@ type
                  availability: Availability): Future[void] {.gcsafe, upraises: [].}
   OnProve = proc(request: StorageRequest,
                  slot: UInt256): Future[seq[byte]] {.gcsafe, upraises: [].}
-  OnClear = proc(availability: Availability, request: StorageRequest) {.gcsafe, upraises: [].}
+  OnClear = proc(availability: Availability,
+                 request: StorageRequest,
+                 slotIndex: UInt256) {.gcsafe, upraises: [].}
   OnSale = proc(availability: Availability,
                 request: StorageRequest,
                 slotIndex: UInt256) {.gcsafe, upraises: [].}
 
-func new*(_: type Sales, market: Market, clock: Clock): Sales =
+func new*(_: type Sales,
+          market: Market,
+          clock: Clock,
+          proving: Proving): Sales =
   Sales(
     market: market,
     clock: clock,
+    proving: proving
   )
 
 proc init*(_: type Availability,
@@ -119,13 +128,17 @@ proc finish(agent: SalesAgent, success: bool) =
     waiting.cancel()
 
   if success:
-    if onSale =? agent.sales.onSale and
+    if request =? agent.request and
+       slotIndex =? agent.slotIndex:
+      agent.sales.proving.add(request.slotId(slotIndex))
+
+      if onSale =? agent.sales.onSale:
+        onSale(agent.availability, request, slotIndex)
+  else:
+    if onClear =? agent.sales.onClear and
        request =? agent.request and
        slotIndex =? agent.slotIndex:
-      onSale(agent.availability, request, slotIndex)
+      onClear(agent.availability, request, slotIndex)
-  else:
-    if onClear =? agent.sales.onClear and request =? agent.request:
-      onClear(agent.availability, request)
     agent.sales.add(agent.availability)
 
 proc selectSlot(agent: SalesAgent)  =
@@ -222,7 +235,7 @@ proc start*(sales: Sales) {.async.} =
 
 proc stop*(sales: Sales) {.async.} =
   if subscription =? sales.subscription:
-    sales.subscription = Subscription.none
+    sales.subscription = market.Subscription.none
     try:
       await subscription.unsubscribe()
     except CatchableError as e:

codex/storageproofs/timing/proofs.nim

@@ -18,6 +18,14 @@ method periodicity*(proofs: Proofs):
                    Future[Periodicity] {.base, async.} =
   raiseAssert("not implemented")
 
+method isSlotCancelled*(proofs: Proofs,
+                        id: ContractId): Future[bool] {.base, async.} =
+  raiseAssert("not implemented")
+
+method isCancelled*(proofs: Proofs,
+                    id: array[32, byte]): Future[bool] {.base, async.} =
+  raiseAssert("not implemented")
+
 method isProofRequired*(proofs: Proofs,
                         id: SlotId): Future[bool] {.base, async.} =
   raiseAssert("not implemented")

tests/codex/helpers/mockproofs.nim

@@ -7,6 +7,7 @@ import pkg/codex/storageproofs
 type
   MockProofs* = ref object of Proofs
     periodicity: Periodicity
+    cancelledRequests: HashSet[ContractId]
     proofsRequired: HashSet[SlotId]
     proofsToBeRequired: HashSet[SlotId]
     proofEnds: Table[SlotId, UInt256]
@@ -32,6 +33,20 @@ proc setProofRequired*(mock: MockProofs, id: SlotId, required: bool) =
   else:
     mock.proofsRequired.excl(id)
 
+proc setCancelled*(mock: MockProofs, id: ContractId, required: bool) =
+  if required:
+    mock.cancelledRequests.incl(id)
+  else:
+    mock.cancelledRequests.excl(id)
+
+method isCancelled*(mock: MockProofs,
+                    id: array[32, byte]): Future[bool] {.async.} =
+  return mock.cancelledRequests.contains(id)
+
+method isSlotCancelled*(mock: MockProofs,
+                        id: ContractId): Future[bool] {.async.} =
+  return mock.cancelledRequests.contains(id)
+
 method isProofRequired*(mock: MockProofs,
                         id: SlotId): Future[bool] {.async.} =
   return mock.proofsRequired.contains(id)

tests/codex/testproving.nim

@@ -92,6 +92,20 @@ suite "Proving":
     await proofs.advanceToNextPeriod()
     check not called
 
+  test "stops watching when contract is cancelled":
+    let id = ContractId.example
+    proving.add(id)
+    var called: bool
+    proc onProofRequired(id: ContractId) =
+      called = true
+    proofs.setProofRequired(id, true)
+    await proofs.advanceToNextPeriod()
+    proving.onProofRequired = onProofRequired
+    proofs.setCancelled(id, true)
+    await proofs.advanceToNextPeriod()
+    check not proving.contracts.contains(id)
+    check not called
+
   test "submits proofs":
     let id = SlotId.example
     let proof = seq[byte].example

tests/codex/testsales.nim

@@ -1,5 +1,8 @@
+import std/sets
 import pkg/asynctest
 import pkg/chronos
+import pkg/codex/contracts/requests
+import pkg/codex/proving
 import pkg/codex/sales
 import ./helpers/mockmarket
 import ./helpers/mockclock
@@ -28,11 +31,13 @@ suite "Sales":
   var sales: Sales
   var market: MockMarket
   var clock: MockClock
+  var proving: Proving
 
   setup:
     market = MockMarket.new()
     clock = MockClock.new()
-    sales = Sales.new(market, clock)
+    proving = Proving.new()
+    sales = Sales.new(market, clock, proving)
     sales.onStore = proc(request: StorageRequest,
                          slot: UInt256,
                          availability: Availability) {.async.} =
@@ -151,18 +156,25 @@ suite "Sales":
     check soldSlotIndex < request.ask.slots.u256
 
   test "calls onClear when storage becomes available again":
+    # fail the proof intentionally to trigger `agent.finish(success=false)`,
+    # which then calls the onClear callback
     sales.onProve = proc(request: StorageRequest,
                          slot: UInt256): Future[seq[byte]] {.async.} =
       raise newException(IOError, "proof failed")
     var clearedAvailability: Availability
     var clearedRequest: StorageRequest
-    sales.onClear = proc(availability: Availability, request: StorageRequest) =
+    var clearedSlotIndex: UInt256
+    sales.onClear = proc(availability: Availability,
+                         request: StorageRequest,
+                         slotIndex: UInt256) =
       clearedAvailability = availability
       clearedRequest = request
+      clearedSlotIndex = slotIndex
     sales.add(availability)
     discard await market.requestStorage(request)
     check clearedAvailability == availability
     check clearedRequest == request
+    check clearedSlotIndex < request.ask.slots.u256
 
   test "makes storage available again when other host fills the slot":
     let otherHost = Address.example
@@ -186,3 +198,15 @@ suite "Sales":
     clock.set(request.expiry.truncate(int64))
     await sleepAsync(2.seconds)
     check sales.available == @[availability]
+
+  test "adds proving for slot when slot is filled":
+    var soldSlotIndex: UInt256
+    sales.onSale = proc(availability: Availability,
+                        request: StorageRequest,
+                        slotIndex: UInt256) =
+      soldSlotIndex = slotIndex
+    check proving.contracts.len == 0
+    sales.add(availability)
+    discard await market.requestStorage(request)
+    check proving.contracts.len == 1
+    check proving.contracts.contains(request.slotId(soldSlotIndex))

tests/contracts/matchers.nim

@@ -0,0 +1,26 @@
+import std/json
+import std/strutils
+import pkg/asynctest
+import pkg/ethers
+
+proc revertReason*(e: ref ValueError): string =
+  try:
+    let json = parseJson(e.msg)
+    var msg = json{"message"}.getStr
+    const revertPrefix =
+      "Error: VM Exception while processing transaction: reverted with " &
+      "reason string "
+    msg = msg.replace(revertPrefix)
+    msg = msg.replace("\'", "")
+    return msg
+  except JsonParsingError:
+    return ""
+
+
+template revertsWith*(reason, body) =
+  var revertReason = ""
+  try:
+    body
+  except ValueError as e:
+    revertReason = e.revertReason
+  check revertReason == reason

tests/contracts/testContracts.nim

@@ -5,6 +5,7 @@ import codex/contracts/testtoken
 import codex/storageproofs
 import ../ethertest
 import ./examples
+import ./matchers
 import ./time
 
 ethersuite "Storage contracts":
@@ -74,3 +75,21 @@ ethersuite "Storage contracts":
     switchAccount(host)
     await provider.advanceTimeTo(await storage.proofEnd(slotId))
     await storage.payoutSlot(request.id, 0.u256)
+
+  test "a request is cancelled after expiry":
+    check not await storage.isCancelled(request.id)
+    await provider.advanceTimeTo(request.expiry + 1)
+    check await storage.isCancelled(request.id)
+
+  test "a slot is cancelled after expiry":
+    check not await storage.isSlotCancelled(id)
+    await provider.advanceTimeTo(request.expiry + 1)
+    check await storage.isSlotCancelled(id)
+
+  test "cannot mark proofs missing for cancelled request":
+    await provider.advanceTimeTo(request.expiry + 1)
+    switchAccount(client)
+    let missingPeriod = periodicity.periodOf(await provider.currentTime())
+    await provider.advanceTime(periodicity.seconds)
+    revertsWith "Request was cancelled":
+      await storage.markProofAsMissing(id, missingPeriod)