[marketplace] Simulate invalid proof submissions (#393)

codex.nimble

@@ -50,24 +50,23 @@ proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
     for i in 2..<paramCount():
       extra_params &= " " & paramStr(i)
 
-
   exec "nim " & lang & " --out:build/" & name & " " & extra_params & " " & srcDir & name & ".nim"
 
-proc test(name: string, srcDir = "tests/", lang = "c") =
+proc test(name: string, srcDir = "tests/", params = "", lang = "c") =
-  buildBinary name, srcDir
+  buildBinary name, srcDir, params
   exec "build/" & name
 
 task codex, "build codex binary":
   buildBinary "codex", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
 
 task testCodex, "Build & run Codex tests":
-  test "testCodex"
+  test "testCodex", params = "-d:codex_enable_proof_failures=true"
 
 task testContracts, "Build & run Codex Contract tests":
   test "testContracts"
 
 task testIntegration, "Run integration tests":
-  codexTask()
+  buildBinary "codex", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true"
   test "testIntegration"
 
 task test, "Run tests":

codex/codex.nim

@@ -82,7 +82,13 @@ proc bootstrapInteractions(config: CodexConf, repo: RepoStore): Future[Contracts
   var validator: ?ValidatorInteractions
   if config.persistence:
     let purchasing = Purchasing.new(market, clock)
-    let proving = Proving.new(market, clock)
+    when codex_enable_proof_failures:
+      let proving = if config.simulateProofFailures > 0:
+                      SimulatedProving.new(market, clock,
+                                           config.simulateProofFailures)
+                    else: Proving.new(market, clock)
+    else:
+      let proving = Proving.new(market, clock)
     let sales = Sales.new(market, clock, proving, repo)
     client = some ClientInteractions.new(clock, purchasing)
     host = some HostInteractions.new(clock, sales, proving)

codex/conf.nim

@@ -37,6 +37,7 @@ export DefaultCacheSizeMiB, net, DefaultQuotaBytes, DefaultBlockTtl, DefaultBloc
 
 const
   codex_enable_api_debug_peers* {.booldefine.} = false
+  codex_enable_proof_failures* {.booldefine.} = false
 
 type
   StartUpCommand* {.pure.} = enum
@@ -240,6 +241,13 @@ type
         name: "validator-max-slots"
       .}: int
 
+      simulateProofFailures* {.
+          desc: "Simulates proof failures once every N proofs. 0 = disabled."
+          defaultValue: 0
+          name: "simulate-proof-failures"
+          hidden
+        .}: uint
+
     of initNode:
       discard
 

codex/contracts/market.nim

@@ -5,6 +5,7 @@ import pkg/ethers
 import pkg/ethers/testing
 import pkg/upraises
 import pkg/questionable
+import pkg/chronicles
 import ../market
 import ./marketplace
 

codex/contracts/requests.nim

@@ -3,6 +3,7 @@ import pkg/contractabi
 import pkg/nimcrypto
 import pkg/ethers/fields
 import pkg/questionable/results
+import pkg/stew/byteutils
 
 export contractabi
 
@@ -63,6 +64,15 @@ func toArray*(id: RequestId | SlotId | Nonce): array[32, byte] =
 proc `$`*(id: RequestId | SlotId | Nonce): string =
   id.toArray.toHex
 
+proc fromHex*(T: type RequestId, hex: string): T =
+  T array[32, byte].fromHex(hex)
+
+proc fromHex*(T: type SlotId, hex: string): T =
+  T array[32, byte].fromHex(hex)
+
+proc fromHex*(T: type Nonce, hex: string): T =
+  T array[32, byte].fromHex(hex)
+
 func fromTuple(_: type StorageRequest, tupl: tuple): StorageRequest =
   StorageRequest(
     client: tupl[0],

codex/proving.nim

@@ -1,97 +1,5 @@
-import std/sets
+import ./proving/proving
-import pkg/upraises
+import ./proving/simulated
-import pkg/questionable
-import pkg/chronicles
-import ./market
-import ./clock
 
-export sets
+export proving
-
+export simulated
-logScope:
-    topics = "marketplace proving"
-
-type
-  Proving* = ref object
-    market: Market
-    clock: Clock
-    loop: ?Future[void]
-    slots*: HashSet[Slot]
-    onProve: ?OnProve
-  OnProve* = proc(slot: Slot): Future[seq[byte]] {.gcsafe, upraises: [].}
-
-func new*(_: type Proving, market: Market, clock: Clock): Proving =
-  Proving(market: market, clock: clock)
-
-proc onProve*(proving: Proving): ?OnProve =
-  proving.onProve
-
-proc `onProve=`*(proving: Proving, callback: OnProve) =
-  proving.onProve = some callback
-
-func add*(proving: Proving, slot: Slot) =
-  proving.slots.incl(slot)
-
-proc getCurrentPeriod(proving: Proving): Future[Period] {.async.} =
-  let periodicity = await proving.market.periodicity()
-  return periodicity.periodOf(proving.clock.now().u256)
-
-proc waitUntilPeriod(proving: Proving, period: Period) {.async.} =
-  let periodicity = await proving.market.periodicity()
-  await proving.clock.waitUntil(periodicity.periodStart(period).truncate(int64))
-
-proc removeEndedContracts(proving: Proving) {.async.} =
-  var ended: HashSet[Slot]
-  for slot in proving.slots:
-    let state = await proving.market.slotState(slot.id)
-    if state == SlotState.Finished:
-      debug "Collecting finished slot's reward", slot = $slot.id
-      await proving.market.freeSlot(slot.id)
-
-    if state != SlotState.Filled:
-      debug "Request ended, cleaning up slot", slot = $slot.id
-      ended.incl(slot)
-  proving.slots.excl(ended)
-
-proc prove(proving: Proving, slot: Slot) {.async.} =
-  without onProve =? proving.onProve:
-    raiseAssert "onProve callback not set"
-  try:
-    debug "Proving slot"
-    let proof = await onProve(slot)
-    await proving.market.submitProof(slot.id, proof)
-  except CatchableError as e:
-    error "Submitting proof failed", msg = e.msg
-
-proc run(proving: Proving) {.async.} =
-  try:
-    while true:
-      let currentPeriod = await proving.getCurrentPeriod()
-      debug "Proving for new period", period = currentPeriod
-      await proving.removeEndedContracts()
-      for slot in proving.slots:
-        let id = slot.id
-        if (await proving.market.isProofRequired(id)) or
-           (await proving.market.willProofBeRequired(id)):
-          asyncSpawn proving.prove(slot)
-      await proving.waitUntilPeriod(currentPeriod + 1)
-  except CancelledError:
-    discard
-  except CatchableError as e:
-    error "Proving failed", msg = e.msg
-
-proc start*(proving: Proving) {.async.} =
-  if proving.loop.isSome:
-    return
-
-  proving.loop = some proving.run()
-
-proc stop*(proving: Proving) {.async.} =
-  if loop =? proving.loop:
-    proving.loop = Future[void].none
-    if not loop.finished:
-      await loop.cancelAndWait()
-
-proc subscribeProofSubmission*(proving: Proving,
-                               callback: OnProofSubmitted):
-                              Future[Subscription] =
-  proving.market.subscribeProofSubmission(callback)

codex/proving/proving.nim

@@ -0,0 +1,101 @@
+import std/sets
+import pkg/upraises
+import pkg/questionable
+import pkg/chronicles
+import ../market
+import ../clock
+
+export sets
+
+logScope:
+  topics = "marketplace proving"
+
+type
+  Proving* = ref object of RootObj
+    market*: Market
+    clock: Clock
+    loop: ?Future[void]
+    slots*: HashSet[Slot]
+    onProve: ?OnProve
+  OnProve* = proc(slot: Slot): Future[seq[byte]] {.gcsafe, upraises: [].}
+
+func new*(T: type Proving, market: Market, clock: Clock): T =
+  T(market: market, clock: clock)
+
+proc onProve*(proving: Proving): ?OnProve =
+  proving.onProve
+
+proc `onProve=`*(proving: Proving, callback: OnProve) =
+  proving.onProve = some callback
+
+func add*(proving: Proving, slot: Slot) =
+  proving.slots.incl(slot)
+
+proc getCurrentPeriod*(proving: Proving): Future[Period] {.async.} =
+  let periodicity = await proving.market.periodicity()
+  return periodicity.periodOf(proving.clock.now().u256)
+
+proc waitUntilPeriod(proving: Proving, period: Period) {.async.} =
+  let periodicity = await proving.market.periodicity()
+  await proving.clock.waitUntil(periodicity.periodStart(period).truncate(int64))
+
+proc removeEndedContracts(proving: Proving) {.async.} =
+  var ended: HashSet[Slot]
+  for slot in proving.slots:
+    let state = await proving.market.slotState(slot.id)
+    if state == SlotState.Finished:
+      debug "Collecting finished slot's reward", slot = $slot.id
+      await proving.market.freeSlot(slot.id)
+
+    if state != SlotState.Filled:
+      debug "Request ended, cleaning up slot", slot = $slot.id
+      ended.incl(slot)
+  proving.slots.excl(ended)
+
+method prove*(proving: Proving, slot: Slot) {.base, async.} =
+  logScope:
+    currentPeriod = await proving.getCurrentPeriod()
+
+  without onProve =? proving.onProve:
+    raiseAssert "onProve callback not set"
+  try:
+    debug "Proving slot"
+    let proof = await onProve(slot)
+    trace "submitting proof", currentPeriod = await proving.getCurrentPeriod()
+    await proving.market.submitProof(slot.id, proof)
+  except CatchableError as e:
+    error "Submitting proof failed", msg = e.msg
+
+proc run(proving: Proving) {.async.} =
+  try:
+    while true:
+      let currentPeriod = await proving.getCurrentPeriod()
+      debug "Proving for new period", period = currentPeriod
+      await proving.removeEndedContracts()
+      for slot in proving.slots:
+        let id = slot.id
+        if (await proving.market.isProofRequired(id)) or
+           (await proving.market.willProofBeRequired(id)):
+          asyncSpawn proving.prove(slot)
+      await proving.waitUntilPeriod(currentPeriod + 1)
+  except CancelledError:
+    discard
+  except CatchableError as e:
+    error "Proving failed", msg = e.msg
+
+proc start*(proving: Proving) {.async.} =
+  if proving.loop.isSome:
+    return
+
+  proving.loop = some proving.run()
+
+proc stop*(proving: Proving) {.async.} =
+  if loop =? proving.loop:
+    proving.loop = Future[void].none
+    if not loop.finished:
+      await loop.cancelAndWait()
+
+proc subscribeProofSubmission*(proving: Proving,
+                               callback: OnProofSubmitted):
+                              Future[Subscription] =
+  proving.market.subscribeProofSubmission(callback)

codex/proving/simulated.nim

@@ -0,0 +1,47 @@
+import ../conf
+when codex_enable_proof_failures:
+
+  import std/strutils
+  import pkg/chronicles
+  import pkg/ethers
+  import pkg/ethers/testing
+  import ../market
+  import ../clock
+  import ./proving
+
+  type
+    SimulatedProving* = ref object of Proving
+      failEveryNProofs: uint
+      proofCount: uint
+
+  logScope:
+    topics = "simulated proving"
+
+  func new*(_: type SimulatedProving,
+            market: Market,
+            clock: Clock,
+            failEveryNProofs: uint): SimulatedProving =
+
+    let p = SimulatedProving.new(market, clock)
+    p.failEveryNProofs = failEveryNProofs
+    return p
+
+  proc onSubmitProofError(error: ref CatchableError, period: UInt256) =
+    error "Submitting invalid proof failed", period, msg = error.msg
+
+  method prove(proving: SimulatedProving, slot: Slot) {.async.} =
+    let period = await proving.getCurrentPeriod()
+    proving.proofCount += 1
+    if proving.failEveryNProofs > 0'u and
+      proving.proofCount mod proving.failEveryNProofs == 0'u:
+      proving.proofCount = 0
+      try:
+        trace "submitting INVALID proof", currentPeriod = await proving.getCurrentPeriod()
+        await proving.market.submitProof(slot.id, newSeq[byte](0))
+      except ProviderError as e:
+        if not e.revertReason.contains("Invalid proof"):
+          onSubmitProofError(e, period)
+      except CatchableError as e:
+        onSubmitProofError(e, period)
+    else:
+      await procCall Proving(proving).prove(slot)

codex/rest/json.nim

@@ -70,4 +70,5 @@ func `%`*(purchase: Purchase): JsonNode =
     "state": purchase.state |? "none",
     "error": purchase.error.?msg,
     "request": purchase.request,
+    "requestId": purchase.requestId
   }

codex/sales/states/finished.nim

@@ -29,8 +29,9 @@ method run*(state: SaleFinished, machine: Machine): Future[?State] {.async.} =
 
   if request =? data.request and
       slotIndex =? data.slotIndex:
-    debug "Adding request to proving list", requestId = $data.requestId
+    let slot = Slot(request: request, slotIndex: slotIndex)
-    context.proving.add(Slot(request: request, slotIndex: slotIndex))
+    debug "Adding slot to proving list", slotId = $slot.id
+    context.proving.add(slot)
 
     if onSale =? context.onSale:
       onSale(request, slotIndex)

codex/sales/states/proving.nim

@@ -35,8 +35,8 @@ method run*(state: SaleProving, machine: Machine): Future[?State] {.async.} =
   without onProve =? context.proving.onProve:
     raiseAssert "onProve callback not set"
 
-  debug "Start proving", requestId = $data.requestId
+  debug "Start proof generation", requestId = $data.requestId
   let proof = await onProve(Slot(request: request, slotIndex: data.slotIndex))
-  debug "Finished proving", requestId = $data.requestId
+  debug "Finished proof generation", requestId = $data.requestId
 
   return some State(SaleFilling(proof: proof))

codex/validation.nim

@@ -62,14 +62,18 @@ proc removeSlotsThatHaveEnded(validation: Validation) {.async.} =
 proc markProofAsMissing(validation: Validation,
                         slotId: SlotId,
                         period: Period) {.async.} =
+  logScope:
+    currentPeriod = validation.getCurrentPeriod()
+
   try:
     if await validation.market.canProofBeMarkedAsMissing(slotId, period):
-      trace "Marking proof as missing", slotId = $slotId, period = period
+      trace "Marking proof as missing", slotId = $slotId, periodProofMissed = period
       await validation.market.markProofAsMissing(slotId, period)
+    else: trace "Proof not missing", checkedPeriod = period
   except CancelledError:
     raise
   except CatchableError as e:
-    debug "Marking proof as missing failed", msg = e.msg
+    error "Marking proof as missing failed", msg = e.msg
 
 proc markProofsAsMissing(validation: Validation) {.async.} =
   for slotId in validation.slots:

tests/codex/testproving.nim

@@ -1,3 +1,4 @@
+import std/sequtils
 import pkg/asynctest
 import pkg/chronos
 import pkg/codex/proving
@@ -122,3 +123,65 @@ suite "Proving":
     check eventually receivedIds == @[slot.id] and receivedProofs == @[proof]
 
     await subscription.unsubscribe()
+
+suite "Simulated proving":
+
+  var proving: SimulatedProving
+  var subscription: Subscription
+  var market: MockMarket
+  var clock: MockClock
+  var submitted: seq[seq[byte]]
+  var proof: seq[byte]
+  let slot = Slot.example
+  var proofSubmitted: Future[void]
+
+  setup:
+    proof = exampleProof()
+    submitted = @[]
+    market = MockMarket.new()
+    clock = MockClock.new()
+    proofSubmitted = newFuture[void]("proofSubmitted")
+
+  teardown:
+    await subscription.unsubscribe()
+    await proving.stop()
+
+  proc newSimulatedProving(failEveryNProofs: uint) {.async.} =
+    proc onProofSubmission(id: SlotId, proof: seq[byte]) =
+      submitted.add(proof)
+      proofSubmitted.complete()
+      proofSubmitted = newFuture[void]("proofSubmitted")
+
+    proving = SimulatedProving.new(market, clock, failEveryNProofs)
+    proving.onProve = proc (slot: Slot): Future[seq[byte]] {.async.} =
+      return proof
+    subscription = await proving.subscribeProofSubmission(onProofSubmission)
+    proving.add(slot)
+    market.slotState[slot.id] = SlotState.Filled
+    market.setProofRequired(slot.id, true)
+    await proving.start()
+
+  proc advanceToNextPeriod(market: Market) {.async.} =
+    let periodicity = await market.periodicity()
+    clock.advance(periodicity.seconds.truncate(int64))
+
+  proc waitForProvingRounds(market: Market, rounds: uint) {.async.} =
+    var rnds = rounds - 1 # proof round runs prior to advancing
+    while rnds > 0:
+      await market.advanceToNextPeriod()
+      await proofSubmitted
+      rnds -= 1
+
+  test "submits invalid proof every 3 proofs":
+    let failEveryNProofs = 3'u
+    let totalProofs = 6'u
+    await newSimulatedProving(failEveryNProofs)
+    await market.waitForProvingRounds(totalProofs)
+    check submitted == @[proof, proof, @[], proof, proof, @[]]
+
+  test "does not submit invalid proofs when failEveryNProofs is 0":
+    let failEveryNProofs = 0'u
+    let totalProofs = 6'u
+    await newSimulatedProving(failEveryNProofs)
+    await market.waitForProvingRounds(totalProofs)
+    check submitted == proof.repeat(totalProofs)

tests/contracts/testDeployment.nim

@@ -1,4 +1,3 @@
-import std/os
 import pkg/asynctest
 import pkg/ethers
 import codex/contracts/deployment

tests/integration/multinodes.nim

@@ -0,0 +1,123 @@
+import std/os
+import std/macros
+import std/json
+import std/httpclient
+import pkg/chronicles
+import ../ethertest
+import ./codexclient
+import ./nodes
+
+export ethertest
+export codexclient
+export nodes
+
+template multinodesuite*(name: string,
+  startNodes: StartNodes, debugNodes: DebugNodes, body: untyped) =
+
+  if (debugNodes.client or debugNodes.provider) and
+      (enabledLogLevel > LogLevel.TRACE or
+      enabledLogLevel == LogLevel.NONE):
+    echo ""
+    echo "More test debug logging is available by running the tests with " &
+      "'-d:chronicles_log_level=TRACE " &
+      "-d:chronicles_disabled_topics=websock " &
+      "-d:chronicles_default_output_device=stdout " &
+      "-d:chronicles_sinks=textlines'"
+    echo ""
+
+  ethersuite name:
+
+    var running: seq[RunningNode]
+    var bootstrap: string
+
+    proc newNodeProcess(index: int,
+                        addlOptions: seq[string],
+                        debug: bool): (NodeProcess, string, Address) =
+
+      if index > accounts.len - 1:
+        raiseAssert("Cannot start node at index " & $index &
+          ", not enough eth accounts.")
+
+      let datadir = getTempDir() / "Codex" & $index
+      var options = @[
+        "--api-port=" & $(8080 + index),
+        "--data-dir=" & datadir,
+        "--nat=127.0.0.1",
+        "--disc-ip=127.0.0.1",
+        "--disc-port=" & $(8090 + index),
+        "--eth-account=" & $accounts[index]]
+        .concat(addlOptions)
+      if debug: options.add "--log-level=INFO;TRACE: " & debugNodes.topics
+      let node = startNode(options, debug = debug)
+      (node, datadir, accounts[index])
+
+    proc newCodexClient(index: int): CodexClient =
+      CodexClient.new("http://localhost:" & $(8080 + index) & "/api/codex/v1")
+
+    proc startClientNode() =
+      let index = running.len
+      let (node, datadir, account) = newNodeProcess(
+        index, @["--persistence"], debugNodes.client)
+      let restClient = newCodexClient(index)
+      running.add RunningNode.new(Role.Client, node, restClient, datadir,
+                                  account)
+      if debugNodes.client:
+        debug "started new client node and codex client",
+          restApiPort = 8080 + index, discPort = 8090 + index, account
+
+    proc startProviderNode(failEveryNProofs: uint = 0) =
+      let index = running.len
+      let (node, datadir, account) = newNodeProcess(index, @[
+        "--bootstrap-node=" & bootstrap,
+        "--persistence",
+        "--simulate-proof-failures=" & $failEveryNProofs],
+        debugNodes.provider)
+      let restClient = newCodexClient(index)
+      running.add RunningNode.new(Role.Provider, node, restClient, datadir,
+                                  account)
+      if debugNodes.provider:
+        debug "started new provider node and codex client",
+          restApiPort = 8080 + index, discPort = 8090 + index, account
+
+    proc startValidatorNode() =
+      let index = running.len
+      let (node, datadir, account) = newNodeProcess(index, @[
+        "--bootstrap-node=" & bootstrap,
+        "--validator"],
+        debugNodes.validator)
+      let restClient = newCodexClient(index)
+      running.add RunningNode.new(Role.Validator, node, restClient, datadir,
+                                  account)
+      if debugNodes.validator:
+        debug "started new validator node and codex client",
+          restApiPort = 8080 + index, discPort = 8090 + index, account
+
+    proc clients(): seq[RunningNode] =
+      running.filter(proc(r: RunningNode): bool = r.role == Role.Client)
+
+    proc providers(): seq[RunningNode] =
+      running.filter(proc(r: RunningNode): bool = r.role == Role.Provider)
+
+    proc validators(): seq[RunningNode] =
+      running.filter(proc(r: RunningNode): bool = r.role == Role.Validator)
+
+    setup:
+      for i in 0..<startNodes.clients:
+        startClientNode()
+        if i == 0:
+          bootstrap = running[0].restClient.info()["spr"].getStr()
+
+      for i in 0..<startNodes.providers:
+        startProviderNode()
+
+      for i in 0..<startNodes.validators:
+        startValidatorNode()
+
+    teardown:
+      for r in running:
+        r.restClient.close()
+        r.node.stop()
+        removeDir(r.datadir)
+      running = @[]
+
+    body

tests/integration/nodes.nim

@@ -2,14 +2,58 @@ import std/osproc
 import std/os
 import std/streams
 import std/strutils
+import pkg/ethers
+import ./codexclient
 
 const workingDir = currentSourcePath() / ".." / ".." / ".."
 const executable = "build" / "codex"
 
-type NodeProcess* = ref object
+type
-  process: Process
+  NodeProcess* = ref object
-  arguments: seq[string]
+    process: Process
-  debug: bool
+    arguments: seq[string]
+    debug: bool
+  Role* = enum
+    Client,
+    Provider,
+    Validator
+  RunningNode* = ref object
+    role*: Role
+    node*: NodeProcess
+    restClient*: CodexClient
+    datadir*: string
+    ethAccount*: Address
+  StartNodes* = object
+    clients*: uint
+    providers*: uint
+    validators*: uint
+  DebugNodes* = object
+    client*: bool
+    provider*: bool
+    validator*: bool
+    topics*: string
+
+proc new*(_: type RunningNode,
+          role: Role,
+          node: NodeProcess,
+          restClient: CodexClient,
+          datadir: string,
+          ethAccount: Address): RunningNode =
+  RunningNode(role: role,
+              node: node,
+              restClient: restClient,
+              datadir: datadir,
+              ethAccount: ethAccount)
+
+proc init*(_: type StartNodes,
+          clients, providers, validators: uint): StartNodes =
+  StartNodes(clients: clients, providers: providers, validators: validators)
+
+proc init*(_: type DebugNodes,
+          client, provider, validator: bool,
+          topics: string = "validator,proving,market"): DebugNodes =
+  DebugNodes(client: client, provider: provider, validator: validator,
+             topics: topics)
 
 proc start(node: NodeProcess) =
   if node.debug:

tests/integration/testIntegration.nim

@@ -110,6 +110,5 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
     # hence we must use `advanceTime` over `sleepAsync` as Hardhat does mine new blocks
     # only with new transaction
     await provider.advanceTime(duration.u256)
-    await sleepAsync(1.seconds)
 
     check eventually (await token.balanceOf(account2)) - startBalance == duration.u256*reward.u256

tests/integration/testproofs.nim

@@ -1,11 +1,17 @@
+import std/sequtils
 import std/os
+from std/times import getTime, toUnix
+import pkg/chronicles
 import codex/contracts/marketplace
-import codex/contracts/deployment
 import codex/periods
 import ../contracts/time
 import ../contracts/deployment
 import ../codex/helpers/eventually
 import ./twonodes
+import ./multinodes
+
+logScope:
+  topics = "test proofs"
 
 twonodessuite "Proving integration test", debug1=false, debug2=false:
 
@@ -95,3 +101,118 @@ twonodessuite "Proving integration test", debug1=false, debug2=false:
 
     await subscription.unsubscribe()
     stopValidator(validator)
+
+multinodesuite "Simulate invalid proofs",
+  StartNodes.init(clients=1'u, providers=0'u, validators=1'u),
+  DebugNodes.init(client=false, provider=false, validator=false):
+
+  var marketplace: Marketplace
+  var period: uint64
+  var slotId: SlotId
+
+  setup:
+    marketplace = Marketplace.new(Marketplace.address, provider)
+    let config = await marketplace.config()
+    period = config.proofs.period.truncate(uint64)
+    slotId = SlotId(array[32, byte].default) # ensure we aren't reusing from prev test
+
+    # Our Hardhat configuration does use automine, which means that time tracked by `provider.currentTime()` is not
+    # advanced until blocks are mined and that happens only when transaction is submitted.
+    # As we use in tests provider.currentTime() which uses block timestamp this can lead to synchronization issues.
+    await provider.advanceTime(1.u256)
+
+  proc periods(p: Ordinal | uint): uint64 =
+    when p is uint:
+      p * period
+    else: p.uint * period
+
+  proc advanceToNextPeriod {.async.} =
+    let periodicity = Periodicity(seconds: period.u256)
+    let currentPeriod = periodicity.periodOf(await provider.currentTime())
+    let endOfPeriod = periodicity.periodEnd(currentPeriod)
+    await provider.advanceTimeTo(endOfPeriod + 1)
+
+  proc waitUntilPurchaseIsStarted(proofProbability: uint64 = 1,
+                                  duration: uint64 = 12.periods,
+                                  expiry: uint64 = 4.periods) {.async.} =
+
+    if clients().len < 1 or providers().len < 1:
+      raiseAssert("must start at least one client and one provider")
+
+    let client = clients()[0].restClient
+    let storageProvider = providers()[0].restClient
+
+    discard storageProvider.postAvailability(
+      size=0xFFFFF,
+      duration=duration,
+      minPrice=300,
+      maxCollateral=200
+    )
+    let cid = client.upload("some file contents " & $ getTime().toUnix)
+    let expiry = (await provider.currentTime()) + expiry.u256
+    # avoid timing issues by filling the slot at the start of the next period
+    await advanceToNextPeriod()
+    let purchase = client.requestStorage(
+      cid,
+      expiry=expiry,
+      duration=duration,
+      proofProbability=proofProbability,
+      collateral=100,
+      reward=400
+    )
+    check eventually client.getPurchase(purchase){"state"} == %"started"
+    let requestId = RequestId.fromHex client.getPurchase(purchase){"requestId"}.getStr
+    slotId = slotId(requestId, 0.u256)
+
+  # TODO: these are very loose tests in that they are not testing EXACTLY how
+  # proofs were marked as missed by the validator. These tests should be
+  # tightened so that they are showing, as an integration test, that specific
+  # proofs are being marked as missed by the validator.
+
+  test "slot is freed after too many invalid proofs submitted":
+    let failEveryNProofs = 2'u
+    let totalProofs = 100'u
+    startProviderNode(failEveryNProofs)
+
+    await waitUntilPurchaseIsStarted(duration=totalProofs.periods)
+
+    var slotWasFreed = false
+    proc onSlotFreed(event: SlotFreed) =
+      if event.slotId == slotId:
+        slotWasFreed = true
+    let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed)
+
+    for _ in 0..<totalProofs:
+      if slotWasFreed:
+        break
+      else:
+        await advanceToNextPeriod()
+        await sleepAsync(1.seconds)
+
+    check slotWasFreed
+
+    await subscription.unsubscribe()
+
+  test "slot is not freed when not enough invalid proofs submitted":
+    let failEveryNProofs = 3'u
+    let totalProofs = 12'u
+    startProviderNode(failEveryNProofs)
+
+    await waitUntilPurchaseIsStarted(duration=totalProofs.periods)
+
+    var slotWasFreed = false
+    proc onSlotFreed(event: SlotFreed) =
+      if event.slotId == slotId:
+        slotWasFreed = true
+    let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed)
+
+    for _ in 0..<totalProofs:
+      if slotWasFreed:
+        break
+      else:
+        await advanceToNextPeriod()
+        await sleepAsync(1.seconds)
+
+    check not slotWasFreed
+
+    await subscription.unsubscribe()