[marketplace] Simulate invalid proof submissions

Closes #392.

Create a simulated prover that sends an invalid proof (seq[byte] of length 0) every `N` proofs, where `N` is defined by a new cli param `--simulate-failed-proofs`.

codex/codex.nim

@@ -138,7 +138,10 @@ proc new(_: type Contracts,
   var validator: ?ValidatorInteractions
   if config.persistence:
     let purchasing = Purchasing.new(market, clock)
-    let proving = Proving.new(market, clock)
+    let proving = if config.simulateProofFailures > 0:
+                  SimulatedProving.new(market, clock, provider,
+                                       config.simulateProofFailures)
+                  else: Proving.new(market, clock)
     let sales = Sales.new(market, clock, proving, repo)
     client = some ClientInteractions.new(clock, purchasing)
     host = some HostInteractions.new(clock, sales, proving)

codex/conf.nim

@@ -230,6 +230,12 @@ type
         name: "validator-max-slots"
       .}: int
       
+      simulateProofFailures* {.
+        desc: "Simulates proof failures once every N proofs. 0 = disabled."
+        defaultValue: 0
+        name: "simulate-proof-failures"
+      .}: uint
+
     of initNode:
       discard
 

codex/contracts/market.nim

@@ -39,6 +39,9 @@ proc approveFunds(market: OnChainMarket, amount: UInt256) {.async.} =
 method getSigner*(market: OnChainMarket): Future[Address] {.async.} =
   return await market.signer.getAddress()
 
+method isMainnet*(market: OnChainMarket): Future[bool] {.async.} =
+  return (await market.signer.provider.getChainId()) == 1.u256
+
 method periodicity*(market: OnChainMarket): Future[Periodicity] {.async.} =
   let config = await market.contract.config()
   let period = config.proofs.period

codex/market.nim

@@ -26,6 +26,9 @@ type
 method getSigner*(market: Market): Future[Address] {.base, async.} =
   raiseAssert("not implemented")
 
+method isMainnet*(market: Market): Future[bool] {.async.} =
+  raiseAssert("not implemented")
+
 method periodicity*(market: Market): Future[Periodicity] {.base, async.} =
   raiseAssert("not implemented")
 

codex/proving.nim

@@ -1,87 +1,5 @@
-import std/sets
-import pkg/upraises
-import pkg/questionable
-import pkg/chronicles
-import ./market
-import ./clock
+import ./proving/proving
+import ./proving/simulated
 
-export sets
-
-type
-  Proving* = ref object
-    market: Market
-    clock: Clock
-    loop: ?Future[void]
-    slots*: HashSet[Slot]
-    onProve: ?OnProve
-  OnProve* = proc(slot: Slot): Future[seq[byte]] {.gcsafe, upraises: [].}
-
-func new*(_: type Proving, market: Market, clock: Clock): Proving =
-  Proving(market: market, clock: clock)
-
-proc onProve*(proving: Proving): ?OnProve =
-  proving.onProve
-
-proc `onProve=`*(proving: Proving, callback: OnProve) =
-  proving.onProve = some callback
-
-func add*(proving: Proving, slot: Slot) =
-  proving.slots.incl(slot)
-
-proc getCurrentPeriod(proving: Proving): Future[Period] {.async.} =
-  let periodicity = await proving.market.periodicity()
-  return periodicity.periodOf(proving.clock.now().u256)
-
-proc waitUntilPeriod(proving: Proving, period: Period) {.async.} =
-  let periodicity = await proving.market.periodicity()
-  await proving.clock.waitUntil(periodicity.periodStart(period).truncate(int64))
-
-proc removeEndedContracts(proving: Proving) {.async.} =
-  var ended: HashSet[Slot]
-  for slot in proving.slots:
-    let state = await proving.market.slotState(slot.id)
-    if state != SlotState.Filled:
-      ended.incl(slot)
-  proving.slots.excl(ended)
-
-proc prove(proving: Proving, slot: Slot) {.async.} =
-  without onProve =? proving.onProve:
-    raiseAssert "onProve callback not set"
-  try:
-    let proof = await onProve(slot)
-    await proving.market.submitProof(slot.id, proof)
-  except CatchableError as e:
-    error "Submitting proof failed", msg = e.msg
-
-proc run(proving: Proving) {.async.} =
-  try:
-    while true:
-      let currentPeriod = await proving.getCurrentPeriod()
-      await proving.removeEndedContracts()
-      for slot in proving.slots:
-        let id = slot.id
-        if (await proving.market.isProofRequired(id)) or
-           (await proving.market.willProofBeRequired(id)):
-          asyncSpawn proving.prove(slot)
-      await proving.waitUntilPeriod(currentPeriod + 1)
-  except CancelledError:
-    discard
-  except CatchableError as e:
-    error "Proving failed", msg = e.msg
-
-proc start*(proving: Proving) {.async.} =
-  if proving.loop.isSome:
-    return
-
-  proving.loop = some proving.run()
-
-proc stop*(proving: Proving) {.async.} =
-  if loop =? proving.loop:
-    proving.loop = Future[void].none
-    if not loop.finished:
-      await loop.cancelAndWait()
-
-proc subscribeProofSubmission*(proving: Proving,
-                               callback: OnProofSubmitted):
-                              Future[Subscription] =
-  proving.market.subscribeProofSubmission(callback)
+export proving
+export simulated

codex/proving/proving.nim

@@ -0,0 +1,92 @@
+import std/sets
+import pkg/upraises
+import pkg/questionable
+import pkg/chronicles
+import ../market
+import ../clock
+
+export sets
+
+type
+  Proving* = ref object of RootObj
+    market*: Market
+    clock: Clock
+    loop: ?Future[void]
+    slots*: HashSet[Slot]
+    onProve: ?OnProve
+  OnProve* = proc(slot: Slot): Future[seq[byte]] {.gcsafe, upraises: [].}
+
+func new*(T: type Proving, market: Market, clock: Clock): T =
+  T(market: market, clock: clock)
+
+method init*(proving: Proving) {.base, async.} =
+  discard
+
+proc onProve*(proving: Proving): ?OnProve =
+  proving.onProve
+
+proc `onProve=`*(proving: Proving, callback: OnProve) =
+  proving.onProve = some callback
+
+func add*(proving: Proving, slot: Slot) =
+  proving.slots.incl(slot)
+
+proc getCurrentPeriod(proving: Proving): Future[Period] {.async.} =
+  let periodicity = await proving.market.periodicity()
+  return periodicity.periodOf(proving.clock.now().u256)
+
+proc waitUntilPeriod(proving: Proving, period: Period) {.async.} =
+  let periodicity = await proving.market.periodicity()
+  await proving.clock.waitUntil(periodicity.periodStart(period).truncate(int64))
+
+proc removeEndedContracts(proving: Proving) {.async.} =
+  var ended: HashSet[Slot]
+  for slot in proving.slots:
+    let state = await proving.market.slotState(slot.id)
+    if state != SlotState.Filled:
+      ended.incl(slot)
+  proving.slots.excl(ended)
+
+method prove*(proving: Proving, slot: Slot) {.base, async.} =
+  without onProve =? proving.onProve:
+    raiseAssert "onProve callback not set"
+  try:
+    let proof = await onProve(slot)
+    await proving.market.submitProof(slot.id, proof)
+  except CatchableError as e:
+    error "Submitting proof failed", msg = e.msg
+
+proc run(proving: Proving) {.async.} =
+  try:
+    while true:
+      let currentPeriod = await proving.getCurrentPeriod()
+      await proving.removeEndedContracts()
+      for slot in proving.slots:
+        let id = slot.id
+        if (await proving.market.isProofRequired(id)) or
+           (await proving.market.willProofBeRequired(id)):
+          asyncSpawn proving.prove(slot)
+      await proving.waitUntilPeriod(currentPeriod + 1)
+  except CancelledError:
+    discard
+  except CatchableError as e:
+    error "Proving failed", msg = e.msg
+
+proc start*(proving: Proving) {.async.} =
+  if proving.loop.isSome:
+    return
+
+  await proving.init()
+
+  proving.loop = some proving.run()
+
+proc stop*(proving: Proving) {.async.} =
+  if loop =? proving.loop:
+    proving.loop = Future[void].none
+    if not loop.finished:
+      await loop.cancelAndWait()
+
+proc subscribeProofSubmission*(proving: Proving,
+                               callback: OnProofSubmitted):
+                              Future[Subscription] =
+  proving.market.subscribeProofSubmission(callback)

codex/proving/simulated.nim

@@ -0,0 +1,47 @@
+import std/strutils
+import pkg/chronicles
+import pkg/ethers
+import pkg/ethers/testing
+import ../market
+import ../clock
+import ./proving
+
+type
+  SimulatedProving* = ref object of Proving
+    failEveryNProofs: uint
+    proofCount: uint
+
+func new*(_: type SimulatedProving,
+          market: Market,
+          clock: Clock,
+          failEveryNProofs: uint): SimulatedProving =
+
+  let p = SimulatedProving.new(market, clock)
+  p.failEveryNProofs = failEveryNProofs
+  return p
+
+method init(proving: SimulatedProving) {.async.} =
+  if proving.failEveryNProofs > 0'u and await proving.market.isMainnet():
+    warn "Connected to mainnet, simulated proof failures will not be run. " &
+         "Consider changing the value of --simulate-proof-failures and/or " &
+         "--eth-provider."
+    proving.failEveryNProofs = 0'u
+
+proc onSubmitProofError(error: ref CatchableError) =
+  error "Submitting invalid proof failed", msg = error.msg
+
+method prove(proving: SimulatedProving, slot: Slot) {.async.} =
+  proving.proofCount += 1
+  if proving.failEveryNProofs > 0'u and
+     proving.proofCount mod proving.failEveryNProofs == 0'u:
+    proving.proofCount = 0
+    try:
+      await proving.market.submitProof(slot.id, newSeq[byte](0))
+    except ProviderError as e:
+      if not e.revertReason.contains("Invalid proof"):
+        onSubmitProofError(e)
+    except CatchableError as e:
+      onSubmitProofError(e)
+  else:
+    await procCall Proving(proving).prove(slot)
+

tests/codex/helpers/mockmarket.nim

@@ -12,6 +12,7 @@ export tables
 
 type
   MockMarket* = ref object of Market
+    isMainnet: bool
     periodicity: Periodicity
     activeRequests*: Table[Address, seq[RequestId]]
     activeSlots*: Table[Address, seq[SlotId]]
@@ -100,6 +101,12 @@ proc new*(_: type MockMarket): MockMarket =
 method getSigner*(market: MockMarket): Future[Address] {.async.} =
   return market.signer
 
+method isMainnet*(market: MockMarket): Future[bool] {.async.} =
+  return market.isMainnet
+
+method setMainnet*(market: MockMarket, isMainnet: bool) =
+  market.isMainnet = isMainnet
+
 method periodicity*(mock: MockMarket): Future[Periodicity] {.async.} =
   return Periodicity(seconds: mock.config.proofs.period)
 

tests/codex/testproving.nim

@@ -1,5 +1,7 @@
+import std/sequtils
 import pkg/asynctest
 import pkg/chronos
+import pkg/chronicles # DELETE ME
 import pkg/codex/proving
 import ./helpers/mockmarket
 import ./helpers/mockclock
@@ -122,3 +124,73 @@ suite "Proving":
     check eventually receivedIds == @[slot.id] and receivedProofs == @[proof]
 
     await subscription.unsubscribe()
+
+suite "Simulated proving":
+
+  var proving: SimulatedProving
+  var subscription: Subscription
+  var market: MockMarket
+  var clock: MockClock
+  var submitted: seq[seq[byte]]
+  var proof: seq[byte]
+  let slot = Slot.example
+  var proofSubmitted: Future[void]
+
+  setup:
+    proof = exampleProof()
+    submitted = @[]
+    market = MockMarket.new()
+    clock = MockClock.new()
+    proofSubmitted = newFuture[void]("proofSubmitted")
+
+  teardown:
+    await subscription.unsubscribe()
+    await proving.stop()
+
+  proc newSimulatedProving(failEveryNProofs: uint) {.async.} =
+    proc onProofSubmission(id: SlotId, proof: seq[byte]) =
+      submitted.add(proof)
+      proofSubmitted.complete()
+      proofSubmitted = newFuture[void]("proofSubmitted")
+
+    proving = SimulatedProving.new(market, clock, failEveryNProofs)
+    proving.onProve = proc (slot: Slot): Future[seq[byte]] {.async.} =
+      return proof
+    subscription = await proving.subscribeProofSubmission(onProofSubmission)
+    proving.add(slot)
+    market.slotState[slot.id] = SlotState.Filled
+    market.setProofRequired(slot.id, true)
+    await proving.start()
+
+  proc advanceToNextPeriod(market: Market) {.async.} =
+    let periodicity = await market.periodicity()
+    clock.advance(periodicity.seconds.truncate(int64))
+
+  proc waitForProvingRounds(market: Market, rounds: uint) {.async.} =
+    var rnds = rounds - 1 # proof round runs prior to advancing
+    while rnds > 0:
+      await market.advanceToNextPeriod()
+      await proofSubmitted
+      rnds -= 1
+
+  test "submits invalid proof every 3 proofs":
+    let failEveryNProofs = 3'u
+    let totalProofs = 6'u
+    await newSimulatedProving(failEveryNProofs)
+    await market.waitForProvingRounds(totalProofs)
+    check submitted == @[proof, proof, @[], proof, proof, @[]]
+
+  test "does not submit invalid proofs when failEveryNProofs is 0":
+    let failEveryNProofs = 0'u
+    let totalProofs = 6'u
+    await newSimulatedProving(failEveryNProofs)
+    await market.waitForProvingRounds(totalProofs)
+    check submitted == proof.repeat(totalProofs)
+
+  test "does not submit invalid proofs when current chain is mainnet":
+    let failEveryNProofs = 3'u
+    let totalProofs = 6'u
+    market.setMainnet(true)
+    await newSimulatedProving(failEveryNProofs)
+    await market.waitForProvingRounds(totalProofs)
+    check submitted == proof.repeat(totalProofs)

tests/integration/testproofs.nim

@@ -95,3 +95,16 @@ twonodessuite "Proving integration test", debug1=false, debug2=false:
 
     await subscription.unsubscribe()
     stopValidator(validator)
+
+  test "simulates invalid proof every N proofs":
+    # TODO: waiting on validation work to be completed before these tests are possible
+    # 1. instantiate node manually (startNode) with --simulate-failed-proofs=3
+    # 2. check that the number of expected proofs are missed
+
+  test "does not simulate invalid proof when --simulate-failed-proofs is 0":
+    # 1. instantiate node manually (startNode) with --simulate-failed-proofs=0
+    # 2. check that the number of expected missed proofs is 0
+
+  test "does not simulate invalid proof when chainId is 1":
+    # 1. instantiate node manually (startNode) with --simulate-failed-proofs=3
+    # 2. check that the number of expected missed proofs is 0