From 78fa4dd3a3bb491c62df69baf82949b11e985cec Mon Sep 17 00:00:00 2001 From: Eric Mastro Date: Thu, 20 Apr 2023 18:38:40 +1000 Subject: [PATCH] multinode test setup for invalid proof submission tests # Conflicts: # codex/contracts/market.nim # codex/validation.nim # tests/integration/testproofs.nim --- codex/contracts/market.nim | 1 + codex/proving/proving.nim | 9 +- codex/proving/simulated.nim | 13 +- codex/validation.nim | 8 +- tests/integration/multinodes.nim | 122 +++++++++++++++++ tests/integration/nodes.nim | 52 +++++++- tests/integration/testproofs.nim | 222 +++++++++++++++++++++++-------- tests/integration/twonodes.nim | 82 ------------ 8 files changed, 358 insertions(+), 151 deletions(-) create mode 100644 tests/integration/multinodes.nim diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 1c4d516c..acdf63cd 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -4,6 +4,7 @@ import pkg/ethers import pkg/ethers/testing import pkg/upraises import pkg/questionable +import pkg/chronicles import ../market import ./marketplace diff --git a/codex/proving/proving.nim b/codex/proving/proving.nim index 29be9bdd..220d31e7 100644 --- a/codex/proving/proving.nim +++ b/codex/proving/proving.nim @@ -7,6 +7,9 @@ import ../clock export sets +logScope: + topics = "proving" + type Proving* = ref object of RootObj market*: Market @@ -31,7 +34,7 @@ proc `onProve=`*(proving: Proving, callback: OnProve) = func add*(proving: Proving, slot: Slot) = proving.slots.incl(slot) -proc getCurrentPeriod(proving: Proving): Future[Period] {.async.} = +proc getCurrentPeriod*(proving: Proving): Future[Period] {.async.} = let periodicity = await proving.market.periodicity() return periodicity.periodOf(proving.clock.now().u256) @@ -48,10 +51,14 @@ proc removeEndedContracts(proving: Proving) {.async.} = proving.slots.excl(ended) method prove*(proving: Proving, slot: Slot) {.base, async.} = + logScope: + currentPeriod = await proving.getCurrentPeriod() + without onProve =? proving.onProve: raiseAssert "onProve callback not set" try: let proof = await onProve(slot) + debug "submitting proof" await proving.market.submitProof(slot.id, proof) except CatchableError as e: error "Submitting proof failed", msg = e.msg diff --git a/codex/proving/simulated.nim b/codex/proving/simulated.nim index 151bb71f..0730a95f 100644 --- a/codex/proving/simulated.nim +++ b/codex/proving/simulated.nim @@ -11,6 +11,9 @@ type failEveryNProofs: uint proofCount: uint +logScope: + topics = "simulated proving" + func new*(_: type SimulatedProving, market: Market, clock: Clock, @@ -27,21 +30,23 @@ method init(proving: SimulatedProving) {.async.} = "--eth-provider." proving.failEveryNProofs = 0'u -proc onSubmitProofError(error: ref CatchableError) = - error "Submitting invalid proof failed", msg = error.msg +proc onSubmitProofError(error: ref CatchableError, period: UInt256) = + error "Submitting invalid proof failed", period, msg = error.msg method prove(proving: SimulatedProving, slot: Slot) {.async.} = + let period = await proving.getCurrentPeriod() proving.proofCount += 1 if proving.failEveryNProofs > 0'u and proving.proofCount mod proving.failEveryNProofs == 0'u: proving.proofCount = 0 try: + debug "submitting INVALID proof" await proving.market.submitProof(slot.id, newSeq[byte](0)) except ProviderError as e: if not e.revertReason.contains("Invalid proof"): - onSubmitProofError(e) + onSubmitProofError(e, period) except CatchableError as e: - onSubmitProofError(e) + onSubmitProofError(e, period) else: await procCall Proving(proving).prove(slot) diff --git a/codex/validation.nim b/codex/validation.nim index 017e212d..0431179b 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -62,14 +62,18 @@ proc removeSlotsThatHaveEnded(validation: Validation) {.async.} = proc markProofAsMissing(validation: Validation, slotId: SlotId, period: Period) {.async.} = + logScope: + currentPeriod = validation.getCurrentPeriod() + try: if await validation.market.canProofBeMarkedAsMissing(slotId, period): - trace "Marking proof as missing", slotId = $slotId, period = period + trace "Marking proof as missing", slotId = $slotId, periodProofMissed = period await validation.market.markProofAsMissing(slotId, period) + else: trace "Proof not missing", checkedPeriod = period except CancelledError: raise except CatchableError as e: - debug "Marking proof as missing failed", msg = e.msg + error "Marking proof as missing failed", msg = e.msg proc markProofsAsMissing(validation: Validation) {.async.} = for slotId in validation.slots: diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim new file mode 100644 index 00000000..a818551e --- /dev/null +++ b/tests/integration/multinodes.nim @@ -0,0 +1,122 @@ +import std/os +import std/macros +import std/json +import std/httpclient +import pkg/chronicles +import ../ethertest +import ./codexclient +import ./nodes + +export ethertest +export codexclient +export nodes + +template multinodesuite*(name: string, + startNodes: StartNodes, debugNodes: DebugNodes, body: untyped) = + + if (debugNodes.client or debugNodes.provider) and + (enabledLogLevel > LogLevel.DEBUG or + enabledLogLevel == LogLevel.NONE): + echo "" + echo "More test debug logging is available by running the tests with " & + "'-d:chronicles_log_level=DEBUG " & + "-d:chronicles_default_output_device=stdout " & + "-d:chronicles_sinks=textlines'" + echo "" + + ethersuite name: + + var running: seq[RunningNode] + var bootstrap: string + + proc newNodeProcess(index: int, + addlOptions: seq[string], + debug: bool): (NodeProcess, string, Address) = + + if index > accounts.len - 1: + raiseAssert("Cannot start node at index " & $index & + ", not enough eth accounts.") + + let datadir = getTempDir() / "Codex" & $index + var options = @[ + "--api-port=" & $(8080 + index), + "--data-dir=" & datadir, + "--nat=127.0.0.1", + "--disc-ip=127.0.0.1", + "--disc-port=" & $(8090 + index), + "--eth-account=" & $accounts[index]] + .concat(addlOptions) + if debug: options.add "--log-level=INFO;DEBUG: " & debugNodes.topics + let node = startNode(options, debug = debug) + (node, datadir, accounts[index]) + + proc newCodexClient(index: int): CodexClient = + CodexClient.new("http://localhost:" & $(8080 + index) & "/api/codex/v1") + + proc startClientNode() = + let index = running.len + let (node, datadir, account) = newNodeProcess( + index, @["--persistence"], debugNodes.client) + let restClient = newCodexClient(index) + running.add RunningNode.new(Role.Client, node, restClient, datadir, + account) + if debugNodes.client: + debug "started new client node and codex client", + restApiPort = 8080 + index, discPort = 8090 + index, account + + proc startProviderNode(failEveryNProofs: uint = 0) = + let index = running.len + let (node, datadir, account) = newNodeProcess(index, @[ + "--bootstrap-node=" & bootstrap, + "--persistence", + "--simulate-proof-failures=" & $failEveryNProofs], + debugNodes.provider) + let restClient = newCodexClient(index) + running.add RunningNode.new(Role.Provider, node, restClient, datadir, + account) + if debugNodes.provider: + debug "started new provider node and codex client", + restApiPort = 8080 + index, discPort = 8090 + index, account + + proc startValidatorNode() = + let index = running.len + let (node, datadir, account) = newNodeProcess(index, @[ + "--bootstrap-node=" & bootstrap, + "--validator"], + debugNodes.validator) + let restClient = newCodexClient(index) + running.add RunningNode.new(Role.Validator, node, restClient, datadir, + account) + if debugNodes.validator: + debug "started new validator node and codex client", + restApiPort = 8080 + index, discPort = 8090 + index, account + + proc clients(): seq[RunningNode] = + running.filter(proc(r: RunningNode): bool = r.role == Role.Client) + + proc providers(): seq[RunningNode] = + running.filter(proc(r: RunningNode): bool = r.role == Role.Provider) + + proc validators(): seq[RunningNode] = + running.filter(proc(r: RunningNode): bool = r.role == Role.Validator) + + setup: + for i in 0..>> proof submitted: ", event.proof submitted.add(event.proof) proofSubmitted.complete() proofSubmitted = newFuture[void]("proofSubmitted") subscription = await marketplace.subscribe(ProofSubmitted, onProofSubmitted) missed = 0.u256 - slotId = SlotId(array[32, byte].default) - validator = startValidator() + slotId = SlotId(array[32, byte].default) # ensure we aren't reusing from prev test + downtime = @[] + periodicity = Periodicity(seconds: period.u256) teardown: await subscription.unsubscribe() - validator.stopValidator() - proc waitUntilPurchaseIsStarted(proofProbability: uint64 = 3, - duration: uint64 = 100 * period, - expiry: uint64 = 30) {.async.} = + proc getCurrentPeriod(): Future[Period] {.async.} = + return periodicity.periodOf(await provider.currentTime()) + + proc waitUntilPurchaseIsStarted(proofProbability: uint64 = 1, + duration: uint64 = 12 * period, + expiry: uint64 = 4 * period, + failEveryNProofs: uint): Future[Period] {.async.} = if clients().len < 1 or providers().len < 1: raiseAssert("must start at least one client and one provider") @@ -153,9 +157,15 @@ invalidproofsuite "Simulate invalid proofs", debugClient=false, debugProvider=fa let client = clients()[0].restClient let storageProvider = providers()[0].restClient + # The last period is the period in which the slot is freed, and therefore + # proofs cannot be submitted. That means that the duration must go an + # additional period longer to allow for invalid proofs to be submitted in + # the second to last period and counted as missed in the last period. + let dur = duration + (1 * period) + discard storageProvider.postAvailability( size=0xFFFFF, - duration=duration, + duration=dur, minPrice=300 ) let cid = client.upload("some file contents " & $ getTime().toUnix) @@ -163,56 +173,152 @@ invalidproofsuite "Simulate invalid proofs", debugClient=false, debugProvider=fa let purchase = client.requestStorage( cid, expiry=expiry, - duration=duration, + duration=dur, proofProbability=proofProbability, reward=400 ) check eventually client.getPurchase(purchase){"state"} == %"started" + debug "purchase state", state = client.getPurchase(purchase){"state"} let requestId = RequestId.fromHex client.getPurchase(purchase){"requestId"}.getStr slotId = slotId(requestId, 0.u256) + return await getCurrentPeriod() + + proc inDowntime: Future[bool] {.async.} = + var periodPointer = await marketplace.getPointer(slotId) + return periodPointer < periodDowntime + + proc advanceToNextPeriod {.async.} = + let periodicity = Periodicity(seconds: period.u256) + let currentPeriod = periodicity.periodOf(await provider.currentTime()) + let endOfPeriod = periodicity.periodEnd(currentPeriod) + await provider.advanceTimeTo(endOfPeriod + 1) + + proc advanceToNextPeriodStart {.async.} = + let currentPeriod = await getCurrentPeriod() + let startOfPeriod = periodicity.periodEnd(currentPeriod + 1) + await provider.advanceTimeTo(startOfPeriod) - proc advanceToNextPeriod() {.async.} = - await provider.advanceTime(period.u256) + proc advanceToCurrentPeriodEnd {.async.} = + let currentPeriod = await getCurrentPeriod() + let endOfPeriod = periodicity.periodEnd(currentPeriod) + await provider.advanceTimeTo(endOfPeriod) - proc waitForProvingRounds(rounds: Positive) {.async.} = - var rnds = rounds - 1 # proof round runs prior to advancing - missed += await marketplace.missingProofs(slotId) + proc recordDowntime() {.async.} = + let currentPeriod = await getCurrentPeriod() + let isInDowntime = await inDowntime() + downtime.add (currentPeriod, isInDowntime) + debug "downtime recorded", currentPeriod, isInDowntime - while rnds > 0: + proc waitUntilSlotNoLongerFilled() {.async.} = + var i = 0 + # await recordDowntime() + # await advanceToNextPeriodStart() + while (await marketplace.slotState(slotId)) == SlotState.Filled: + let currentPeriod = await getCurrentPeriod() + # await advanceToCurrentPeriodEnd() await advanceToNextPeriod() - rnds -= 1 + debug "--------------- PERIOD START ---------------", currentPeriod + await recordDowntime() + # debug "--------------- PERIOD END ---------------", currentPeriod + await sleepAsync(1.seconds) # let validation happen + debug "Checked previous period for missed proofs", missedProofs = $(await marketplace.missingProofs(slotId)) + i += 1 + # downtime.del downtime.len - 1 # remove last downtime as it is an additional proving round adding to duration for checking proofs, and we are offset by one as we are checking previous periods + missed = await marketplace.missingProofs(slotId) + debug "Total missed proofs", missed - proc invalid(proofs: seq[seq[byte]]): uint = - proofs.count(@[]).uint + proc expectedInvalid(startPeriod: Period, + failEveryNProofs: uint, + periods: Positive): seq[(Period, bool)] = + # Create a seq of bools where each bool represents a proving round. + # If true, an invalid proof should have been sent. + var p = startPeriod + 1.u256 + var invalid: seq[(Period, bool)] = @[] + if failEveryNProofs == 0: + for i in 0..