From 8589e63d3457702606a85929559b8a32a595f06a Mon Sep 17 00:00:00 2001 From: markspanbroek Date: Tue, 12 Mar 2024 09:18:25 +0100 Subject: [PATCH] Update contract (#734) * Update codex-contracts-eth * contracts: update G2Point definition * integration: disable automatic advancing of time reason: makes reasoning about timing in tests harder, because the period is set to 60 seconds in the marketplace configuration, but this code switches to a new period every 500 milliseconds * integration: fix parameters of marketplace payouts test * integration: update test settings * integration: fix typo * integration: workaround for hardhat issue Subscriptions expire after 5 minutes when using websockets. Use http and polling instead. * integration: remove origDatasetSizeInBlocks * integration: fix proof parameters for test * integration: do not log output by default * integration: fix failure rate in test * integration: fix warning * integration: include clock in logs * integration: allow for more periods 5 periods was cutting it close, if we get too much pointer downtime, then the test would fail --- codex/contracts/proofs.nim | 14 ++- codex/slots/proofs/backends/converters.nim | 16 +-- tests/examples.nim | 4 +- tests/integration/marketplacesuite.nim | 24 +---- tests/integration/multinodes.nim | 5 +- tests/integration/testmarketplace.nim | 34 +++--- tests/integration/testproofs.nim | 114 +++++++++++---------- vendor/codex-contracts-eth | 2 +- 8 files changed, 103 insertions(+), 110 deletions(-) diff --git a/codex/contracts/proofs.nim b/codex/contracts/proofs.nim index 3b84a2e7..a7a59351 100644 --- a/codex/contracts/proofs.nim +++ b/codex/contracts/proofs.nim @@ -10,13 +10,20 @@ type G1Point* = object x*: UInt256 y*: UInt256 + # A field element F_{p^2} encoded as `real + i * imag` + Fp2Element* = object + real*: UInt256 + imag*: UInt256 G2Point* = object - x*: array[2, UInt256] - y*: array[2, UInt256] + x*: Fp2Element + y*: Fp2Element func solidityType*(_: type G1Point): string = solidityType(G1Point.fieldTypes) +func solidityType*(_: type Fp2Element): string = + solidityType(Fp2Element.fieldTypes) + func solidityType*(_: type G2Point): string = solidityType(G2Point.fieldTypes) @@ -26,6 +33,9 @@ func solidityType*(_: type Groth16Proof): string = func encode*(encoder: var AbiEncoder, point: G1Point) = encoder.write(point.fieldValues) +func encode*(encoder: var AbiEncoder, element: Fp2Element) = + encoder.write(element.fieldValues) + func encode*(encoder: var AbiEncoder, point: G2Point) = encoder.write(point.fieldValues) diff --git a/codex/slots/proofs/backends/converters.nim b/codex/slots/proofs/backends/converters.nim index b405ab1f..60c64f5c 100644 --- a/codex/slots/proofs/backends/converters.nim +++ b/codex/slots/proofs/backends/converters.nim @@ -54,14 +54,14 @@ func toG1*(g: CircomG1): G1Point = func toG2*(g: CircomG2): G2Point = G2Point( - x: [ - UInt256.fromBytesLE(g.x[0]), - UInt256.fromBytesLE(g.x[1]) - ], - y: [ - UInt256.fromBytesLE(g.y[0]), - UInt256.fromBytesLE(g.y[1]) - ]) + x: Fp2Element( + real: UInt256.fromBytesLE(g.x[0]), + imag: UInt256.fromBytesLE(g.x[1]) + ), + y: Fp2Element( + real: UInt256.fromBytesLE(g.y[0]), + imag: UInt256.fromBytesLE(g.y[1]) + )) func toGroth16Proof*(proof: CircomProof): Groth16Proof = Groth16Proof( diff --git a/tests/examples.nim b/tests/examples.nim index c70d0dbb..bb506438 100644 --- a/tests/examples.nim +++ b/tests/examples.nim @@ -78,8 +78,8 @@ proc example(_: type G1Point): G1Point = proc example(_: type G2Point): G2Point = G2Point( - x: [UInt256.example, UInt256.example], - y: [UInt256.example, UInt256.example] + x: Fp2Element(real: UInt256.example, imag: UInt256.example), + y: Fp2Element(real: UInt256.example, imag: UInt256.example) ) proc example*(_: type Groth16Proof): Groth16Proof = diff --git a/tests/integration/marketplacesuite.nim b/tests/integration/marketplacesuite.nim index 4ed7fb9f..35888847 100644 --- a/tests/integration/marketplacesuite.nim +++ b/tests/integration/marketplacesuite.nim @@ -19,7 +19,6 @@ template marketplacesuite*(name: string, body: untyped) = var period: uint64 var periodicity: Periodicity var token {.inject, used.}: Erc20Token - var continuousMineFut: Future[void] proc getCurrentPeriod(): Future[Period] {.async.} = return periodicity.periodOf(await ethProvider.currentTime()) @@ -60,12 +59,6 @@ template marketplacesuite*(name: string, body: untyped) = maxCollateral=200.u256 ) - proc validateRequest(nodes, tolerance, origDatasetSizeInBlocks: uint) = - if nodes > 1: - doAssert(origDatasetSizeInBlocks >= 3, - "dataset size must be greater than or equal to 3 blocks with " & - "more than one node") - proc requestStorage(client: CodexClient, cid: Cid, proofProbability: uint64 = 1, @@ -74,9 +67,7 @@ template marketplacesuite*(name: string, body: untyped) = collateral = 100.u256, expiry: uint64 = 4.periods, nodes = providers().len, - tolerance = 0, - origDatasetSizeInBlocks: int): Future[PurchaseId] {.async.} = - + tolerance = 0): Future[PurchaseId] {.async.} = let expiry = (await ethProvider.currentTime()) + expiry.u256 let id = client.requestStorage( @@ -92,14 +83,6 @@ template marketplacesuite*(name: string, body: untyped) = return id - proc continuouslyAdvanceEvery(every: chronos.Duration) {.async.} = - try: - while true: - await advanceToNextPeriod() - await sleepAsync(every) - except CancelledError: - discard - setup: # TODO: This is currently the address of the marketplace with a dummy # verifier. Use real marketplace address, `Marketplace.address` once we @@ -112,9 +95,4 @@ template marketplacesuite*(name: string, body: untyped) = period = config.proofs.period.truncate(uint64) periodicity = Periodicity(seconds: period.u256) - continuousMineFut = continuouslyAdvanceEvery(chronos.millis(500)) - - teardown: - await continuousMineFut.cancelAndWait() - body diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 10eff165..a2976b48 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -221,7 +221,10 @@ template multinodesuite*(name: string, body: untyped) = running.add RunningNode(role: Role.Hardhat, node: node) try: - ethProvider = JsonRpcProvider.new("ws://localhost:8545") + # Workaround for https://github.com/NomicFoundation/hardhat/issues/2053 + # Do not use websockets, but use http and polling to stop subscriptions + # from being removed after 5 minutes + ethProvider = JsonRpcProvider.new("http://localhost:8545") # if hardhat was NOT started by the test, take a snapshot so it can be # reverted in the test teardown if nodeConfigs.hardhat.isNil: diff --git a/tests/integration/testmarketplace.nim b/tests/integration/testmarketplace.nim index e9973d8f..605f6ee0 100644 --- a/tests/integration/testmarketplace.nim +++ b/tests/integration/testmarketplace.nim @@ -13,38 +13,35 @@ marketplacesuite "Marketplace payouts": clients: CodexConfig() - .nodes(1) + .nodes(1), # .debug() # uncomment to enable console log output.debug() - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("node", "erasure"), + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("node", "erasure"), providers: CodexConfig() .nodes(1) # .debug() # uncomment to enable console log output - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("node", "marketplace", "sales", "reservations", "node", "proving", "clock"), + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("node", "marketplace", "sales", "reservations", "node", "proving", "clock"), ): let reward = 400.u256 - let duration = 100.periods + let duration = 10.periods let collateral = 200.u256 - let expiry = 10.periods - let datasetSizeInBlocks = 3 - let data = await RandomChunker.example(blocks=datasetSizeInBlocks) + let expiry = 5.periods + let data = await RandomChunker.example(blocks=8) let client = clients()[0] let provider = providers()[0] let clientApi = client.client let providerApi = provider.client let startBalanceProvider = await token.balanceOf(provider.ethAccount) let startBalanceClient = await token.balanceOf(client.ethAccount) - # original data = 3 blocks so slot size will be 4 blocks - let slotSize = (DefaultBlockSize * 4.NBytes).Natural.u256 # provider makes storage available discard providerApi.postAvailability( - # make availability size large enough to only fill 1 slot, thus causing a - # cancellation - size=slotSize, + # make availability size small enough that we can't fill all the slots, + # thus causing a cancellation + size=(data.len div 2).u256, duration=duration.u256, minPrice=reward, maxCollateral=collateral) @@ -57,7 +54,7 @@ marketplacesuite "Marketplace payouts": let subscription = await marketplace.subscribe(SlotFilled, onSlotFilled) - # client requests storage but requires two nodes to host the content + # client requests storage but requires multiple slots to host the content let id = await clientApi.requestStorage( cid, duration=duration, @@ -65,18 +62,17 @@ marketplacesuite "Marketplace payouts": expiry=expiry, collateral=collateral, nodes=3, - tolerance=1, - origDatasetSizeInBlocks=datasetSizeInBlocks + tolerance=1 ) # wait until one slot is filled - check eventually slotIdxFilled.isSome + check eventually(slotIdxFilled.isSome, timeout=expiry.int * 1000) # wait until sale is cancelled without requestId =? clientApi.requestId(id): fail() let slotId = slotId(requestId, !slotIdxFilled) - check eventually(providerApi.saleStateIs(slotId, "SaleCancelled")) + check eventually(providerApi.saleStateIs(slotId, "SaleCancelled"), timeout=expiry.int * 1000) check eventually ( let endBalanceProvider = (await token.balanceOf(provider.ethAccount)); diff --git a/tests/integration/testproofs.nim b/tests/integration/testproofs.nim index 43363135..3e2c4dfd 100644 --- a/tests/integration/testproofs.nim +++ b/tests/integration/testproofs.nim @@ -22,32 +22,35 @@ marketplacesuite "Hosts submit regular proofs": clients: CodexConfig() - .nodes(1) + .nodes(1), # .debug() # uncomment to enable console log output - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("node"), + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("node"), providers: CodexConfig() .nodes(1) # .debug() # uncomment to enable console log output - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("marketplace", "sales", "reservations", "node"), + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("marketplace", "sales", "reservations", "node"), ): let client0 = clients()[0].client - let totalPeriods = 50 - let datasetSizeInBlocks = 2 + let expiry = 5.periods + let duration = expiry + 5.periods - let data = await RandomChunker.example(blocks=1) - createAvailabilities(data.len, totalPeriods.periods) + let data = await RandomChunker.example(blocks=8) + createAvailabilities(data.len * 2, duration) # TODO: better value for data.len let cid = client0.upload(data).get let purchaseId = await client0.requestStorage( cid, - duration=totalPeriods.periods, - origDatasetSizeInBlocks = datasetSizeInBlocks) - check eventually client0.purchaseStateIs(purchaseId, "started") + expiry=expiry, + duration=duration, + nodes=3, + tolerance=1 + ) + check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000) var proofWasSubmitted = false proc onProofSubmitted(event: ProofSubmitted) = @@ -55,8 +58,7 @@ marketplacesuite "Hosts submit regular proofs": let subscription = await marketplace.subscribe(ProofSubmitted, onProofSubmitted) - let currentPeriod = await getCurrentPeriod() - check eventuallyP(proofWasSubmitted, currentPeriod + totalPeriods.u256 + 1) + check eventually(proofWasSubmitted, timeout=(duration - expiry).int * 1000) await subscription.unsubscribe() @@ -74,54 +76,55 @@ marketplacesuite "Simulate invalid proofs": clients: CodexConfig() - .nodes(1) + .nodes(1), # .debug() # uncomment to enable console log output - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("node"), + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("node", "clock"), providers: CodexConfig() .nodes(1) - .simulateProofFailuresFor(providerIdx=0, failEveryNProofs=1) + .simulateProofFailuresFor(providerIdx=0, failEveryNProofs=1), # .debug() # uncomment to enable console log output - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("marketplace", "sales", "reservations", "node"), + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("marketplace", "sales", "reservations", "node", "clock"), validators: CodexConfig() .nodes(1) - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log # .debug() # uncomment to enable console log output - .withLogTopics("validator", "onchain", "ethers") + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("validator", "onchain", "ethers", "clock") ): let client0 = clients()[0].client - let totalPeriods = 50 + let expiry = 5.periods + let duration = expiry + 10.periods - let datasetSizeInBlocks = 2 - let data = await RandomChunker.example(blocks=datasetSizeInBlocks) - createAvailabilities(data.len, totalPeriods.periods) + let data = await RandomChunker.example(blocks=8) + createAvailabilities(data.len * 2, duration) # TODO: better value for data.len let cid = client0.upload(data).get let purchaseId = await client0.requestStorage( cid, - expiry=10.periods, - duration=totalPeriods.periods, - origDatasetSizeInBlocks=datasetSizeInBlocks) + expiry=expiry, + duration=duration, + nodes=3, + tolerance=1, + proofProbability=1 + ) let requestId = client0.requestId(purchaseId).get - check eventually client0.purchaseStateIs(purchaseId, "started") + check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000) var slotWasFreed = false proc onSlotFreed(event: SlotFreed) = - if event.requestId == requestId and - event.slotIndex == 0.u256: # assume only one slot, so index 0 + if event.requestId == requestId: slotWasFreed = true let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) - let currentPeriod = await getCurrentPeriod() - check eventuallyP(slotWasFreed, currentPeriod + totalPeriods.u256 + 1) + check eventually(slotWasFreed, timeout=(duration - expiry).int * 1000) await subscription.unsubscribe() @@ -131,55 +134,58 @@ marketplacesuite "Simulate invalid proofs": clients: CodexConfig() - .nodes(1) + .nodes(1), # .debug() # uncomment to enable console log output - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("node"), + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("marketplace", "sales", "reservations", "node", "clock"), providers: CodexConfig() .nodes(1) - .simulateProofFailuresFor(providerIdx=0, failEveryNProofs=3) + .simulateProofFailuresFor(providerIdx=0, failEveryNProofs=1), # .debug() # uncomment to enable console log output - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("marketplace", "sales", "reservations", "node"), + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("marketplace", "sales", "reservations", "node"), validators: CodexConfig() .nodes(1) # .debug() - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("validator", "onchain", "ethers") + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("validator", "onchain", "ethers", "clock") ): let client0 = clients()[0].client - let totalPeriods = 25 + let expiry = 5.periods + # In 2 periods you cannot have enough invalid proofs submitted: + let duration = expiry + 2.periods - let datasetSizeInBlocks = 2 - let data = await RandomChunker.example(blocks=datasetSizeInBlocks) - createAvailabilities(data.len, totalPeriods.periods) + let data = await RandomChunker.example(blocks=8) + createAvailabilities(data.len * 2, duration) # TODO: better value for data.len let cid = client0.upload(data).get let purchaseId = await client0.requestStorage( cid, - expiry=10.periods, - duration=totalPeriods.periods, - origDatasetSizeInBlocks=datasetSizeInBlocks) + expiry=expiry, + duration=duration, + nodes=3, + tolerance=1, + proofProbability=1 + ) let requestId = client0.requestId(purchaseId).get - check eventually client0.purchaseStateIs(purchaseId, "started") + check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000) var slotWasFreed = false proc onSlotFreed(event: SlotFreed) = - if event.requestId == requestId and - event.slotIndex == 0.u256: + if event.requestId == requestId: slotWasFreed = true let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) # check not freed - let currentPeriod = await getCurrentPeriod() - check not eventuallyP(slotWasFreed, currentPeriod + totalPeriods.u256 + 1) + await sleepAsync((duration - expiry).int.seconds) + check not slotWasFreed await subscription.unsubscribe() diff --git a/vendor/codex-contracts-eth b/vendor/codex-contracts-eth index 6c9f797f..118ee0b2 160000 --- a/vendor/codex-contracts-eth +++ b/vendor/codex-contracts-eth @@ -1 +1 @@ -Subproject commit 6c9f797f408608958714024b9055fcc330e3842f +Subproject commit 118ee0b22b2d12c8fbf3376cf201d203d0a7cf97