status-im
/
nim-codex
mirror of https://github.com/status-im/nim-codex.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Update contract (#734) 

* Update codex-contracts-eth

* contracts: update G2Point definition

* integration: disable automatic advancing of time

reason: makes reasoning about timing in tests harder,
because the period is set to 60 seconds in the
marketplace configuration, but this code switches to
a new period every 500 milliseconds

* integration: fix parameters of marketplace payouts test

* integration: update test settings

* integration: fix typo

* integration: workaround for hardhat issue

Subscriptions expire after 5 minutes when using
websockets. Use http and polling instead.

* integration: remove origDatasetSizeInBlocks

* integration: fix proof parameters for test

* integration: do not log output by default

* integration: fix failure rate in test

* integration: fix warning

* integration: include clock in logs

* integration: allow for more periods

5 periods was cutting it close, if we get too much
pointer downtime, then the test would fail
This commit is contained in:
markspanbroek 2024-03-12 09:18:25 +01:00 committed by GitHub
parent e654e93c71
commit 8589e63d34
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 103 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
codex/contracts/proofs.nim
View File

@ -10,13 +10,20 @@ type
G1Point* = object G1Point* = object
x*: UInt256 x*: UInt256
y*: UInt256 y*: UInt256
# A field element F_{p^2} encoded as `real + i * imag`
Fp2Element* = object
real*: UInt256
imag*: UInt256
G2Point* = object G2Point* = object
x*: array[2, UInt256] x*: Fp2Element
y*: array[2, UInt256] y*: Fp2Element
func solidityType*(_: type G1Point): string = func solidityType*(_: type G1Point): string =
solidityType(G1Point.fieldTypes) solidityType(G1Point.fieldTypes)
func solidityType*(_: type Fp2Element): string =
solidityType(Fp2Element.fieldTypes)
func solidityType*(_: type G2Point): string = func solidityType*(_: type G2Point): string =
solidityType(G2Point.fieldTypes) solidityType(G2Point.fieldTypes)
@ -26,6 +33,9 @@ func solidityType*(_: type Groth16Proof): string =
func encode*(encoder: var AbiEncoder, point: G1Point) = func encode*(encoder: var AbiEncoder, point: G1Point) =
encoder.write(point.fieldValues) encoder.write(point.fieldValues)
func encode*(encoder: var AbiEncoder, element: Fp2Element) =
encoder.write(element.fieldValues)
func encode*(encoder: var AbiEncoder, point: G2Point) = func encode*(encoder: var AbiEncoder, point: G2Point) =
encoder.write(point.fieldValues) encoder.write(point.fieldValues)

16
codex/slots/proofs/backends/converters.nim
View File

@ -54,14 +54,14 @@ func toG1*(g: CircomG1): G1Point =
func toG2*(g: CircomG2): G2Point = func toG2*(g: CircomG2): G2Point =
G2Point( G2Point(
x: [ x: Fp2Element(
UInt256.fromBytesLE(g.x[0]), real: UInt256.fromBytesLE(g.x[0]),
UInt256.fromBytesLE(g.x[1]) imag: UInt256.fromBytesLE(g.x[1])
], ),
y: [ y: Fp2Element(
UInt256.fromBytesLE(g.y[0]), real: UInt256.fromBytesLE(g.y[0]),
UInt256.fromBytesLE(g.y[1]) imag: UInt256.fromBytesLE(g.y[1])
]) ))
func toGroth16Proof*(proof: CircomProof): Groth16Proof = func toGroth16Proof*(proof: CircomProof): Groth16Proof =
Groth16Proof( Groth16Proof(

4
tests/examples.nim
View File

@ -78,8 +78,8 @@ proc example(_: type G1Point): G1Point =
proc example(_: type G2Point): G2Point = proc example(_: type G2Point): G2Point =
G2Point( G2Point(
x: [UInt256.example, UInt256.example], x: Fp2Element(real: UInt256.example, imag: UInt256.example),
y: [UInt256.example, UInt256.example] y: Fp2Element(real: UInt256.example, imag: UInt256.example)
) )
proc example*(_: type Groth16Proof): Groth16Proof = proc example*(_: type Groth16Proof): Groth16Proof =

24
tests/integration/marketplacesuite.nim
View File

@ -19,7 +19,6 @@ template marketplacesuite*(name: string, body: untyped) =
var period: uint64 var period: uint64
var periodicity: Periodicity var periodicity: Periodicity
var token {.inject, used.}: Erc20Token var token {.inject, used.}: Erc20Token
var continuousMineFut: Future[void]
proc getCurrentPeriod(): Future[Period] {.async.} = proc getCurrentPeriod(): Future[Period] {.async.} =
return periodicity.periodOf(await ethProvider.currentTime()) return periodicity.periodOf(await ethProvider.currentTime())
@ -60,12 +59,6 @@ template marketplacesuite*(name: string, body: untyped) =
maxCollateral=200.u256 maxCollateral=200.u256
) )
proc validateRequest(nodes, tolerance, origDatasetSizeInBlocks: uint) =
if nodes > 1:
doAssert(origDatasetSizeInBlocks >= 3,
"dataset size must be greater than or equal to 3 blocks with " &
"more than one node")
proc requestStorage(client: CodexClient, proc requestStorage(client: CodexClient,
cid: Cid, cid: Cid,
proofProbability: uint64 = 1, proofProbability: uint64 = 1,
@ -74,9 +67,7 @@ template marketplacesuite*(name: string, body: untyped) =
collateral = 100.u256, collateral = 100.u256,
expiry: uint64 = 4.periods, expiry: uint64 = 4.periods,
nodes = providers().len, nodes = providers().len,
tolerance = 0, tolerance = 0): Future[PurchaseId] {.async.} =
origDatasetSizeInBlocks: int): Future[PurchaseId] {.async.} =
let expiry = (await ethProvider.currentTime()) + expiry.u256 let expiry = (await ethProvider.currentTime()) + expiry.u256
let id = client.requestStorage( let id = client.requestStorage(
@ -92,14 +83,6 @@ template marketplacesuite*(name: string, body: untyped) =
return id return id
proc continuouslyAdvanceEvery(every: chronos.Duration) {.async.} =
try:
while true:
await advanceToNextPeriod()
await sleepAsync(every)
except CancelledError:
discard
setup: setup:
# TODO: This is currently the address of the marketplace with a dummy # TODO: This is currently the address of the marketplace with a dummy
# verifier. Use real marketplace address, `Marketplace.address` once we # verifier. Use real marketplace address, `Marketplace.address` once we
@ -112,9 +95,4 @@ template marketplacesuite*(name: string, body: untyped) =
period = config.proofs.period.truncate(uint64) period = config.proofs.period.truncate(uint64)
periodicity = Periodicity(seconds: period.u256) periodicity = Periodicity(seconds: period.u256)
continuousMineFut = continuouslyAdvanceEvery(chronos.millis(500))
teardown:
await continuousMineFut.cancelAndWait()
body body

5
tests/integration/multinodes.nim
View File

@ -221,7 +221,10 @@ template multinodesuite*(name: string, body: untyped) =
running.add RunningNode(role: Role.Hardhat, node: node) running.add RunningNode(role: Role.Hardhat, node: node)
try: try:
ethProvider = JsonRpcProvider.new("ws://localhost:8545") # Workaround for https://github.com/NomicFoundation/hardhat/issues/2053
# Do not use websockets, but use http and polling to stop subscriptions
# from being removed after 5 minutes
ethProvider = JsonRpcProvider.new("http://localhost:8545")
# if hardhat was NOT started by the test, take a snapshot so it can be # if hardhat was NOT started by the test, take a snapshot so it can be
# reverted in the test teardown # reverted in the test teardown
if nodeConfigs.hardhat.isNil: if nodeConfigs.hardhat.isNil:

34
tests/integration/testmarketplace.nim
View File

@ -13,38 +13,35 @@ marketplacesuite "Marketplace payouts":
clients: clients:
CodexConfig() CodexConfig()
.nodes(1) .nodes(1),
# .debug() # uncomment to enable console log output.debug() # .debug() # uncomment to enable console log output.debug()
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log # .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("node", "erasure"), # .withLogTopics("node", "erasure"),
providers: providers:
CodexConfig() CodexConfig()
.nodes(1) .nodes(1)
# .debug() # uncomment to enable console log output # .debug() # uncomment to enable console log output
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log # .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("node", "marketplace", "sales", "reservations", "node", "proving", "clock"), # .withLogTopics("node", "marketplace", "sales", "reservations", "node", "proving", "clock"),
): ):
let reward = 400.u256 let reward = 400.u256
let duration = 100.periods let duration = 10.periods
let collateral = 200.u256 let collateral = 200.u256
let expiry = 10.periods let expiry = 5.periods
let datasetSizeInBlocks = 3 let data = await RandomChunker.example(blocks=8)
let data = await RandomChunker.example(blocks=datasetSizeInBlocks)
let client = clients()[0] let client = clients()[0]
let provider = providers()[0] let provider = providers()[0]
let clientApi = client.client let clientApi = client.client
let providerApi = provider.client let providerApi = provider.client
let startBalanceProvider = await token.balanceOf(provider.ethAccount) let startBalanceProvider = await token.balanceOf(provider.ethAccount)
let startBalanceClient = await token.balanceOf(client.ethAccount) let startBalanceClient = await token.balanceOf(client.ethAccount)
# original data = 3 blocks so slot size will be 4 blocks
let slotSize = (DefaultBlockSize * 4.NBytes).Natural.u256
# provider makes storage available # provider makes storage available
discard providerApi.postAvailability( discard providerApi.postAvailability(
# make availability size large enough to only fill 1 slot, thus causing a # make availability size small enough that we can't fill all the slots,
# cancellation # thus causing a cancellation
size=slotSize, size=(data.len div 2).u256,
duration=duration.u256, duration=duration.u256,
minPrice=reward, minPrice=reward,
maxCollateral=collateral) maxCollateral=collateral)
@ -57,7 +54,7 @@ marketplacesuite "Marketplace payouts":
let subscription = await marketplace.subscribe(SlotFilled, onSlotFilled) let subscription = await marketplace.subscribe(SlotFilled, onSlotFilled)
# client requests storage but requires two nodes to host the content # client requests storage but requires multiple slots to host the content
let id = await clientApi.requestStorage( let id = await clientApi.requestStorage(
cid, cid,
duration=duration, duration=duration,
@ -65,18 +62,17 @@ marketplacesuite "Marketplace payouts":
expiry=expiry, expiry=expiry,
collateral=collateral, collateral=collateral,
nodes=3, nodes=3,
tolerance=1, tolerance=1
origDatasetSizeInBlocks=datasetSizeInBlocks
) )
# wait until one slot is filled # wait until one slot is filled
check eventually slotIdxFilled.isSome check eventually(slotIdxFilled.isSome, timeout=expiry.int * 1000)
# wait until sale is cancelled # wait until sale is cancelled
without requestId =? clientApi.requestId(id): without requestId =? clientApi.requestId(id):
fail() fail()
let slotId = slotId(requestId, !slotIdxFilled) let slotId = slotId(requestId, !slotIdxFilled)
check eventually(providerApi.saleStateIs(slotId, "SaleCancelled")) check eventually(providerApi.saleStateIs(slotId, "SaleCancelled"), timeout=expiry.int * 1000)
check eventually ( check eventually (
let endBalanceProvider = (await token.balanceOf(provider.ethAccount)); let endBalanceProvider = (await token.balanceOf(provider.ethAccount));

114
tests/integration/testproofs.nim
View File

@ -22,32 +22,35 @@ marketplacesuite "Hosts submit regular proofs":
clients: clients:
CodexConfig() CodexConfig()
.nodes(1) .nodes(1),
# .debug() # uncomment to enable console log output # .debug() # uncomment to enable console log output
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log # .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("node"), # .withLogTopics("node"),
providers: providers:
CodexConfig() CodexConfig()
.nodes(1) .nodes(1)
# .debug() # uncomment to enable console log output # .debug() # uncomment to enable console log output
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log # .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("marketplace", "sales", "reservations", "node"), # .withLogTopics("marketplace", "sales", "reservations", "node"),
): ):
let client0 = clients()[0].client let client0 = clients()[0].client
let totalPeriods = 50 let expiry = 5.periods
let datasetSizeInBlocks = 2 let duration = expiry + 5.periods
let data = await RandomChunker.example(blocks=1) let data = await RandomChunker.example(blocks=8)
createAvailabilities(data.len, totalPeriods.periods) createAvailabilities(data.len * 2, duration) # TODO: better value for data.len
let cid = client0.upload(data).get let cid = client0.upload(data).get
let purchaseId = await client0.requestStorage( let purchaseId = await client0.requestStorage(
cid, cid,
duration=totalPeriods.periods, expiry=expiry,
origDatasetSizeInBlocks = datasetSizeInBlocks) duration=duration,
check eventually client0.purchaseStateIs(purchaseId, "started") nodes=3,
tolerance=1
)
check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000)
var proofWasSubmitted = false var proofWasSubmitted = false
proc onProofSubmitted(event: ProofSubmitted) = proc onProofSubmitted(event: ProofSubmitted) =
@ -55,8 +58,7 @@ marketplacesuite "Hosts submit regular proofs":
let subscription = await marketplace.subscribe(ProofSubmitted, onProofSubmitted) let subscription = await marketplace.subscribe(ProofSubmitted, onProofSubmitted)
let currentPeriod = await getCurrentPeriod() check eventually(proofWasSubmitted, timeout=(duration - expiry).int * 1000)
check eventuallyP(proofWasSubmitted, currentPeriod + totalPeriods.u256 + 1)
await subscription.unsubscribe() await subscription.unsubscribe()
@ -74,54 +76,55 @@ marketplacesuite "Simulate invalid proofs":
clients: clients:
CodexConfig() CodexConfig()
.nodes(1) .nodes(1),
# .debug() # uncomment to enable console log output # .debug() # uncomment to enable console log output
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log # .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("node"), # .withLogTopics("node", "clock"),
providers: providers:
CodexConfig() CodexConfig()
.nodes(1) .nodes(1)
.simulateProofFailuresFor(providerIdx=0, failEveryNProofs=1) .simulateProofFailuresFor(providerIdx=0, failEveryNProofs=1),
# .debug() # uncomment to enable console log output # .debug() # uncomment to enable console log output
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log # .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("marketplace", "sales", "reservations", "node"), # .withLogTopics("marketplace", "sales", "reservations", "node", "clock"),
validators: validators:
CodexConfig() CodexConfig()
.nodes(1) .nodes(1)
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
# .debug() # uncomment to enable console log output # .debug() # uncomment to enable console log output
.withLogTopics("validator", "onchain", "ethers") # .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
# .withLogTopics("validator", "onchain", "ethers", "clock")
): ):
let client0 = clients()[0].client let client0 = clients()[0].client
let totalPeriods = 50 let expiry = 5.periods
let duration = expiry + 10.periods
let datasetSizeInBlocks = 2 let data = await RandomChunker.example(blocks=8)
let data = await RandomChunker.example(blocks=datasetSizeInBlocks) createAvailabilities(data.len * 2, duration) # TODO: better value for data.len
createAvailabilities(data.len, totalPeriods.periods)
let cid = client0.upload(data).get let cid = client0.upload(data).get
let purchaseId = await client0.requestStorage( let purchaseId = await client0.requestStorage(
cid, cid,
expiry=10.periods, expiry=expiry,
duration=totalPeriods.periods, duration=duration,
origDatasetSizeInBlocks=datasetSizeInBlocks) nodes=3,
tolerance=1,
proofProbability=1
)
let requestId = client0.requestId(purchaseId).get let requestId = client0.requestId(purchaseId).get
check eventually client0.purchaseStateIs(purchaseId, "started") check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000)
var slotWasFreed = false var slotWasFreed = false
proc onSlotFreed(event: SlotFreed) = proc onSlotFreed(event: SlotFreed) =
if event.requestId == requestId and if event.requestId == requestId:
event.slotIndex == 0.u256: # assume only one slot, so index 0
slotWasFreed = true slotWasFreed = true
let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed)
let currentPeriod = await getCurrentPeriod() check eventually(slotWasFreed, timeout=(duration - expiry).int * 1000)
check eventuallyP(slotWasFreed, currentPeriod + totalPeriods.u256 + 1)
await subscription.unsubscribe() await subscription.unsubscribe()
@ -131,55 +134,58 @@ marketplacesuite "Simulate invalid proofs":
clients: clients:
CodexConfig() CodexConfig()
.nodes(1) .nodes(1),
# .debug() # uncomment to enable console log output # .debug() # uncomment to enable console log output
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log # .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("node"), # .withLogTopics("marketplace", "sales", "reservations", "node", "clock"),
providers: providers:
CodexConfig() CodexConfig()
.nodes(1) .nodes(1)
.simulateProofFailuresFor(providerIdx=0, failEveryNProofs=3) .simulateProofFailuresFor(providerIdx=0, failEveryNProofs=1),
# .debug() # uncomment to enable console log output # .debug() # uncomment to enable console log output
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log # .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("marketplace", "sales", "reservations", "node"), # .withLogTopics("marketplace", "sales", "reservations", "node"),
validators: validators:
CodexConfig() CodexConfig()
.nodes(1) .nodes(1)
# .debug() # .debug()
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log # .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("validator", "onchain", "ethers") # .withLogTopics("validator", "onchain", "ethers", "clock")
): ):
let client0 = clients()[0].client let client0 = clients()[0].client
let totalPeriods = 25 let expiry = 5.periods
# In 2 periods you cannot have enough invalid proofs submitted:
let duration = expiry + 2.periods
let datasetSizeInBlocks = 2 let data = await RandomChunker.example(blocks=8)
let data = await RandomChunker.example(blocks=datasetSizeInBlocks) createAvailabilities(data.len * 2, duration) # TODO: better value for data.len
createAvailabilities(data.len, totalPeriods.periods)
let cid = client0.upload(data).get let cid = client0.upload(data).get
let purchaseId = await client0.requestStorage( let purchaseId = await client0.requestStorage(
cid, cid,
expiry=10.periods, expiry=expiry,
duration=totalPeriods.periods, duration=duration,
origDatasetSizeInBlocks=datasetSizeInBlocks) nodes=3,
tolerance=1,
proofProbability=1
)
let requestId = client0.requestId(purchaseId).get let requestId = client0.requestId(purchaseId).get
check eventually client0.purchaseStateIs(purchaseId, "started") check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000)
var slotWasFreed = false var slotWasFreed = false
proc onSlotFreed(event: SlotFreed) = proc onSlotFreed(event: SlotFreed) =
if event.requestId == requestId and if event.requestId == requestId:
event.slotIndex == 0.u256:
slotWasFreed = true slotWasFreed = true
let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed)
# check not freed # check not freed
let currentPeriod = await getCurrentPeriod() await sleepAsync((duration - expiry).int.seconds)
check not eventuallyP(slotWasFreed, currentPeriod + totalPeriods.u256 + 1) check not slotWasFreed
await subscription.unsubscribe() await subscription.unsubscribe()

2
vendor/codex-contracts-eth vendored

@ -1 +1 @@
Subproject commit 6c9f797f408608958714024b9055fcc330e3842f Subproject commit 118ee0b22b2d12c8fbf3376cf201d203d0a7cf97