codex-storage
/
nim-codex
mirror of https://github.com/codex-storage/nim-codex.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update contract (#734) 

* Update codex-contracts-eth

* contracts: update G2Point definition

* integration: disable automatic advancing of time

reason: makes reasoning about timing in tests harder,
because the period is set to 60 seconds in the
marketplace configuration, but this code switches to
a new period every 500 milliseconds

* integration: fix parameters of marketplace payouts test

* integration: update test settings

* integration: fix typo

* integration: workaround for hardhat issue

Subscriptions expire after 5 minutes when using
websockets. Use http and polling instead.

* integration: remove origDatasetSizeInBlocks

* integration: fix proof parameters for test

* integration: do not log output by default

* integration: fix failure rate in test

* integration: fix warning

* integration: include clock in logs

* integration: allow for more periods

5 periods was cutting it close, if we get too much
pointer downtime, then the test would fail
This commit is contained in:
markspanbroek 2024-03-12 09:18:25 +01:00 committed by GitHub
parent e654e93c71
commit 8589e63d34
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 103 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
codex/contracts/proofs.nim
View File

@ -10,13 +10,20 @@ type
G1Point* = object
x*: UInt256
y*: UInt256
# A field element F_{p^2} encoded as `real + i * imag`
Fp2Element* = object
real*: UInt256
imag*: UInt256
G2Point* = object
x*: array[2, UInt256]
y*: array[2, UInt256]
x*: Fp2Element
y*: Fp2Element
func solidityType*(_: type G1Point): string =
solidityType(G1Point.fieldTypes)
func solidityType*(_: type Fp2Element): string =
solidityType(Fp2Element.fieldTypes)
func solidityType*(_: type G2Point): string =
solidityType(G2Point.fieldTypes)
@ -26,6 +33,9 @@ func solidityType*(_: type Groth16Proof): string =
func encode*(encoder: var AbiEncoder, point: G1Point) =
encoder.write(point.fieldValues)
func encode*(encoder: var AbiEncoder, element: Fp2Element) =
encoder.write(element.fieldValues)
func encode*(encoder: var AbiEncoder, point: G2Point) =
encoder.write(point.fieldValues)

16
codex/slots/proofs/backends/converters.nim
View File

@ -54,14 +54,14 @@ func toG1*(g: CircomG1): G1Point =
func toG2*(g: CircomG2): G2Point =
G2Point(
x: [
UInt256.fromBytesLE(g.x[0]),
UInt256.fromBytesLE(g.x[1])
],
y: [
UInt256.fromBytesLE(g.y[0]),
UInt256.fromBytesLE(g.y[1])
])
x: Fp2Element(
real: UInt256.fromBytesLE(g.x[0]),
imag: UInt256.fromBytesLE(g.x[1])
),
y: Fp2Element(
real: UInt256.fromBytesLE(g.y[0]),
imag: UInt256.fromBytesLE(g.y[1])
))
func toGroth16Proof*(proof: CircomProof): Groth16Proof =
Groth16Proof(

4
tests/examples.nim
View File

@ -78,8 +78,8 @@ proc example(_: type G1Point): G1Point =
proc example(_: type G2Point): G2Point =
G2Point(
x: [UInt256.example, UInt256.example],
y: [UInt256.example, UInt256.example]
x: Fp2Element(real: UInt256.example, imag: UInt256.example),
y: Fp2Element(real: UInt256.example, imag: UInt256.example)
)
proc example*(_: type Groth16Proof): Groth16Proof =

24
tests/integration/marketplacesuite.nim
View File

@ -19,7 +19,6 @@ template marketplacesuite*(name: string, body: untyped) =
var period: uint64
var periodicity: Periodicity
var token {.inject, used.}: Erc20Token
var continuousMineFut: Future[void]
proc getCurrentPeriod(): Future[Period] {.async.} =
return periodicity.periodOf(await ethProvider.currentTime())
@ -60,12 +59,6 @@ template marketplacesuite*(name: string, body: untyped) =
maxCollateral=200.u256
)
proc validateRequest(nodes, tolerance, origDatasetSizeInBlocks: uint) =
if nodes > 1:
doAssert(origDatasetSizeInBlocks >= 3,
"dataset size must be greater than or equal to 3 blocks with " &
"more than one node")
proc requestStorage(client: CodexClient,
cid: Cid,
proofProbability: uint64 = 1,
@ -74,9 +67,7 @@ template marketplacesuite*(name: string, body: untyped) =
collateral = 100.u256,
expiry: uint64 = 4.periods,
nodes = providers().len,
tolerance = 0,
origDatasetSizeInBlocks: int): Future[PurchaseId] {.async.} =
tolerance = 0): Future[PurchaseId] {.async.} =
let expiry = (await ethProvider.currentTime()) + expiry.u256
let id = client.requestStorage(
@ -92,14 +83,6 @@ template marketplacesuite*(name: string, body: untyped) =
return id
proc continuouslyAdvanceEvery(every: chronos.Duration) {.async.} =
try:
while true:
await advanceToNextPeriod()
await sleepAsync(every)
except CancelledError:
discard
setup:
# TODO: This is currently the address of the marketplace with a dummy
# verifier. Use real marketplace address, `Marketplace.address` once we
@ -112,9 +95,4 @@ template marketplacesuite*(name: string, body: untyped) =
period = config.proofs.period.truncate(uint64)
periodicity = Periodicity(seconds: period.u256)
continuousMineFut = continuouslyAdvanceEvery(chronos.millis(500))
teardown:
await continuousMineFut.cancelAndWait()
body

5
tests/integration/multinodes.nim
View File

@ -221,7 +221,10 @@ template multinodesuite*(name: string, body: untyped) =
running.add RunningNode(role: Role.Hardhat, node: node)
try:
ethProvider = JsonRpcProvider.new("ws://localhost:8545")
# Workaround for https://github.com/NomicFoundation/hardhat/issues/2053
# Do not use websockets, but use http and polling to stop subscriptions
# from being removed after 5 minutes
ethProvider = JsonRpcProvider.new("http://localhost:8545")
# if hardhat was NOT started by the test, take a snapshot so it can be
# reverted in the test teardown
if nodeConfigs.hardhat.isNil:

34
tests/integration/testmarketplace.nim
View File

@ -13,38 +13,35 @@ marketplacesuite "Marketplace payouts":
clients:
CodexConfig()
.nodes(1)
.nodes(1),
# .debug() # uncomment to enable console log output.debug()
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("node", "erasure"),
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
# .withLogTopics("node", "erasure"),
providers:
CodexConfig()
.nodes(1)
# .debug() # uncomment to enable console log output
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("node", "marketplace", "sales", "reservations", "node", "proving", "clock"),
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
# .withLogTopics("node", "marketplace", "sales", "reservations", "node", "proving", "clock"),
):
let reward = 400.u256
let duration = 100.periods
let duration = 10.periods
let collateral = 200.u256
let expiry = 10.periods
let datasetSizeInBlocks = 3
let data = await RandomChunker.example(blocks=datasetSizeInBlocks)
let expiry = 5.periods
let data = await RandomChunker.example(blocks=8)
let client = clients()[0]
let provider = providers()[0]
let clientApi = client.client
let providerApi = provider.client
let startBalanceProvider = await token.balanceOf(provider.ethAccount)
let startBalanceClient = await token.balanceOf(client.ethAccount)
# original data = 3 blocks so slot size will be 4 blocks
let slotSize = (DefaultBlockSize * 4.NBytes).Natural.u256
# provider makes storage available
discard providerApi.postAvailability(
# make availability size large enough to only fill 1 slot, thus causing a
# cancellation
size=slotSize,
# make availability size small enough that we can't fill all the slots,
# thus causing a cancellation
size=(data.len div 2).u256,
duration=duration.u256,
minPrice=reward,
maxCollateral=collateral)
@ -57,7 +54,7 @@ marketplacesuite "Marketplace payouts":
let subscription = await marketplace.subscribe(SlotFilled, onSlotFilled)
# client requests storage but requires two nodes to host the content
# client requests storage but requires multiple slots to host the content
let id = await clientApi.requestStorage(
cid,
duration=duration,
@ -65,18 +62,17 @@ marketplacesuite "Marketplace payouts":
expiry=expiry,
collateral=collateral,
nodes=3,
tolerance=1,
origDatasetSizeInBlocks=datasetSizeInBlocks
tolerance=1
)
# wait until one slot is filled
check eventually slotIdxFilled.isSome
check eventually(slotIdxFilled.isSome, timeout=expiry.int * 1000)
# wait until sale is cancelled
without requestId =? clientApi.requestId(id):
fail()
let slotId = slotId(requestId, !slotIdxFilled)
check eventually(providerApi.saleStateIs(slotId, "SaleCancelled"))
check eventually(providerApi.saleStateIs(slotId, "SaleCancelled"), timeout=expiry.int * 1000)
check eventually (
let endBalanceProvider = (await token.balanceOf(provider.ethAccount));

114
tests/integration/testproofs.nim
View File

@ -22,32 +22,35 @@ marketplacesuite "Hosts submit regular proofs":
clients:
CodexConfig()
.nodes(1)
.nodes(1),
# .debug() # uncomment to enable console log output
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("node"),
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
# .withLogTopics("node"),
providers:
CodexConfig()
.nodes(1)
# .debug() # uncomment to enable console log output
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("marketplace", "sales", "reservations", "node"),
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
# .withLogTopics("marketplace", "sales", "reservations", "node"),
):
let client0 = clients()[0].client
let totalPeriods = 50
let datasetSizeInBlocks = 2
let expiry = 5.periods
let duration = expiry + 5.periods
let data = await RandomChunker.example(blocks=1)
createAvailabilities(data.len, totalPeriods.periods)
let data = await RandomChunker.example(blocks=8)
createAvailabilities(data.len * 2, duration) # TODO: better value for data.len
let cid = client0.upload(data).get
let purchaseId = await client0.requestStorage(
cid,
duration=totalPeriods.periods,
origDatasetSizeInBlocks = datasetSizeInBlocks)
check eventually client0.purchaseStateIs(purchaseId, "started")
expiry=expiry,
duration=duration,
nodes=3,
tolerance=1
)
check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000)
var proofWasSubmitted = false
proc onProofSubmitted(event: ProofSubmitted) =
@ -55,8 +58,7 @@ marketplacesuite "Hosts submit regular proofs":
let subscription = await marketplace.subscribe(ProofSubmitted, onProofSubmitted)
let currentPeriod = await getCurrentPeriod()
check eventuallyP(proofWasSubmitted, currentPeriod + totalPeriods.u256 + 1)
check eventually(proofWasSubmitted, timeout=(duration - expiry).int * 1000)
await subscription.unsubscribe()
@ -74,54 +76,55 @@ marketplacesuite "Simulate invalid proofs":
clients:
CodexConfig()
.nodes(1)
.nodes(1),
# .debug() # uncomment to enable console log output
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("node"),
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
# .withLogTopics("node", "clock"),
providers:
CodexConfig()
.nodes(1)
.simulateProofFailuresFor(providerIdx=0, failEveryNProofs=1)
.simulateProofFailuresFor(providerIdx=0, failEveryNProofs=1),
# .debug() # uncomment to enable console log output
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("marketplace", "sales", "reservations", "node"),
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
# .withLogTopics("marketplace", "sales", "reservations", "node", "clock"),
validators:
CodexConfig()
.nodes(1)
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
# .debug() # uncomment to enable console log output
.withLogTopics("validator", "onchain", "ethers")
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
# .withLogTopics("validator", "onchain", "ethers", "clock")
):
let client0 = clients()[0].client
let totalPeriods = 50
let expiry = 5.periods
let duration = expiry + 10.periods
let datasetSizeInBlocks = 2
let data = await RandomChunker.example(blocks=datasetSizeInBlocks)
createAvailabilities(data.len, totalPeriods.periods)
let data = await RandomChunker.example(blocks=8)
createAvailabilities(data.len * 2, duration) # TODO: better value for data.len
let cid = client0.upload(data).get
let purchaseId = await client0.requestStorage(
cid,
expiry=10.periods,
duration=totalPeriods.periods,
origDatasetSizeInBlocks=datasetSizeInBlocks)
expiry=expiry,
duration=duration,
nodes=3,
tolerance=1,
proofProbability=1
)
let requestId = client0.requestId(purchaseId).get
check eventually client0.purchaseStateIs(purchaseId, "started")
check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000)
var slotWasFreed = false
proc onSlotFreed(event: SlotFreed) =
if event.requestId == requestId and
event.slotIndex == 0.u256: # assume only one slot, so index 0
if event.requestId == requestId:
slotWasFreed = true
let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed)
let currentPeriod = await getCurrentPeriod()
check eventuallyP(slotWasFreed, currentPeriod + totalPeriods.u256 + 1)
check eventually(slotWasFreed, timeout=(duration - expiry).int * 1000)
await subscription.unsubscribe()
@ -131,55 +134,58 @@ marketplacesuite "Simulate invalid proofs":
clients:
CodexConfig()
.nodes(1)
.nodes(1),
# .debug() # uncomment to enable console log output
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("node"),
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
# .withLogTopics("marketplace", "sales", "reservations", "node", "clock"),
providers:
CodexConfig()
.nodes(1)
.simulateProofFailuresFor(providerIdx=0, failEveryNProofs=3)
.simulateProofFailuresFor(providerIdx=0, failEveryNProofs=1),
# .debug() # uncomment to enable console log output
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("marketplace", "sales", "reservations", "node"),
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
# .withLogTopics("marketplace", "sales", "reservations", "node"),
validators:
CodexConfig()
.nodes(1)
# .debug()
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("validator", "onchain", "ethers")
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
# .withLogTopics("validator", "onchain", "ethers", "clock")
):
let client0 = clients()[0].client
let totalPeriods = 25
let expiry = 5.periods
# In 2 periods you cannot have enough invalid proofs submitted:
let duration = expiry + 2.periods
let datasetSizeInBlocks = 2
let data = await RandomChunker.example(blocks=datasetSizeInBlocks)
createAvailabilities(data.len, totalPeriods.periods)
let data = await RandomChunker.example(blocks=8)
createAvailabilities(data.len * 2, duration) # TODO: better value for data.len
let cid = client0.upload(data).get
let purchaseId = await client0.requestStorage(
cid,
expiry=10.periods,
duration=totalPeriods.periods,
origDatasetSizeInBlocks=datasetSizeInBlocks)
expiry=expiry,
duration=duration,
nodes=3,
tolerance=1,
proofProbability=1
)
let requestId = client0.requestId(purchaseId).get
check eventually client0.purchaseStateIs(purchaseId, "started")
check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000)
var slotWasFreed = false
proc onSlotFreed(event: SlotFreed) =
if event.requestId == requestId and
event.slotIndex == 0.u256:
if event.requestId == requestId:
slotWasFreed = true
let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed)
# check not freed
let currentPeriod = await getCurrentPeriod()
check not eventuallyP(slotWasFreed, currentPeriod + totalPeriods.u256 + 1)
await sleepAsync((duration - expiry).int.seconds)
check not slotWasFreed
await subscription.unsubscribe()

2
vendor/codex-contracts-eth vendored

@ -1 +1 @@
Subproject commit 6c9f797f408608958714024b9055fcc330e3842f
Subproject commit 118ee0b22b2d12c8fbf3376cf201d203d0a7cf97