codex-storage
/
nim-codex
mirror of https://github.com/codex-storage/nim-codex.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Generate proofs when required (#383) 

* [maintenance] speedup integration test

* [rest api] add proofProbability parameter to storage requests

* [integration] negotiation test ends when contract starts

* [integration] reusable 2 node setup for tests

* [integration] introduce CodexClient for tests

* [node] submit storage proofs when required

* [contracts] Add Slot type

* [proving] replace onProofRequired & submitProof with onProve

Removes duplication between Sales.onProve() and
Proving.onProofRequired()
This commit is contained in:
markspanbroek 2023-03-27 15:47:25 +02:00 committed by GitHub
parent 067c1c9625
commit 4ffe7b8e06
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 360 additions and 236 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
codex/contracts/requests.nim
View File

@ -30,6 +30,9 @@ type
u*: seq[byte] u*: seq[byte]
publicKey*: seq[byte] publicKey*: seq[byte]
name*: seq[byte] name*: seq[byte]
Slot* = object
request*: StorageRequest
slotIndex*: UInt256
SlotId* = distinct array[32, byte] SlotId* = distinct array[32, byte]
RequestId* = distinct array[32, byte] RequestId* = distinct array[32, byte]
Nonce* = distinct array[32, byte] Nonce* = distinct array[32, byte]
@ -50,6 +53,8 @@ proc `==`*(x, y: Nonce): bool {.borrow.}
proc `==`*(x, y: RequestId): bool {.borrow.} proc `==`*(x, y: RequestId): bool {.borrow.}
proc `==`*(x, y: SlotId): bool {.borrow.} proc `==`*(x, y: SlotId): bool {.borrow.}
proc hash*(x: SlotId): Hash {.borrow.} proc hash*(x: SlotId): Hash {.borrow.}
proc hash*(x: Nonce): Hash {.borrow.}
proc hash*(x: Address): Hash {.borrow.}
func toArray*(id: RequestId | SlotId | Nonce): array[32, byte] = func toArray*(id: RequestId | SlotId | Nonce): array[32, byte] =
array[32, byte](id) array[32, byte](id)
@ -159,6 +164,9 @@ func slotId*(requestId: RequestId, slot: UInt256): SlotId =
func slotId*(request: StorageRequest, slot: UInt256): SlotId = func slotId*(request: StorageRequest, slot: UInt256): SlotId =
slotId(request.id, slot) slotId(request.id, slot)
func id*(slot: Slot): SlotId =
slotId(slot.request, slot.slotIndex)
func pricePerSlot*(ask: StorageAsk): UInt256 = func pricePerSlot*(ask: StorageAsk): UInt256 =
ask.duration * ask.reward ask.duration * ask.reward

5
codex/node.nim
View File

@ -235,6 +235,7 @@ proc store*(
proc requestStorage*(self: CodexNodeRef, proc requestStorage*(self: CodexNodeRef,
cid: Cid, cid: Cid,
duration: UInt256, duration: UInt256,
proofProbability: UInt256,
nodes: uint, nodes: uint,
tolerance: uint, tolerance: uint,
reward: UInt256, reward: UInt256,
@ -280,6 +281,7 @@ proc requestStorage*(self: CodexNodeRef,
slots: nodes + tolerance, slots: nodes + tolerance,
slotSize: (encoded.blockSize * encoded.steps).u256, slotSize: (encoded.blockSize * encoded.steps).u256,
duration: duration, duration: duration,
proofProbability: proofProbability,
reward: reward, reward: reward,
maxSlotLoss: tolerance maxSlotLoss: tolerance
), ),
@ -360,8 +362,7 @@ proc start*(node: CodexNodeRef) {.async.} =
# TODO: remove data from local storage # TODO: remove data from local storage
discard discard
contracts.sales.onProve = proc(request: StorageRequest, contracts.proving.onProve = proc(slot: Slot): Future[seq[byte]] {.async.} =
slot: UInt256): Future[seq[byte]] {.async.} =
# TODO: generate proof # TODO: generate proof
return @[42'u8] return @[42'u8]

45
codex/proving.nim
View File

@ -13,18 +13,21 @@ type
proofs: Proofs proofs: Proofs
clock: Clock clock: Clock
loop: ?Future[void] loop: ?Future[void]
slots*: HashSet[SlotId] slots*: HashSet[Slot]
onProofRequired: ?OnProofRequired onProve: ?OnProve
OnProofRequired* = proc (id: SlotId) {.gcsafe, upraises:[].} OnProve* = proc(slot: Slot): Future[seq[byte]] {.gcsafe, upraises: [].}
func new*(_: type Proving, proofs: Proofs, clock: Clock): Proving = func new*(_: type Proving, proofs: Proofs, clock: Clock): Proving =
Proving(proofs: proofs, clock: clock) Proving(proofs: proofs, clock: clock)
proc `onProofRequired=`*(proving: Proving, callback: OnProofRequired) = proc onProve*(proving: Proving): ?OnProve =
proving.onProofRequired = some callback proving.onProve
func add*(proving: Proving, id: SlotId) = proc `onProve=`*(proving: Proving, callback: OnProve) =
proving.slots.incl(id) proving.onProve = some callback
func add*(proving: Proving, slot: Slot) =
proving.slots.incl(slot)
proc getCurrentPeriod(proving: Proving): Future[Period] {.async.} = proc getCurrentPeriod(proving: Proving): Future[Period] {.async.} =
let periodicity = await proving.proofs.periodicity() let periodicity = await proving.proofs.periodicity()
@ -35,23 +38,32 @@ proc waitUntilPeriod(proving: Proving, period: Period) {.async.} =
await proving.clock.waitUntil(periodicity.periodStart(period).truncate(int64)) await proving.clock.waitUntil(periodicity.periodStart(period).truncate(int64))
proc removeEndedContracts(proving: Proving) {.async.} = proc removeEndedContracts(proving: Proving) {.async.} =
var ended: HashSet[SlotId] var ended: HashSet[Slot]
for id in proving.slots: for slot in proving.slots:
let state = await proving.proofs.slotState(id) let state = await proving.proofs.slotState(slot.id)
if state != SlotState.Filled: if state != SlotState.Filled:
ended.incl(id) ended.incl(slot)
proving.slots.excl(ended) proving.slots.excl(ended)
proc prove(proving: Proving, slot: Slot) {.async.} =
without onProve =? proving.onProve:
raiseAssert "onProve callback not set"
try:
let proof = await onProve(slot)
await proving.proofs.submitProof(slot.id, proof)
except CatchableError as e:
error "Submitting proof failed", msg = e.msg
proc run(proving: Proving) {.async.} = proc run(proving: Proving) {.async.} =
try: try:
while true: while true:
let currentPeriod = await proving.getCurrentPeriod() let currentPeriod = await proving.getCurrentPeriod()
await proving.removeEndedContracts() await proving.removeEndedContracts()
for id in proving.slots: for slot in proving.slots:
let id = slot.id
if (await proving.proofs.isProofRequired(id)) or if (await proving.proofs.isProofRequired(id)) or
(await proving.proofs.willProofBeRequired(id)): (await proving.proofs.willProofBeRequired(id)):
if callback =? proving.onProofRequired: asyncSpawn proving.prove(slot)
callback(id)
await proving.waitUntilPeriod(currentPeriod + 1) await proving.waitUntilPeriod(currentPeriod + 1)
except CancelledError: except CancelledError:
discard discard
@ -70,9 +82,6 @@ proc stop*(proving: Proving) {.async.} =
if not loop.finished: if not loop.finished:
await loop.cancelAndWait() await loop.cancelAndWait()
proc submitProof*(proving: Proving, id: SlotId, proof: seq[byte]) {.async.} =
await proving.proofs.submitProof(id, proof)
proc subscribeProofSubmission*(proving: Proving, proc subscribeProofSubmission*(proving: Proving,
callback: OnProofSubmitted): callback: OnProofSubmitted):
Future[Subscription] = Future[Subscription] =

14
codex/rest/api.nim
View File

@ -133,12 +133,13 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
"/api/codex/v1/storage/request/{cid}") do (cid: Cid) -> RestApiResponse: "/api/codex/v1/storage/request/{cid}") do (cid: Cid) -> RestApiResponse:
## Create a request for storage ## Create a request for storage
## ##
## cid - the cid of a previously uploaded dataset ## cid - the cid of a previously uploaded dataset
## duration - the duration of the request in seconds ## duration - the duration of the request in seconds
## reward - the maximum amount of tokens paid per second per slot to hosts the client is willing to pay ## proofProbability - how often storage proofs are required
## expiry - timestamp, in seconds, when the request expires if the Request does not find requested amount of nodes to host the data ## reward - the maximum amount of tokens paid per second per slot to hosts the client is willing to pay
## nodes - minimal number of nodes the content should be stored on ## expiry - timestamp, in seconds, when the request expires if the Request does not find requested amount of nodes to host the data
## tolerance - allowed number of nodes that can be lost before pronouncing the content lost ## nodes - minimal number of nodes the content should be stored on
## tolerance - allowed number of nodes that can be lost before pronouncing the content lost
without cid =? cid.tryGet.catch, error: without cid =? cid.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg) return RestApiResponse.error(Http400, error.msg)
@ -154,6 +155,7 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
without purchaseId =? await node.requestStorage( without purchaseId =? await node.requestStorage(
cid, cid,
params.duration, params.duration,
params.proofProbability,
nodes, nodes,
tolerance, tolerance,
params.reward, params.reward,

3
codex/rest/json.nim
View File

@ -8,6 +8,7 @@ import ../purchasing
type type
StorageRequestParams* = object StorageRequestParams* = object
duration*: UInt256 duration*: UInt256
proofProbability*: UInt256
reward*: UInt256 reward*: UInt256
expiry*: ?UInt256 expiry*: ?UInt256
nodes*: ?uint nodes*: ?uint
@ -24,12 +25,14 @@ proc fromJson*(_: type StorageRequestParams,
bytes: seq[byte]): ?! StorageRequestParams = bytes: seq[byte]): ?! StorageRequestParams =
let json = ?catch parseJson(string.fromBytes(bytes)) let json = ?catch parseJson(string.fromBytes(bytes))
let duration = ?catch UInt256.fromHex(json["duration"].getStr) let duration = ?catch UInt256.fromHex(json["duration"].getStr)
let proofProbability = ?catch UInt256.fromHex(json["proofProbability"].getStr)
let reward = ?catch UInt256.fromHex(json["reward"].getStr) let reward = ?catch UInt256.fromHex(json["reward"].getStr)
let expiry = UInt256.fromHex(json["expiry"].getStr).catch.option let expiry = UInt256.fromHex(json["expiry"].getStr).catch.option
let nodes = strutils.fromHex[uint](json["nodes"].getStr).catch.option let nodes = strutils.fromHex[uint](json["nodes"].getStr).catch.option
let tolerance = strutils.fromHex[uint](json["tolerance"].getStr).catch.option let tolerance = strutils.fromHex[uint](json["tolerance"].getStr).catch.option
success StorageRequestParams( success StorageRequestParams(
duration: duration, duration: duration,
proofProbability: proofProbability,
reward: reward, reward: reward,
expiry: expiry, expiry: expiry,
nodes: nodes, nodes: nodes,

5
codex/sales.nim
View File

@ -47,9 +47,6 @@ type
proc `onStore=`*(sales: Sales, onStore: OnStore) = proc `onStore=`*(sales: Sales, onStore: OnStore) =
sales.context.onStore = some onStore sales.context.onStore = some onStore
proc `onProve=`*(sales: Sales, onProve: OnProve) =
sales.context.onProve = some onProve
proc `onClear=`*(sales: Sales, onClear: OnClear) = proc `onClear=`*(sales: Sales, onClear: OnClear) =
sales.context.onClear = some onClear sales.context.onClear = some onClear
@ -58,8 +55,6 @@ proc `onSale=`*(sales: Sales, callback: OnSale) =
proc onStore*(sales: Sales): ?OnStore = sales.context.onStore proc onStore*(sales: Sales): ?OnStore = sales.context.onStore
proc onProve*(sales: Sales): ?OnProve = sales.context.onProve
proc onClear*(sales: Sales): ?OnClear = sales.context.onClear proc onClear*(sales: Sales): ?OnClear = sales.context.onClear
proc onSale*(sales: Sales): ?OnSale = sales.context.onSale proc onSale*(sales: Sales): ?OnSale = sales.context.onSale

3
codex/sales/salescontext.nim
View File

@ -9,7 +9,6 @@ type
market*: Market market*: Market
clock*: Clock clock*: Clock
onStore*: ?OnStore onStore*: ?OnStore
onProve*: ?OnProve
onClear*: ?OnClear onClear*: ?OnClear
onSale*: ?OnSale onSale*: ?OnSale
onSaleErrored*: ?OnSaleErrored onSaleErrored*: ?OnSaleErrored
@ -17,8 +16,6 @@ type
OnStore* = proc(request: StorageRequest, OnStore* = proc(request: StorageRequest,
slot: UInt256, slot: UInt256,
availability: ?Availability): Future[void] {.gcsafe, upraises: [].} availability: ?Availability): Future[void] {.gcsafe, upraises: [].}
OnProve* = proc(request: StorageRequest,
slot: UInt256): Future[seq[byte]] {.gcsafe, upraises: [].}
OnClear* = proc(availability: ?Availability,# TODO: when availability changes introduced, make availability non-optional (if we need to keep it at all) OnClear* = proc(availability: ?Availability,# TODO: when availability changes introduced, make availability non-optional (if we need to keep it at all)
request: StorageRequest, request: StorageRequest,
slotIndex: UInt256) {.gcsafe, upraises: [].} slotIndex: UInt256) {.gcsafe, upraises: [].}

2
codex/sales/states/finished.nim
View File

@ -23,7 +23,7 @@ method run*(state: SaleFinished, machine: Machine): Future[?State] {.async.} =
if request =? data.request and if request =? data.request and
slotIndex =? data.slotIndex: slotIndex =? data.slotIndex:
context.proving.add(request.slotId(slotIndex)) context.proving.add(Slot(request: request, slotIndex: slotIndex))
if onSale =? context.onSale: if onSale =? context.onSale:
onSale(data.availability, request, slotIndex) onSale(data.availability, request, slotIndex)

4
codex/sales/states/proving.nim
View File

@ -28,8 +28,8 @@ method run*(state: SaleProving, machine: Machine): Future[?State] {.async.} =
without request =? data.request: without request =? data.request:
raiseAssert "no sale request" raiseAssert "no sale request"
without onProve =? context.onProve: without onProve =? context.proving.onProve:
raiseAssert "onProve callback not set" raiseAssert "onProve callback not set"
let proof = await onProve(request, data.slotIndex) let proof = await onProve(Slot(request: request, slotIndex: data.slotIndex))
return some State(SaleFilling(proof: proof)) return some State(SaleFilling(proof: proof))

7
docs/TWOCLIENTTEST.md
View File

@ -152,11 +152,14 @@ curl --location 'http://localhost:8080/api/codex/v1/storage/request/<CID>' \
--header 'Content-Type: application/json' \ --header 'Content-Type: application/json' \
--data '{ --data '{
"reward": "0x400", "reward": "0x400",
"duration": "0x78" "duration": "0x78",
"proofProbability": "0x10"
}' }'
``` ```
This creates a storage Request for `<CID>` (that you have to fill in) for duration of 2 minutes and with reward of 1024 tokens. This creates a storage Request for `<CID>` (that you have to fill in) for
duration of 2 minutes and with reward of 1024 tokens. It expects hosts to
provide a storage proof once every 16 periods on average.
It returns Request ID which you can then use to query for the Request's state as follows: It returns Request ID which you can then use to query for the Request's state as follows:

14
openapi.yaml
View File

@ -42,6 +42,10 @@ components:
type: string type: string
description: The duration of the request in seconds as hexadecimal string description: The duration of the request in seconds as hexadecimal string
ProofProbability:
type: string
description: How often storage proofs are required as hexadecimal string
Expiry: Expiry:
type: string type: string
description: A timestamp as seconds since unix epoch at which this request expires if the Request does not find requested amount of nodes to host the data. description: A timestamp as seconds since unix epoch at which this request expires if the Request does not find requested amount of nodes to host the data.
@ -114,19 +118,22 @@ components:
required: required:
- reward - reward
- duration - duration
- proofProbability
properties: properties:
duration: duration:
$ref: "#/components/schemas/Duration" $ref: "#/components/schemas/Duration"
reward: reward:
$ref: "#/components/schemas/Reward" $ref: "#/components/schemas/Reward"
proofProbability:
$ref: "#/components/schemas/ProofProbability"
nodes: nodes:
type: number type: number
description: Minimal number of nodes the content should be stored on description: Minimal number of nodes the content should be stored on
default: 1 node default: 1
tolerance: tolerance:
type: number type: number
description: Additional number of nodes on top of the `nodes` property that can be lost before pronouncing the content lost description: Additional number of nodes on top of the `nodes` property that can be lost before pronouncing the content lost
default: 0 nodes default: 0
StorageAsk: StorageAsk:
type: object type: object
@ -142,8 +149,7 @@ components:
duration: duration:
$ref: "#/components/schemas/Duration" $ref: "#/components/schemas/Duration"
proofProbability: proofProbability:
type: string $ref: "#/components/schemas/ProofProbability"
description: How often storage proofs are required as hexadecimal string
reward: reward:
$ref: "#/components/schemas/Reward" $ref: "#/components/schemas/Reward"
maxSlotLoss: maxSlotLoss:

15
tests/codex/sales/testsales.nim
View File

@ -47,8 +47,7 @@ suite "Sales":
slot: UInt256, slot: UInt256,
availability: ?Availability) {.async.} = availability: ?Availability) {.async.} =
discard discard
sales.onProve = proc(request: StorageRequest, proving.onProve = proc(slot: Slot): Future[seq[byte]] {.async.} =
slot: UInt256): Future[seq[byte]] {.async.} =
return proof return proof
await sales.start() await sales.start()
request.expiry = (clock.now() + 42).u256 request.expiry = (clock.now() + 42).u256
@ -146,10 +145,9 @@ suite "Sales":
test "generates proof of storage": test "generates proof of storage":
var provingRequest: StorageRequest var provingRequest: StorageRequest
var provingSlot: UInt256 var provingSlot: UInt256
sales.onProve = proc(request: StorageRequest, proving.onProve = proc(slot: Slot): Future[seq[byte]] {.async.} =
slot: UInt256): Future[seq[byte]] {.async.} = provingRequest = slot.request
provingRequest = request provingSlot = slot.slotIndex
provingSlot = slot
sales.add(availability) sales.add(availability)
await market.requestStorage(request) await market.requestStorage(request)
check eventually provingRequest == request check eventually provingRequest == request
@ -184,8 +182,7 @@ suite "Sales":
test "calls onClear when storage becomes available again": test "calls onClear when storage becomes available again":
# fail the proof intentionally to trigger `agent.finish(success=false)`, # fail the proof intentionally to trigger `agent.finish(success=false)`,
# which then calls the onClear callback # which then calls the onClear callback
sales.onProve = proc(request: StorageRequest, proving.onProve = proc(slot: Slot): Future[seq[byte]] {.async.} =
slot: UInt256): Future[seq[byte]] {.async.} =
raise newException(IOError, "proof failed") raise newException(IOError, "proof failed")
var clearedAvailability: Availability var clearedAvailability: Availability
var clearedRequest: StorageRequest var clearedRequest: StorageRequest
@ -238,7 +235,7 @@ suite "Sales":
sales.add(availability) sales.add(availability)
await market.requestStorage(request) await market.requestStorage(request)
check eventually proving.slots.len == 1 check eventually proving.slots.len == 1
check proving.slots.contains(request.slotId(soldSlotIndex)) check proving.slots.contains(Slot(request: request, slotIndex: soldSlotIndex))
test "loads active slots from market": test "loads active slots from market":
let me = await market.getSigner() let me = await market.getSigner()

112
tests/codex/testproving.nim
View File

@ -25,87 +25,85 @@ suite "Proving":
let periodicity = await proofs.periodicity() let periodicity = await proofs.periodicity()
clock.advance(periodicity.seconds.truncate(int64)) clock.advance(periodicity.seconds.truncate(int64))
test "maintains a list of contract ids to watch": test "maintains a list of slots to watch":
let id1, id2 = SlotId.example let slot1, slot2 = Slot.example
check proving.slots.len == 0 check proving.slots.len == 0
proving.add(id1) proving.add(slot1)
check proving.slots.contains(id1) check proving.slots.contains(slot1)
proving.add(id2) proving.add(slot2)
check proving.slots.contains(id1) check proving.slots.contains(slot1)
check proving.slots.contains(id2) check proving.slots.contains(slot2)
test "removes duplicate contract ids": test "removes duplicate slots":
let id = SlotId.example let slot = Slot.example
proving.add(id) proving.add(slot)
proving.add(id) proving.add(slot)
check proving.slots.len == 1 check proving.slots.len == 1
test "invokes callback when proof is required": test "invokes callback when proof is required":
let id = SlotId.example let slot = Slot.example
proving.add(id) proving.add(slot)
var called: bool var called: bool
proc onProofRequired(id: SlotId) = proc onProve(slot: Slot): Future[seq[byte]] {.async.} =
called = true called = true
proving.onProofRequired = onProofRequired proving.onProve = onProve
proofs.setSlotState(id, SlotState.Filled) proofs.setSlotState(slot.id, SlotState.Filled)
proofs.setProofRequired(id, true) proofs.setProofRequired(slot.id, true)
await proofs.advanceToNextPeriod() await proofs.advanceToNextPeriod()
check eventually called check eventually called
test "callback receives id of contract for which proof is required": test "callback receives slot for which proof is required":
let id1, id2 = SlotId.example let slot1, slot2 = Slot.example
proving.add(id1) proving.add(slot1)
proving.add(id2) proving.add(slot2)
var callbackIds: seq[SlotId] var callbackSlots: seq[Slot]
proc onProofRequired(id: SlotId) = proc onProve(slot: Slot): Future[seq[byte]] {.async.} =
callbackIds.add(id) callbackSlots.add(slot)
proving.onProofRequired = onProofRequired proving.onProve = onProve
proofs.setSlotState(id1, SlotState.Filled) proofs.setSlotState(slot1.id, SlotState.Filled)
proofs.setSlotState(id2, SlotState.Filled) proofs.setSlotState(slot2.id, SlotState.Filled)
proofs.setProofRequired(id1, true) proofs.setProofRequired(slot1.id, true)
await proofs.advanceToNextPeriod() await proofs.advanceToNextPeriod()
check eventually callbackIds == @[id1] check eventually callbackSlots == @[slot1]
proofs.setProofRequired(id1, false) proofs.setProofRequired(slot1.id, false)
proofs.setProofRequired(id2, true) proofs.setProofRequired(slot2.id, true)
await proofs.advanceToNextPeriod() await proofs.advanceToNextPeriod()
check eventually callbackIds == @[id1, id2] check eventually callbackSlots == @[slot1, slot2]
test "invokes callback when proof is about to be required": test "invokes callback when proof is about to be required":
let id = SlotId.example let slot = Slot.example
proving.add(id) proving.add(slot)
var called: bool var called: bool
proc onProofRequired(id: SlotId) = proc onProve(slot: Slot): Future[seq[byte]] {.async.} =
called = true called = true
proving.onProofRequired = onProofRequired proving.onProve = onProve
proofs.setProofRequired(id, false) proofs.setProofRequired(slot.id, false)
proofs.setProofToBeRequired(id, true) proofs.setProofToBeRequired(slot.id, true)
proofs.setSlotState(id, SlotState.Filled) proofs.setSlotState(slot.id, SlotState.Filled)
await proofs.advanceToNextPeriod() await proofs.advanceToNextPeriod()
check eventually called check eventually called
test "stops watching when contract has ended": test "stops watching when slot is finished":
let id = SlotId.example let slot = Slot.example
proving.add(id) proving.add(slot)
proofs.setProofEnd(id, clock.now().u256) proofs.setProofEnd(slot.id, clock.now().u256)
await proofs.advanceToNextPeriod() await proofs.advanceToNextPeriod()
var called: bool var called: bool
proc onProofRequired(id: SlotId) = proc onProve(slot: Slot): Future[seq[byte]] {.async.} =
called = true called = true
proving.onProofRequired = onProofRequired proving.onProve = onProve
proofs.setProofRequired(id, true) proofs.setProofRequired(slot.id, true)
await proofs.advanceToNextPeriod() await proofs.advanceToNextPeriod()
proofs.setSlotState(id, SlotState.Finished) proofs.setSlotState(slot.id, SlotState.Finished)
check eventually (not proving.slots.contains(id)) check eventually (not proving.slots.contains(slot))
check not called check not called
test "submits proofs": test "submits proofs":
let id = SlotId.example let slot = Slot.example
let proof = exampleProof() let proof = exampleProof()
await proving.submitProof(id, proof)
test "supports proof submission subscriptions": proving.onProve = proc (slot: Slot): Future[seq[byte]] {.async.} =
let id = SlotId.example return proof
let proof = exampleProof()
var receivedIds: seq[SlotId] var receivedIds: seq[SlotId]
var receivedProofs: seq[seq[byte]] var receivedProofs: seq[seq[byte]]
@ -116,9 +114,11 @@ suite "Proving":
let subscription = await proving.subscribeProofSubmission(onProofSubmission) let subscription = await proving.subscribeProofSubmission(onProofSubmission)
await proving.submitProof(id, proof) proving.add(slot)
proofs.setSlotState(slot.id, SlotState.Filled)
proofs.setProofRequired(slot.id, true)
await proofs.advanceToNextPeriod()
check receivedIds == @[id] check eventually receivedIds == @[slot.id] and receivedProofs == @[proof]
check receivedProofs == @[proof]
await subscription.unsubscribe() await subscription.unsubscribe()

5
tests/examples.nim
View File

@ -47,6 +47,11 @@ proc example*(_: type StorageRequest): StorageRequest =
nonce: Nonce.example nonce: Nonce.example
) )
proc example*(_: type Slot): Slot =
let request = StorageRequest.example
let slotIndex = rand(request.ask.slots.int).u256
Slot(request: request, slotIndex: slotIndex)
proc exampleProof*(): seq[byte] = proc exampleProof*(): seq[byte] =
var proof: seq[byte] var proof: seq[byte]
while proof.len == 0: while proof.len == 0:

73
tests/integration/codexclient.nim Normal file
View File

@ -0,0 +1,73 @@
import std/httpclient
import std/json
import std/strutils
import pkg/stint
import pkg/questionable/results
type CodexClient* = ref object
http: HttpClient
baseurl: string
proc new*(_: type CodexClient, baseurl: string): CodexClient =
CodexClient(http: newHttpClient(), baseurl: baseurl)
proc info*(client: CodexClient): JsonNode =
let url = client.baseurl & "/debug/info"
client.http.getContent(url).parseJson()
proc setLogLevel*(client: CodexClient, level: string) =
let url = client.baseurl & "/debug/chronicles/loglevel?level=" & level
let headers = newHttpHeaders({"Content-Type": "text/plain"})
let response = client.http.request(url, httpMethod=HttpPost, headers=headers)
assert response.status == "200 OK"
proc upload*(client: CodexClient, contents: string): string =
let response = client.http.post(client.baseurl & "/upload", contents)
assert response.status == "200 OK"
response.body
proc requestStorage*(client: CodexClient,
cid: string,
duration: uint64,
reward: uint64,
proofProbability: uint64,
expiry: UInt256): string =
let url = client.baseurl & "/storage/request/" & cid
let json = %*{
"duration": "0x" & duration.toHex,
"reward": "0x" & reward.toHex,
"proofProbability": "0x" & proofProbability.toHex,
"expiry": "0x" & expiry.toHex
}
let response = client.http.post(url, $json)
assert response.status == "200 OK"
response.body
proc getPurchase*(client: CodexClient, purchase: string): JsonNode =
let url = client.baseurl & "/storage/purchases/" & purchase
let body = client.http.getContent(url)
parseJson(body).catch |? nil
proc postAvailability*(client: CodexClient,
size, duration, minPrice: uint64): JsonNode =
let url = client.baseurl & "/sales/availability"
let json = %*{
"size": "0x" & size.toHex,
"duration": "0x" & duration.toHex,
"minPrice": "0x" & minPrice.toHex
}
let response = client.http.post(url, $json)
assert response.status == "200 OK"
parseJson(response.body)
proc getAvailabilities*(client: CodexClient): JsonNode =
let url = client.baseurl & "/sales/availability"
let body = client.http.getContent(url)
parseJson(body)
proc close*(client: CodexClient) =
client.http.close()
proc restart*(client: CodexClient) =
client.http.close()
client.http = newHttpClient()

165
tests/integration/testIntegration.nim
View File

@ -1,156 +1,75 @@
import std/os
import std/httpclient
import std/json import std/json
import std/strutils
import pkg/chronos import pkg/chronos
import ../ethertest
import ../contracts/time import ../contracts/time
import ../codex/helpers/eventually import ../codex/helpers/eventually
import ./nodes import ./twonodes
import ./tokens import ./tokens
ethersuite "Integration tests": twonodessuite "Integration tests", debug1 = false, debug2 = false:
var node1, node2: NodeProcess
var baseurl1, baseurl2: string
var client: HttpClient
let dataDir1 = getTempDir() / "Codex1"
let dataDir2 = getTempDir() / "Codex2"
setup: setup:
await provider.getSigner(accounts[0]).mint() await provider.getSigner(accounts[0]).mint()
await provider.getSigner(accounts[1]).mint() await provider.getSigner(accounts[1]).mint()
await provider.getSigner(accounts[1]).deposit() await provider.getSigner(accounts[1]).deposit()
baseurl1 = "http://localhost:8080/api/codex/v1"
baseurl2 = "http://localhost:8081/api/codex/v1"
client = newHttpClient()
node1 = startNode([
"--api-port=8080",
"--data-dir=" & dataDir1,
"--nat=127.0.0.1",
"--disc-ip=127.0.0.1",
"--disc-port=8090",
"--persistence",
"--eth-account=" & $accounts[0]
], debug = false)
let
bootstrap = strip(
$(parseJson(client.get(baseurl1 & "/debug/info").body)["spr"]),
chars = {'"'})
node2 = startNode([
"--api-port=8081",
"--data-dir=" & dataDir2,
"--nat=127.0.0.1",
"--disc-ip=127.0.0.1",
"--disc-port=8091",
"--bootstrap-node=" & bootstrap,
"--persistence",
"--eth-account=" & $accounts[1]
], debug = false)
teardown:
client.close()
node1.stop()
node2.stop()
dataDir1.removeDir()
dataDir2.removeDir()
test "nodes can print their peer information": test "nodes can print their peer information":
let info1 = client.get(baseurl1 & "/debug/info").body check client1.info() != client2.info()
let info2 = client.get(baseurl2 & "/debug/info").body
check info1 != info2
test "nodes should set chronicles log level": test "nodes can set chronicles log level":
client.headers = newHttpHeaders({ "Content-Type": "text/plain" }) client1.setLogLevel("DEBUG;TRACE:codex")
let filter = "/debug/chronicles/loglevel?level=DEBUG;TRACE:codex"
check client.request(baseurl1 & filter, httpMethod = HttpPost, body = "").status == "200 OK"
test "node accepts file uploads": test "node accepts file uploads":
let url = baseurl1 & "/upload" let cid1 = client1.upload("some file contents")
let response = client.post(url, "some file contents") let cid2 = client1.upload("some other contents")
check response.status == "200 OK" check cid1 != cid2
test "node handles new storage availability": test "node handles new storage availability":
let url = baseurl1 & "/sales/availability" let availability1 = client1.postAvailability(size=1, duration=2, minPrice=3)
let json = %*{"size": "0x1", "duration": "0x2", "minPrice": "0x3"} let availability2 = client1.postAvailability(size=4, duration=5, minPrice=6)
check client.post(url, $json).status == "200 OK" check availability1 != availability2
test "node lists storage that is for sale": test "node lists storage that is for sale":
let url = baseurl1 & "/sales/availability" let availability = client1.postAvailability(size=1, duration=2, minPrice=3)
let json = %*{"size": "0x1", "duration": "0x2", "minPrice": "0x3"} check availability in client1.getAvailabilities()
let availability = parseJson(client.post(url, $json).body)
let response = client.get(url)
check response.status == "200 OK"
check %*availability in parseJson(response.body)
test "node handles storage request": test "node handles storage request":
let cid = client.post(baseurl1 & "/upload", "some file contents").body let expiry = (await provider.currentTime()) + 30
let url = baseurl1 & "/storage/request/" & cid let cid = client1.upload("some file contents")
let json = %*{"duration": "0x1", "reward": "0x2"} let id1 = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry)
let response = client.post(url, $json) let id2 = client1.requestStorage(cid, duration=4, reward=5, proofProbability=6, expiry=expiry)
check response.status == "200 OK" check id1 != id2
test "node retrieves purchase status": test "node retrieves purchase status":
let cid = client.post(baseurl1 & "/upload", "some file contents").body let expiry = (await provider.currentTime()) + 30
let request = %*{"duration": "0x1", "reward": "0x2"} let cid = client1.upload("some file contents")
let id = client.post(baseurl1 & "/storage/request/" & cid, $request).body let id = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry)
let response = client.get(baseurl1 & "/storage/purchases/" & id) let purchase = client1.getPurchase(id)
check response.status == "200 OK" check purchase{"request"}{"ask"}{"duration"} == %"0x1"
let json = parseJson(response.body) check purchase{"request"}{"ask"}{"reward"} == %"0x2"
check json["request"]["ask"]["duration"].getStr == "0x1" check purchase{"request"}{"ask"}{"proofProbability"} == %"0x3"
check json["request"]["ask"]["reward"].getStr == "0x2"
test "node remembers purchase status after restart": test "node remembers purchase status after restart":
let cid = client.post(baseurl1 & "/upload", "some file contents").body let expiry = (await provider.currentTime()) + 30
let request = %*{"duration": "0x1", "reward": "0x2"} let cid = client1.upload("some file contents")
let id = client.post(baseurl1 & "/storage/request/" & cid, $request).body let id = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry)
check eventually client1.getPurchase(id){"state"}.getStr() == "submitted"
proc getPurchase(id: string): JsonNode =
let response = client.get(baseurl1 & "/storage/purchases/" & id)
return parseJson(response.body).catch |? nil
check eventually getPurchase(id){"state"}.getStr == "submitted"
node1.restart() node1.restart()
client1.restart()
client.close() check eventually (not isNil client1.getPurchase(id){"request"}{"ask"})
client = newHttpClient() check client1.getPurchase(id){"request"}{"ask"}{"duration"} == %"0x1"
check client1.getPurchase(id){"request"}{"ask"}{"reward"} == %"0x2"
check eventually (not isNil getPurchase(id){"request"}{"ask"})
check getPurchase(id){"request"}{"ask"}{"duration"}.getStr == "0x1"
check getPurchase(id){"request"}{"ask"}{"reward"}.getStr == "0x2"
test "nodes negotiate contracts on the marketplace": test "nodes negotiate contracts on the marketplace":
proc sell = # client 2 makes storage available
let json = %*{"size": "0xFFFFF", "duration": "0x200", "minPrice": "0x300"} discard client2.postAvailability(size=0xFFFFF, duration=200, minPrice=300)
discard client.post(baseurl2 & "/sales/availability", $json)
proc available: JsonNode = # client 1 requests storage
client.get(baseurl2 & "/sales/availability").body.parseJson let expiry = (await provider.currentTime()) + 30
let cid = client1.upload("some file contents")
let purchase = client1.requestStorage(cid, duration=100, reward=400, proofProbability=3, expiry=expiry)
proc upload: string = check eventually client1.getPurchase(purchase){"state"} == %"started"
client.post(baseurl1 & "/upload", "some file contents").body check client1.getPurchase(purchase){"error"} == newJNull()
check client2.getAvailabilities().len == 0
proc buy(cid: string): string =
let expiry = ((waitFor provider.currentTime()) + 30).toHex
let json = %*{"duration": "0x1", "reward": "0x400", "expiry": expiry}
client.post(baseurl1 & "/storage/request/" & cid, $json).body
proc finish(purchase: string): Future[JsonNode] {.async.} =
while true:
let response = client.get(baseurl1 & "/storage/purchases/" & purchase)
let json = parseJson(response.body)
if json["state"].getStr == "finished": return json
await sleepAsync(1.seconds)
sell()
let purchase = waitFor upload().buy().finish()
check purchase["error"].getStr == ""
check available().len == 0

10
tests/integration/testblockexpiration.nim
View File

@ -29,7 +29,7 @@ ethersuite "Node block expiration tests":
"--disc-ip=127.0.0.1", "--disc-ip=127.0.0.1",
"--disc-port=8090", "--disc-port=8090",
"--block-ttl=" & $blockTtlSeconds, "--block-ttl=" & $blockTtlSeconds,
"--block-mi=3", "--block-mi=1",
"--block-mn=10" "--block-mn=10"
], debug = false) ], debug = false)
@ -49,11 +49,11 @@ ethersuite "Node block expiration tests":
content content
test "node retains not-expired file": test "node retains not-expired file":
startTestNode(blockTtlSeconds = 60 * 60 * 1) startTestNode(blockTtlSeconds = 10)
let contentId = uploadTestFile() let contentId = uploadTestFile()
await sleepAsync(10.seconds) await sleepAsync(2.seconds)
let response = downloadTestFile(contentId) let response = downloadTestFile(contentId)
check: check:
@ -61,11 +61,11 @@ ethersuite "Node block expiration tests":
response.body == content response.body == content
test "node deletes expired file": test "node deletes expired file":
startTestNode(blockTtlSeconds = 5) startTestNode(blockTtlSeconds = 1)
let contentId = uploadTestFile() let contentId = uploadTestFile()
await sleepAsync(10.seconds) await sleepAsync(2.seconds)
expect TimeoutError: expect TimeoutError:
discard downloadTestFile(contentId) discard downloadTestFile(contentId)

43
tests/integration/testproofs.nim Normal file
View File

@ -0,0 +1,43 @@
import codex/contracts/marketplace
import codex/contracts/deployment
import ../contracts/time
import ../codex/helpers/eventually
import ./twonodes
import ./tokens
twonodessuite "Proving integration test", debug1=false, debug2=false:
var marketplace: Marketplace
var config: MarketplaceConfig
setup:
marketplace = Marketplace.new(!deployment().address(Marketplace), provider)
config = await marketplace.config()
await provider.getSigner(accounts[0]).mint()
await provider.getSigner(accounts[1]).mint()
await provider.getSigner(accounts[1]).deposit()
proc waitUntilPurchaseIsStarted {.async.} =
discard client2.postAvailability(size=0xFFFFF, duration=200, minPrice=300)
let expiry = (await provider.currentTime()) + 30
let cid = client1.upload("some file contents")
let purchase = client1.requestStorage(cid, duration=100, reward=400, proofProbability=3, expiry=expiry)
check eventually client1.getPurchase(purchase){"state"} == %"started"
test "hosts submit periodic proofs for slots they fill":
await waitUntilPurchaseIsStarted()
var proofWasSubmitted = false
proc onProofSubmitted(event: ProofSubmitted) =
proofWasSubmitted = true
let subscription = await marketplace.subscribe(ProofSubmitted, onProofSubmitted)
for _ in 0..<100:
if proofWasSubmitted:
break
else:
await provider.advanceTime(config.proofs.period)
await sleepAsync(1.seconds)
check proofWasSubmitted
await subscription.unsubscribe()

62
tests/integration/twonodes.nim Normal file
View File

@ -0,0 +1,62 @@
import std/os
import std/macros
import std/json
import std/httpclient
import ../ethertest
import ./codexclient
import ./nodes
export ethertest
export codexclient
export nodes
template twonodessuite*(name: string, debug1, debug2: bool, body) =
ethersuite name:
var node1 {.inject, used.}: NodeProcess
var node2 {.inject, used.}: NodeProcess
var client1 {.inject, used.}: CodexClient
var client2 {.inject, used.}: CodexClient
let dataDir1 = getTempDir() / "Codex1"
let dataDir2 = getTempDir() / "Codex2"
setup:
client1 = CodexClient.new("http://localhost:8080/api/codex/v1")
client2 = CodexClient.new("http://localhost:8081/api/codex/v1")
node1 = startNode([
"--api-port=8080",
"--data-dir=" & dataDir1,
"--nat=127.0.0.1",
"--disc-ip=127.0.0.1",
"--disc-port=8090",
"--persistence",
"--eth-account=" & $accounts[0]
], debug = debug1)
let bootstrap = client1.info()["spr"].getStr()
node2 = startNode([
"--api-port=8081",
"--data-dir=" & dataDir2,
"--nat=127.0.0.1",
"--disc-ip=127.0.0.1",
"--disc-port=8091",
"--bootstrap-node=" & bootstrap,
"--persistence",
"--eth-account=" & $accounts[1]
], debug = debug2)
teardown:
client1.close()
client2.close()
node1.stop()
node2.stop()
removeDir(dataDir1)
removeDir(dataDir2)
body

1
tests/testIntegration.nim
View File

@ -1,4 +1,5 @@
import ./integration/testIntegration import ./integration/testIntegration
import ./integration/testblockexpiration import ./integration/testblockexpiration
import ./integration/testproofs
{.warning[UnusedImport]:off.} {.warning[UnusedImport]:off.}