codex-storage
/
nim-codex
mirror of https://github.com/codex-storage/nim-codex.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Generate proofs when required (#383) 

* [maintenance] speedup integration test

* [rest api] add proofProbability parameter to storage requests

* [integration] negotiation test ends when contract starts

* [integration] reusable 2 node setup for tests

* [integration] introduce CodexClient for tests

* [node] submit storage proofs when required

* [contracts] Add Slot type

* [proving] replace onProofRequired & submitProof with onProve

Removes duplication between Sales.onProve() and
Proving.onProofRequired()
This commit is contained in:
markspanbroek 2023-03-27 15:47:25 +02:00 committed by GitHub
parent 067c1c9625
commit 4ffe7b8e06
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 360 additions and 236 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
codex/contracts/requests.nim
View File

@ -30,6 +30,9 @@ type
u*: seq[byte]
publicKey*: seq[byte]
name*: seq[byte]
Slot* = object
request*: StorageRequest
slotIndex*: UInt256
SlotId* = distinct array[32, byte]
RequestId* = distinct array[32, byte]
Nonce* = distinct array[32, byte]
@ -50,6 +53,8 @@ proc `==`*(x, y: Nonce): bool {.borrow.}
proc `==`*(x, y: RequestId): bool {.borrow.}
proc `==`*(x, y: SlotId): bool {.borrow.}
proc hash*(x: SlotId): Hash {.borrow.}
proc hash*(x: Nonce): Hash {.borrow.}
proc hash*(x: Address): Hash {.borrow.}
func toArray*(id: RequestId | SlotId | Nonce): array[32, byte] =
array[32, byte](id)
@ -159,6 +164,9 @@ func slotId*(requestId: RequestId, slot: UInt256): SlotId =
func slotId*(request: StorageRequest, slot: UInt256): SlotId =
slotId(request.id, slot)
func id*(slot: Slot): SlotId =
slotId(slot.request, slot.slotIndex)
func pricePerSlot*(ask: StorageAsk): UInt256 =
ask.duration * ask.reward

5
codex/node.nim
View File

@ -235,6 +235,7 @@ proc store*(
proc requestStorage*(self: CodexNodeRef,
cid: Cid,
duration: UInt256,
proofProbability: UInt256,
nodes: uint,
tolerance: uint,
reward: UInt256,
@ -280,6 +281,7 @@ proc requestStorage*(self: CodexNodeRef,
slots: nodes + tolerance,
slotSize: (encoded.blockSize * encoded.steps).u256,
duration: duration,
proofProbability: proofProbability,
reward: reward,
maxSlotLoss: tolerance
),
@ -360,8 +362,7 @@ proc start*(node: CodexNodeRef) {.async.} =
# TODO: remove data from local storage
discard
contracts.sales.onProve = proc(request: StorageRequest,
slot: UInt256): Future[seq[byte]] {.async.} =
contracts.proving.onProve = proc(slot: Slot): Future[seq[byte]] {.async.} =
# TODO: generate proof
return @[42'u8]

45
codex/proving.nim
View File

@ -13,18 +13,21 @@ type
proofs: Proofs
clock: Clock
loop: ?Future[void]
slots*: HashSet[SlotId]
onProofRequired: ?OnProofRequired
OnProofRequired* = proc (id: SlotId) {.gcsafe, upraises:[].}
slots*: HashSet[Slot]
onProve: ?OnProve
OnProve* = proc(slot: Slot): Future[seq[byte]] {.gcsafe, upraises: [].}
func new*(_: type Proving, proofs: Proofs, clock: Clock): Proving =
Proving(proofs: proofs, clock: clock)
proc `onProofRequired=`*(proving: Proving, callback: OnProofRequired) =
proving.onProofRequired = some callback
proc onProve*(proving: Proving): ?OnProve =
proving.onProve
func add*(proving: Proving, id: SlotId) =
proving.slots.incl(id)
proc `onProve=`*(proving: Proving, callback: OnProve) =
proving.onProve = some callback
func add*(proving: Proving, slot: Slot) =
proving.slots.incl(slot)
proc getCurrentPeriod(proving: Proving): Future[Period] {.async.} =
let periodicity = await proving.proofs.periodicity()
@ -35,23 +38,32 @@ proc waitUntilPeriod(proving: Proving, period: Period) {.async.} =
await proving.clock.waitUntil(periodicity.periodStart(period).truncate(int64))
proc removeEndedContracts(proving: Proving) {.async.} =
var ended: HashSet[SlotId]
for id in proving.slots:
let state = await proving.proofs.slotState(id)
var ended: HashSet[Slot]
for slot in proving.slots:
let state = await proving.proofs.slotState(slot.id)
if state != SlotState.Filled:
ended.incl(id)
ended.incl(slot)
proving.slots.excl(ended)
proc prove(proving: Proving, slot: Slot) {.async.} =
without onProve =? proving.onProve:
raiseAssert "onProve callback not set"
try:
let proof = await onProve(slot)
await proving.proofs.submitProof(slot.id, proof)
except CatchableError as e:
error "Submitting proof failed", msg = e.msg
proc run(proving: Proving) {.async.} =
try:
while true:
let currentPeriod = await proving.getCurrentPeriod()
await proving.removeEndedContracts()
for id in proving.slots:
for slot in proving.slots:
let id = slot.id
if (await proving.proofs.isProofRequired(id)) or
(await proving.proofs.willProofBeRequired(id)):
if callback =? proving.onProofRequired:
callback(id)
(await proving.proofs.willProofBeRequired(id)):
asyncSpawn proving.prove(slot)
await proving.waitUntilPeriod(currentPeriod + 1)
except CancelledError:
discard
@ -70,9 +82,6 @@ proc stop*(proving: Proving) {.async.} =
if not loop.finished:
await loop.cancelAndWait()
proc submitProof*(proving: Proving, id: SlotId, proof: seq[byte]) {.async.} =
await proving.proofs.submitProof(id, proof)
proc subscribeProofSubmission*(proving: Proving,
callback: OnProofSubmitted):
Future[Subscription] =

14
codex/rest/api.nim
View File

@ -133,12 +133,13 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
"/api/codex/v1/storage/request/{cid}") do (cid: Cid) -> RestApiResponse:
## Create a request for storage
##
## cid - the cid of a previously uploaded dataset
## duration - the duration of the request in seconds
## reward - the maximum amount of tokens paid per second per slot to hosts the client is willing to pay
## expiry - timestamp, in seconds, when the request expires if the Request does not find requested amount of nodes to host the data
## nodes - minimal number of nodes the content should be stored on
## tolerance - allowed number of nodes that can be lost before pronouncing the content lost
## cid - the cid of a previously uploaded dataset
## duration - the duration of the request in seconds
## proofProbability - how often storage proofs are required
## reward - the maximum amount of tokens paid per second per slot to hosts the client is willing to pay
## expiry - timestamp, in seconds, when the request expires if the Request does not find requested amount of nodes to host the data
## nodes - minimal number of nodes the content should be stored on
## tolerance - allowed number of nodes that can be lost before pronouncing the content lost
without cid =? cid.tryGet.catch, error:
return RestApiResponse.error(Http400, error.msg)
@ -154,6 +155,7 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
without purchaseId =? await node.requestStorage(
cid,
params.duration,
params.proofProbability,
nodes,
tolerance,
params.reward,

3
codex/rest/json.nim
View File

@ -8,6 +8,7 @@ import ../purchasing
type
StorageRequestParams* = object
duration*: UInt256
proofProbability*: UInt256
reward*: UInt256
expiry*: ?UInt256
nodes*: ?uint
@ -24,12 +25,14 @@ proc fromJson*(_: type StorageRequestParams,
bytes: seq[byte]): ?! StorageRequestParams =
let json = ?catch parseJson(string.fromBytes(bytes))
let duration = ?catch UInt256.fromHex(json["duration"].getStr)
let proofProbability = ?catch UInt256.fromHex(json["proofProbability"].getStr)
let reward = ?catch UInt256.fromHex(json["reward"].getStr)
let expiry = UInt256.fromHex(json["expiry"].getStr).catch.option
let nodes = strutils.fromHex[uint](json["nodes"].getStr).catch.option
let tolerance = strutils.fromHex[uint](json["tolerance"].getStr).catch.option
success StorageRequestParams(
duration: duration,
proofProbability: proofProbability,
reward: reward,
expiry: expiry,
nodes: nodes,

5
codex/sales.nim
View File

@ -47,9 +47,6 @@ type
proc `onStore=`*(sales: Sales, onStore: OnStore) =
sales.context.onStore = some onStore
proc `onProve=`*(sales: Sales, onProve: OnProve) =
sales.context.onProve = some onProve
proc `onClear=`*(sales: Sales, onClear: OnClear) =
sales.context.onClear = some onClear
@ -58,8 +55,6 @@ proc `onSale=`*(sales: Sales, callback: OnSale) =
proc onStore*(sales: Sales): ?OnStore = sales.context.onStore
proc onProve*(sales: Sales): ?OnProve = sales.context.onProve
proc onClear*(sales: Sales): ?OnClear = sales.context.onClear
proc onSale*(sales: Sales): ?OnSale = sales.context.onSale

3
codex/sales/salescontext.nim
View File

@ -9,7 +9,6 @@ type
market*: Market
clock*: Clock
onStore*: ?OnStore
onProve*: ?OnProve
onClear*: ?OnClear
onSale*: ?OnSale
onSaleErrored*: ?OnSaleErrored
@ -17,8 +16,6 @@ type
OnStore* = proc(request: StorageRequest,
slot: UInt256,
availability: ?Availability): Future[void] {.gcsafe, upraises: [].}
OnProve* = proc(request: StorageRequest,
slot: UInt256): Future[seq[byte]] {.gcsafe, upraises: [].}
OnClear* = proc(availability: ?Availability,# TODO: when availability changes introduced, make availability non-optional (if we need to keep it at all)
request: StorageRequest,
slotIndex: UInt256) {.gcsafe, upraises: [].}

2
codex/sales/states/finished.nim
View File

@ -23,7 +23,7 @@ method run*(state: SaleFinished, machine: Machine): Future[?State] {.async.} =
if request =? data.request and
slotIndex =? data.slotIndex:
context.proving.add(request.slotId(slotIndex))
context.proving.add(Slot(request: request, slotIndex: slotIndex))
if onSale =? context.onSale:
onSale(data.availability, request, slotIndex)

4
codex/sales/states/proving.nim
View File

@ -28,8 +28,8 @@ method run*(state: SaleProving, machine: Machine): Future[?State] {.async.} =
without request =? data.request:
raiseAssert "no sale request"
without onProve =? context.onProve:
without onProve =? context.proving.onProve:
raiseAssert "onProve callback not set"
let proof = await onProve(request, data.slotIndex)
let proof = await onProve(Slot(request: request, slotIndex: data.slotIndex))
return some State(SaleFilling(proof: proof))

7
docs/TWOCLIENTTEST.md
View File

@ -152,11 +152,14 @@ curl --location 'http://localhost:8080/api/codex/v1/storage/request/<CID>' \
--header 'Content-Type: application/json' \
--data '{
"reward": "0x400",
"duration": "0x78"
"duration": "0x78",
"proofProbability": "0x10"
}'
```
This creates a storage Request for `<CID>` (that you have to fill in) for duration of 2 minutes and with reward of 1024 tokens.
This creates a storage Request for `<CID>` (that you have to fill in) for
duration of 2 minutes and with reward of 1024 tokens. It expects hosts to
provide a storage proof once every 16 periods on average.
It returns Request ID which you can then use to query for the Request's state as follows:

14
openapi.yaml
View File

@ -42,6 +42,10 @@ components:
type: string
description: The duration of the request in seconds as hexadecimal string
ProofProbability:
type: string
description: How often storage proofs are required as hexadecimal string
Expiry:
type: string
description: A timestamp as seconds since unix epoch at which this request expires if the Request does not find requested amount of nodes to host the data.
@ -114,19 +118,22 @@ components:
required:
- reward
- duration
- proofProbability
properties:
duration:
$ref: "#/components/schemas/Duration"
reward:
$ref: "#/components/schemas/Reward"
proofProbability:
$ref: "#/components/schemas/ProofProbability"
nodes:
type: number
description: Minimal number of nodes the content should be stored on
default: 1 node
default: 1
tolerance:
type: number
description: Additional number of nodes on top of the `nodes` property that can be lost before pronouncing the content lost
default: 0 nodes
default: 0
StorageAsk:
type: object
@ -142,8 +149,7 @@ components:
duration:
$ref: "#/components/schemas/Duration"
proofProbability:
type: string
description: How often storage proofs are required as hexadecimal string
$ref: "#/components/schemas/ProofProbability"
reward:
$ref: "#/components/schemas/Reward"
maxSlotLoss:

15
tests/codex/sales/testsales.nim
View File

@ -47,8 +47,7 @@ suite "Sales":
slot: UInt256,
availability: ?Availability) {.async.} =
discard
sales.onProve = proc(request: StorageRequest,
slot: UInt256): Future[seq[byte]] {.async.} =
proving.onProve = proc(slot: Slot): Future[seq[byte]] {.async.} =
return proof
await sales.start()
request.expiry = (clock.now() + 42).u256
@ -146,10 +145,9 @@ suite "Sales":
test "generates proof of storage":
var provingRequest: StorageRequest
var provingSlot: UInt256
sales.onProve = proc(request: StorageRequest,
slot: UInt256): Future[seq[byte]] {.async.} =
provingRequest = request
provingSlot = slot
proving.onProve = proc(slot: Slot): Future[seq[byte]] {.async.} =
provingRequest = slot.request
provingSlot = slot.slotIndex
sales.add(availability)
await market.requestStorage(request)
check eventually provingRequest == request
@ -184,8 +182,7 @@ suite "Sales":
test "calls onClear when storage becomes available again":
# fail the proof intentionally to trigger `agent.finish(success=false)`,
# which then calls the onClear callback
sales.onProve = proc(request: StorageRequest,
slot: UInt256): Future[seq[byte]] {.async.} =
proving.onProve = proc(slot: Slot): Future[seq[byte]] {.async.} =
raise newException(IOError, "proof failed")
var clearedAvailability: Availability
var clearedRequest: StorageRequest
@ -238,7 +235,7 @@ suite "Sales":
sales.add(availability)
await market.requestStorage(request)
check eventually proving.slots.len == 1
check proving.slots.contains(request.slotId(soldSlotIndex))
check proving.slots.contains(Slot(request: request, slotIndex: soldSlotIndex))
test "loads active slots from market":
let me = await market.getSigner()

112
tests/codex/testproving.nim
View File

@ -25,87 +25,85 @@ suite "Proving":
let periodicity = await proofs.periodicity()
clock.advance(periodicity.seconds.truncate(int64))
test "maintains a list of contract ids to watch":
let id1, id2 = SlotId.example
test "maintains a list of slots to watch":
let slot1, slot2 = Slot.example
check proving.slots.len == 0
proving.add(id1)
check proving.slots.contains(id1)
proving.add(id2)
check proving.slots.contains(id1)
check proving.slots.contains(id2)
proving.add(slot1)
check proving.slots.contains(slot1)
proving.add(slot2)
check proving.slots.contains(slot1)
check proving.slots.contains(slot2)
test "removes duplicate contract ids":
let id = SlotId.example
proving.add(id)
proving.add(id)
test "removes duplicate slots":
let slot = Slot.example
proving.add(slot)
proving.add(slot)
check proving.slots.len == 1
test "invokes callback when proof is required":
let id = SlotId.example
proving.add(id)
let slot = Slot.example
proving.add(slot)
var called: bool
proc onProofRequired(id: SlotId) =
proc onProve(slot: Slot): Future[seq[byte]] {.async.} =
called = true
proving.onProofRequired = onProofRequired
proofs.setSlotState(id, SlotState.Filled)
proofs.setProofRequired(id, true)
proving.onProve = onProve
proofs.setSlotState(slot.id, SlotState.Filled)
proofs.setProofRequired(slot.id, true)
await proofs.advanceToNextPeriod()
check eventually called
test "callback receives id of contract for which proof is required":
let id1, id2 = SlotId.example
proving.add(id1)
proving.add(id2)
var callbackIds: seq[SlotId]
proc onProofRequired(id: SlotId) =
callbackIds.add(id)
proving.onProofRequired = onProofRequired
proofs.setSlotState(id1, SlotState.Filled)
proofs.setSlotState(id2, SlotState.Filled)
proofs.setProofRequired(id1, true)
test "callback receives slot for which proof is required":
let slot1, slot2 = Slot.example
proving.add(slot1)
proving.add(slot2)
var callbackSlots: seq[Slot]
proc onProve(slot: Slot): Future[seq[byte]] {.async.} =
callbackSlots.add(slot)
proving.onProve = onProve
proofs.setSlotState(slot1.id, SlotState.Filled)
proofs.setSlotState(slot2.id, SlotState.Filled)
proofs.setProofRequired(slot1.id, true)
await proofs.advanceToNextPeriod()
check eventually callbackIds == @[id1]
proofs.setProofRequired(id1, false)
proofs.setProofRequired(id2, true)
check eventually callbackSlots == @[slot1]
proofs.setProofRequired(slot1.id, false)
proofs.setProofRequired(slot2.id, true)
await proofs.advanceToNextPeriod()
check eventually callbackIds == @[id1, id2]
check eventually callbackSlots == @[slot1, slot2]
test "invokes callback when proof is about to be required":
let id = SlotId.example
proving.add(id)
let slot = Slot.example
proving.add(slot)
var called: bool
proc onProofRequired(id: SlotId) =
proc onProve(slot: Slot): Future[seq[byte]] {.async.} =
called = true
proving.onProofRequired = onProofRequired
proofs.setProofRequired(id, false)
proofs.setProofToBeRequired(id, true)
proofs.setSlotState(id, SlotState.Filled)
proving.onProve = onProve
proofs.setProofRequired(slot.id, false)
proofs.setProofToBeRequired(slot.id, true)
proofs.setSlotState(slot.id, SlotState.Filled)
await proofs.advanceToNextPeriod()
check eventually called
test "stops watching when contract has ended":
let id = SlotId.example
proving.add(id)
proofs.setProofEnd(id, clock.now().u256)
test "stops watching when slot is finished":
let slot = Slot.example
proving.add(slot)
proofs.setProofEnd(slot.id, clock.now().u256)
await proofs.advanceToNextPeriod()
var called: bool
proc onProofRequired(id: SlotId) =
proc onProve(slot: Slot): Future[seq[byte]] {.async.} =
called = true
proving.onProofRequired = onProofRequired
proofs.setProofRequired(id, true)
proving.onProve = onProve
proofs.setProofRequired(slot.id, true)
await proofs.advanceToNextPeriod()
proofs.setSlotState(id, SlotState.Finished)
check eventually (not proving.slots.contains(id))
proofs.setSlotState(slot.id, SlotState.Finished)
check eventually (not proving.slots.contains(slot))
check not called
test "submits proofs":
let id = SlotId.example
let slot = Slot.example
let proof = exampleProof()
await proving.submitProof(id, proof)
test "supports proof submission subscriptions":
let id = SlotId.example
let proof = exampleProof()
proving.onProve = proc (slot: Slot): Future[seq[byte]] {.async.} =
return proof
var receivedIds: seq[SlotId]
var receivedProofs: seq[seq[byte]]
@ -116,9 +114,11 @@ suite "Proving":
let subscription = await proving.subscribeProofSubmission(onProofSubmission)
await proving.submitProof(id, proof)
proving.add(slot)
proofs.setSlotState(slot.id, SlotState.Filled)
proofs.setProofRequired(slot.id, true)
await proofs.advanceToNextPeriod()
check receivedIds == @[id]
check receivedProofs == @[proof]
check eventually receivedIds == @[slot.id] and receivedProofs == @[proof]
await subscription.unsubscribe()

5
tests/examples.nim
View File

@ -47,6 +47,11 @@ proc example*(_: type StorageRequest): StorageRequest =
nonce: Nonce.example
)
proc example*(_: type Slot): Slot =
let request = StorageRequest.example
let slotIndex = rand(request.ask.slots.int).u256
Slot(request: request, slotIndex: slotIndex)
proc exampleProof*(): seq[byte] =
var proof: seq[byte]
while proof.len == 0:

73
tests/integration/codexclient.nim Normal file
View File

@ -0,0 +1,73 @@
import std/httpclient
import std/json
import std/strutils
import pkg/stint
import pkg/questionable/results
type CodexClient* = ref object
http: HttpClient
baseurl: string
proc new*(_: type CodexClient, baseurl: string): CodexClient =
CodexClient(http: newHttpClient(), baseurl: baseurl)
proc info*(client: CodexClient): JsonNode =
let url = client.baseurl & "/debug/info"
client.http.getContent(url).parseJson()
proc setLogLevel*(client: CodexClient, level: string) =
let url = client.baseurl & "/debug/chronicles/loglevel?level=" & level
let headers = newHttpHeaders({"Content-Type": "text/plain"})
let response = client.http.request(url, httpMethod=HttpPost, headers=headers)
assert response.status == "200 OK"
proc upload*(client: CodexClient, contents: string): string =
let response = client.http.post(client.baseurl & "/upload", contents)
assert response.status == "200 OK"
response.body
proc requestStorage*(client: CodexClient,
cid: string,
duration: uint64,
reward: uint64,
proofProbability: uint64,
expiry: UInt256): string =
let url = client.baseurl & "/storage/request/" & cid
let json = %*{
"duration": "0x" & duration.toHex,
"reward": "0x" & reward.toHex,
"proofProbability": "0x" & proofProbability.toHex,
"expiry": "0x" & expiry.toHex
}
let response = client.http.post(url, $json)
assert response.status == "200 OK"
response.body
proc getPurchase*(client: CodexClient, purchase: string): JsonNode =
let url = client.baseurl & "/storage/purchases/" & purchase
let body = client.http.getContent(url)
parseJson(body).catch |? nil
proc postAvailability*(client: CodexClient,
size, duration, minPrice: uint64): JsonNode =
let url = client.baseurl & "/sales/availability"
let json = %*{
"size": "0x" & size.toHex,
"duration": "0x" & duration.toHex,
"minPrice": "0x" & minPrice.toHex
}
let response = client.http.post(url, $json)
assert response.status == "200 OK"
parseJson(response.body)
proc getAvailabilities*(client: CodexClient): JsonNode =
let url = client.baseurl & "/sales/availability"
let body = client.http.getContent(url)
parseJson(body)
proc close*(client: CodexClient) =
client.http.close()
proc restart*(client: CodexClient) =
client.http.close()
client.http = newHttpClient()

165
tests/integration/testIntegration.nim
View File

@ -1,156 +1,75 @@
import std/os
import std/httpclient
import std/json
import std/strutils
import pkg/chronos
import ../ethertest
import ../contracts/time
import ../codex/helpers/eventually
import ./nodes
import ./twonodes
import ./tokens
ethersuite "Integration tests":
var node1, node2: NodeProcess
var baseurl1, baseurl2: string
var client: HttpClient
let dataDir1 = getTempDir() / "Codex1"
let dataDir2 = getTempDir() / "Codex2"
twonodessuite "Integration tests", debug1 = false, debug2 = false:
setup:
await provider.getSigner(accounts[0]).mint()
await provider.getSigner(accounts[1]).mint()
await provider.getSigner(accounts[1]).deposit()
baseurl1 = "http://localhost:8080/api/codex/v1"
baseurl2 = "http://localhost:8081/api/codex/v1"
client = newHttpClient()
node1 = startNode([
"--api-port=8080",
"--data-dir=" & dataDir1,
"--nat=127.0.0.1",
"--disc-ip=127.0.0.1",
"--disc-port=8090",
"--persistence",
"--eth-account=" & $accounts[0]
], debug = false)
let
bootstrap = strip(
$(parseJson(client.get(baseurl1 & "/debug/info").body)["spr"]),
chars = {'"'})
node2 = startNode([
"--api-port=8081",
"--data-dir=" & dataDir2,
"--nat=127.0.0.1",
"--disc-ip=127.0.0.1",
"--disc-port=8091",
"--bootstrap-node=" & bootstrap,
"--persistence",
"--eth-account=" & $accounts[1]
], debug = false)
teardown:
client.close()
node1.stop()
node2.stop()
dataDir1.removeDir()
dataDir2.removeDir()
test "nodes can print their peer information":
let info1 = client.get(baseurl1 & "/debug/info").body
let info2 = client.get(baseurl2 & "/debug/info").body
check info1 != info2
check client1.info() != client2.info()
test "nodes should set chronicles log level":
client.headers = newHttpHeaders({ "Content-Type": "text/plain" })
let filter = "/debug/chronicles/loglevel?level=DEBUG;TRACE:codex"
check client.request(baseurl1 & filter, httpMethod = HttpPost, body = "").status == "200 OK"
test "nodes can set chronicles log level":
client1.setLogLevel("DEBUG;TRACE:codex")
test "node accepts file uploads":
let url = baseurl1 & "/upload"
let response = client.post(url, "some file contents")
check response.status == "200 OK"
let cid1 = client1.upload("some file contents")
let cid2 = client1.upload("some other contents")
check cid1 != cid2
test "node handles new storage availability":
let url = baseurl1 & "/sales/availability"
let json = %*{"size": "0x1", "duration": "0x2", "minPrice": "0x3"}
check client.post(url, $json).status == "200 OK"
let availability1 = client1.postAvailability(size=1, duration=2, minPrice=3)
let availability2 = client1.postAvailability(size=4, duration=5, minPrice=6)
check availability1 != availability2
test "node lists storage that is for sale":
let url = baseurl1 & "/sales/availability"
let json = %*{"size": "0x1", "duration": "0x2", "minPrice": "0x3"}
let availability = parseJson(client.post(url, $json).body)
let response = client.get(url)
check response.status == "200 OK"
check %*availability in parseJson(response.body)
let availability = client1.postAvailability(size=1, duration=2, minPrice=3)
check availability in client1.getAvailabilities()
test "node handles storage request":
let cid = client.post(baseurl1 & "/upload", "some file contents").body
let url = baseurl1 & "/storage/request/" & cid
let json = %*{"duration": "0x1", "reward": "0x2"}
let response = client.post(url, $json)
check response.status == "200 OK"
let expiry = (await provider.currentTime()) + 30
let cid = client1.upload("some file contents")
let id1 = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry)
let id2 = client1.requestStorage(cid, duration=4, reward=5, proofProbability=6, expiry=expiry)
check id1 != id2
test "node retrieves purchase status":
let cid = client.post(baseurl1 & "/upload", "some file contents").body
let request = %*{"duration": "0x1", "reward": "0x2"}
let id = client.post(baseurl1 & "/storage/request/" & cid, $request).body
let response = client.get(baseurl1 & "/storage/purchases/" & id)
check response.status == "200 OK"
let json = parseJson(response.body)
check json["request"]["ask"]["duration"].getStr == "0x1"
check json["request"]["ask"]["reward"].getStr == "0x2"
let expiry = (await provider.currentTime()) + 30
let cid = client1.upload("some file contents")
let id = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry)
let purchase = client1.getPurchase(id)
check purchase{"request"}{"ask"}{"duration"} == %"0x1"
check purchase{"request"}{"ask"}{"reward"} == %"0x2"
check purchase{"request"}{"ask"}{"proofProbability"} == %"0x3"
test "node remembers purchase status after restart":
let cid = client.post(baseurl1 & "/upload", "some file contents").body
let request = %*{"duration": "0x1", "reward": "0x2"}
let id = client.post(baseurl1 & "/storage/request/" & cid, $request).body
proc getPurchase(id: string): JsonNode =
let response = client.get(baseurl1 & "/storage/purchases/" & id)
return parseJson(response.body).catch |? nil
check eventually getPurchase(id){"state"}.getStr == "submitted"
let expiry = (await provider.currentTime()) + 30
let cid = client1.upload("some file contents")
let id = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry)
check eventually client1.getPurchase(id){"state"}.getStr() == "submitted"
node1.restart()
client1.restart()
client.close()
client = newHttpClient()
check eventually (not isNil getPurchase(id){"request"}{"ask"})
check getPurchase(id){"request"}{"ask"}{"duration"}.getStr == "0x1"
check getPurchase(id){"request"}{"ask"}{"reward"}.getStr == "0x2"
check eventually (not isNil client1.getPurchase(id){"request"}{"ask"})
check client1.getPurchase(id){"request"}{"ask"}{"duration"} == %"0x1"
check client1.getPurchase(id){"request"}{"ask"}{"reward"} == %"0x2"
test "nodes negotiate contracts on the marketplace":
proc sell =
let json = %*{"size": "0xFFFFF", "duration": "0x200", "minPrice": "0x300"}
discard client.post(baseurl2 & "/sales/availability", $json)
# client 2 makes storage available
discard client2.postAvailability(size=0xFFFFF, duration=200, minPrice=300)
proc available: JsonNode =
client.get(baseurl2 & "/sales/availability").body.parseJson
# client 1 requests storage
let expiry = (await provider.currentTime()) + 30
let cid = client1.upload("some file contents")
let purchase = client1.requestStorage(cid, duration=100, reward=400, proofProbability=3, expiry=expiry)
proc upload: string =
client.post(baseurl1 & "/upload", "some file contents").body
proc buy(cid: string): string =
let expiry = ((waitFor provider.currentTime()) + 30).toHex
let json = %*{"duration": "0x1", "reward": "0x400", "expiry": expiry}
client.post(baseurl1 & "/storage/request/" & cid, $json).body
proc finish(purchase: string): Future[JsonNode] {.async.} =
while true:
let response = client.get(baseurl1 & "/storage/purchases/" & purchase)
let json = parseJson(response.body)
if json["state"].getStr == "finished": return json
await sleepAsync(1.seconds)
sell()
let purchase = waitFor upload().buy().finish()
check purchase["error"].getStr == ""
check available().len == 0
check eventually client1.getPurchase(purchase){"state"} == %"started"
check client1.getPurchase(purchase){"error"} == newJNull()
check client2.getAvailabilities().len == 0

10
tests/integration/testblockexpiration.nim
View File

@ -29,7 +29,7 @@ ethersuite "Node block expiration tests":
"--disc-ip=127.0.0.1",
"--disc-port=8090",
"--block-ttl=" & $blockTtlSeconds,
"--block-mi=3",
"--block-mi=1",
"--block-mn=10"
], debug = false)
@ -49,11 +49,11 @@ ethersuite "Node block expiration tests":
content
test "node retains not-expired file":
startTestNode(blockTtlSeconds = 60 * 60 * 1)
startTestNode(blockTtlSeconds = 10)
let contentId = uploadTestFile()
await sleepAsync(10.seconds)
await sleepAsync(2.seconds)
let response = downloadTestFile(contentId)
check:
@ -61,11 +61,11 @@ ethersuite "Node block expiration tests":
response.body == content
test "node deletes expired file":
startTestNode(blockTtlSeconds = 5)
startTestNode(blockTtlSeconds = 1)
let contentId = uploadTestFile()
await sleepAsync(10.seconds)
await sleepAsync(2.seconds)
expect TimeoutError:
discard downloadTestFile(contentId)

43
tests/integration/testproofs.nim Normal file
View File

@ -0,0 +1,43 @@
import codex/contracts/marketplace
import codex/contracts/deployment
import ../contracts/time
import ../codex/helpers/eventually
import ./twonodes
import ./tokens
twonodessuite "Proving integration test", debug1=false, debug2=false:
var marketplace: Marketplace
var config: MarketplaceConfig
setup:
marketplace = Marketplace.new(!deployment().address(Marketplace), provider)
config = await marketplace.config()
await provider.getSigner(accounts[0]).mint()
await provider.getSigner(accounts[1]).mint()
await provider.getSigner(accounts[1]).deposit()
proc waitUntilPurchaseIsStarted {.async.} =
discard client2.postAvailability(size=0xFFFFF, duration=200, minPrice=300)
let expiry = (await provider.currentTime()) + 30
let cid = client1.upload("some file contents")
let purchase = client1.requestStorage(cid, duration=100, reward=400, proofProbability=3, expiry=expiry)
check eventually client1.getPurchase(purchase){"state"} == %"started"
test "hosts submit periodic proofs for slots they fill":
await waitUntilPurchaseIsStarted()
var proofWasSubmitted = false
proc onProofSubmitted(event: ProofSubmitted) =
proofWasSubmitted = true
let subscription = await marketplace.subscribe(ProofSubmitted, onProofSubmitted)
for _ in 0..<100:
if proofWasSubmitted:
break
else:
await provider.advanceTime(config.proofs.period)
await sleepAsync(1.seconds)
check proofWasSubmitted
await subscription.unsubscribe()

62
tests/integration/twonodes.nim Normal file
View File

@ -0,0 +1,62 @@
import std/os
import std/macros
import std/json
import std/httpclient
import ../ethertest
import ./codexclient
import ./nodes
export ethertest
export codexclient
export nodes
template twonodessuite*(name: string, debug1, debug2: bool, body) =
ethersuite name:
var node1 {.inject, used.}: NodeProcess
var node2 {.inject, used.}: NodeProcess
var client1 {.inject, used.}: CodexClient
var client2 {.inject, used.}: CodexClient
let dataDir1 = getTempDir() / "Codex1"
let dataDir2 = getTempDir() / "Codex2"
setup:
client1 = CodexClient.new("http://localhost:8080/api/codex/v1")
client2 = CodexClient.new("http://localhost:8081/api/codex/v1")
node1 = startNode([
"--api-port=8080",
"--data-dir=" & dataDir1,
"--nat=127.0.0.1",
"--disc-ip=127.0.0.1",
"--disc-port=8090",
"--persistence",
"--eth-account=" & $accounts[0]
], debug = debug1)
let bootstrap = client1.info()["spr"].getStr()
node2 = startNode([
"--api-port=8081",
"--data-dir=" & dataDir2,
"--nat=127.0.0.1",
"--disc-ip=127.0.0.1",
"--disc-port=8091",
"--bootstrap-node=" & bootstrap,
"--persistence",
"--eth-account=" & $accounts[1]
], debug = debug2)
teardown:
client1.close()
client2.close()
node1.stop()
node2.stop()
removeDir(dataDir1)
removeDir(dataDir2)
body

1
tests/testIntegration.nim
View File

@ -1,4 +1,5 @@
import ./integration/testIntegration
import ./integration/testblockexpiration
import ./integration/testproofs
{.warning[UnusedImport]:off.}