status-im
/
nim-codex
mirror of https://github.com/status-im/nim-codex.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into pr-add-prover-benchmark-tool

This commit is contained in:
Jaremy Creechley 2024-05-09 19:15:01 +03:00 committed by GitHub
parent c09c47606d 9379c7c662
commit ae9711a6f7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 194 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
codex/blockexchange/engine/discovery.nim
View File

@ -11,6 +11,7 @@ import std/sequtils
import pkg/chronos
import pkg/libp2p/cid
import pkg/libp2p/multicodec
import pkg/metrics
import pkg/questionable
import pkg/questionable/results
@ -25,6 +26,7 @@ import ../../utils
import ../../discovery
import ../../stores/blockstore
import ../../logutils
import ../../manifest
logScope:
topics = "codex discoveryengine"
@ -76,14 +78,32 @@ proc discoveryQueueLoop(b: DiscoveryEngine) {.async.} =
await sleepAsync(b.discoveryLoopSleep)
proc advertiseBlock(b: DiscoveryEngine, cid: Cid) {.async.} =
without isM =? cid.isManifest, err:
warn "Unable to determine if cid is manifest"
return
if isM:
without blk =? await b.localStore.getBlock(cid), err:
error "Error retrieving manifest block", cid, err = err.msg
return
without manifest =? Manifest.decode(blk), err:
error "Unable to decode as manifest", err = err.msg
return
# announce manifest cid and tree cid
await b.advertiseQueue.put(cid)
await b.advertiseQueue.put(manifest.treeCid)
proc advertiseQueueLoop(b: DiscoveryEngine) {.async.} =
while b.discEngineRunning:
if cids =? await b.localStore.listBlocks(blockType = b.advertiseType):
trace "Begin iterating blocks..."
for c in cids:
if cid =? await c:
await b.advertiseQueue.put(cid)
await sleepAsync(50.millis)
b.advertiseBlock(cid)
await sleepAsync(100.millis)
trace "Iterating blocks finished."
await sleepAsync(b.advertiseLoopSleep)
@ -248,7 +268,7 @@ proc new*(
discoveryLoopSleep = DefaultDiscoveryLoopSleep,
advertiseLoopSleep = DefaultAdvertiseLoopSleep,
minPeersPerBlock = DefaultMinPeersPerBlock,
advertiseType = BlockType.Both
advertiseType = BlockType.Manifest
): DiscoveryEngine =
## Create a discovery engine instance for advertising services
##

5
codex/contracts/market.nim
View File

@ -126,6 +126,11 @@ method getRequestEnd*(market: OnChainMarket,
convertEthersError:
return await market.contract.requestEnd(id)
method requestExpiresAt*(market: OnChainMarket,
id: RequestId): Future[SecondsSince1970] {.async.} =
convertEthersError:
return await market.contract.requestExpiry(id)
method getHost(market: OnChainMarket,
requestId: RequestId,
slotIndex: UInt256): Future[?Address] {.async.} =

1
codex/contracts/marketplace.nim
View File

@ -55,6 +55,7 @@ proc mySlots*(marketplace: Marketplace): seq[SlotId] {.contract, view.}
proc requestState*(marketplace: Marketplace, requestId: RequestId): RequestState {.contract, view.}
proc slotState*(marketplace: Marketplace, slotId: SlotId): SlotState {.contract, view.}
proc requestEnd*(marketplace: Marketplace, requestId: RequestId): SecondsSince1970 {.contract, view.}
proc requestExpiry*(marketplace: Marketplace, requestId: RequestId): SecondsSince1970 {.contract, view.}
proc proofTimeout*(marketplace: Marketplace): UInt256 {.contract, view.}

4
codex/market.nim
View File

@ -84,6 +84,10 @@ method getRequestEnd*(market: Market,
id: RequestId): Future[SecondsSince1970] {.base, async.} =
raiseAssert("not implemented")
method requestExpiresAt*(market: Market,
id: RequestId): Future[SecondsSince1970] {.base, async.} =
raiseAssert("not implemented")
method getHost*(market: Market,
requestId: RequestId,
slotIndex: UInt256): Future[?Address] {.base, async.} =

5
codex/purchasing.nim
View File

@ -18,18 +18,15 @@ type
clock: Clock
purchases: Table[PurchaseId, Purchase]
proofProbability*: UInt256
requestExpiryInterval*: UInt256
PurchaseTimeout* = Timeout
const DefaultProofProbability = 100.u256
const DefaultRequestExpiryInterval = (10 * 60).u256
proc new*(_: type Purchasing, market: Market, clock: Clock): Purchasing =
Purchasing(
market: market,
clock: clock,
proofProbability: DefaultProofProbability,
requestExpiryInterval: DefaultRequestExpiryInterval,
)
proc load*(purchasing: Purchasing) {.async.} =
@ -52,8 +49,6 @@ proc populate*(purchasing: Purchasing,
result = request
if result.ask.proofProbability == 0.u256:
result.ask.proofProbability = purchasing.proofProbability
if result.expiry == 0.u256:
result.expiry = (purchasing.clock.now().u256 + purchasing.requestExpiryInterval)
if result.nonce == Nonce.default:
var id = result.nonce.toArray
doAssert randomBytes(id) == 32

2
codex/purchasing/states/submitted.nim
View File

@ -34,7 +34,7 @@ method run*(state: PurchaseSubmitted, machine: Machine): Future[?State] {.async.
await subscription.unsubscribe()
proc withTimeout(future: Future[void]) {.async.} =
let expiry = request.expiry.truncate(int64) + 1
let expiry = (await market.requestExpiresAt(request.id)) + 1
trace "waiting for request fulfillment or expiry", expiry
await future.withTimeout(clock, expiry)

13
codex/rest/api.nim
View File

@ -399,7 +399,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
## duration - the duration of the request in seconds
## proofProbability - how often storage proofs are required
## reward - the maximum amount of tokens paid per second per slot to hosts the client is willing to pay
## expiry - timestamp, in seconds, when the request expires if the Request does not find requested amount of nodes to host the data
## expiry - specifies threshold in seconds from now when the request expires if the Request does not find requested amount of nodes to host the data
## nodes - number of nodes the content should be stored on
## tolerance - allowed number of nodes that can be lost before content is lost
## colateral - requested collateral from hosts when they fill slot
@ -425,15 +425,8 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) =
without expiry =? params.expiry:
return RestApiResponse.error(Http400, "Expiry required")
if node.clock.isNil:
return RestApiResponse.error(Http500)
if expiry <= node.clock.now.u256:
return RestApiResponse.error(Http400, "Expiry needs to be in future. Now: " & $node.clock.now)
let expiryLimit = node.clock.now.u256 + params.duration
if expiry > expiryLimit:
return RestApiResponse.error(Http400, "Expiry has to be before the request's end (now + duration). Limit: " & $expiryLimit)
if expiry <= 0 or expiry >= params.duration:
return RestApiResponse.error(Http400, "Expiry needs value bigger then zero and smaller then the request's duration")
without purchaseId =? await node.requestStorage(
cid,

5
codex/sales/salesagent.nim
View File

@ -72,8 +72,11 @@ proc subscribeCancellation(agent: SalesAgent) {.async.} =
without request =? data.request:
return
let market = agent.context.market
let expiry = await market.requestExpiresAt(data.requestId)
while true:
let deadline = max(clock.now, request.expiry.truncate(int64)) + 1
let deadline = max(clock.now, expiry) + 1
trace "Waiting for request to be cancelled", now=clock.now, expiry=deadline
await clock.waitUntil(deadline)

3
openapi.yaml
View File

@ -213,8 +213,7 @@ components:
description: Number as decimal string that represents how much collateral is asked from hosts that wants to fill a slots
expiry:
type: string
description: Number as decimal string that represents expiry time of the request (in unix timestamp)
description: Number as decimal string that represents expiry threshold in seconds from when the Request is submitted. When the threshold is reached and the Request does not find requested amount of nodes to host the data, the Request is voided. The number of seconds can not be higher then the Request's duration itself.
StorageAsk:
type: object
required:

126
tests/codex/blockexchange/discovery/testdiscovery.nim
View File

@ -50,7 +50,7 @@ asyncchecksuite "Block Advertising and Discovery":
blockDiscovery = MockDiscovery.new()
wallet = WalletRef.example
network = BlockExcNetwork.new(switch)
localStore = CacheStore.new(blocks.mapIt( it ))
localStore = CacheStore.new(blocks.mapIt(it))
peerStore = PeerCtxStore.new()
pendingBlocks = PendingBlocksManager.new()
@ -92,57 +92,40 @@ asyncchecksuite "Block Advertising and Discovery":
blockDiscovery.findBlockProvidersHandler =
proc(d: MockDiscovery, cid: Cid): Future[seq[SignedPeerRecord]] {.async.} =
await engine.resolveBlocks(blocks.filterIt( it.cid == cid ))
await engine.resolveBlocks(blocks.filterIt(it.cid == cid))
await allFuturesThrowing(
allFinished(pendingBlocks))
await engine.stop()
test "Should advertise both manifests and blocks":
test "Should advertise both manifests and trees":
let
cids = @[manifest.cid.tryGet, manifest.treeCid]
advertised = initTable.collect:
for b in (blocks & manifestBlock): {b.cid: newFuture[void]()}
for cid in cids: {cid: newFuture[void]()}
blockDiscovery
.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async.} =
if cid in advertised and not advertised[cid].finished():
advertised[cid].complete()
discovery.advertiseType = BlockType.Both
await engine.start() # fire up advertise loop
await engine.start()
await allFuturesThrowing(
allFinished(toSeq(advertised.values)))
await engine.stop()
test "Should advertise local manifests":
test "Should not advertise local blocks":
let
advertised = newFuture[Cid]()
blockCids = blocks.mapIt(it.cid)
blockDiscovery
.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async.} =
check manifestBlock.cid == cid
advertised.complete(cid)
check:
cid notin blockCids
discovery.advertiseType = BlockType.Manifest
await engine.start() # fire up advertise loop
check (await advertised.wait(10.millis)) == manifestBlock.cid
await engine.stop()
test "Should advertise local blocks":
let
advertised = initTable.collect:
for b in blocks: {b.cid: newFuture[void]()}
blockDiscovery
.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid) {.async.} =
if cid in advertised and not advertised[cid].finished():
advertised[cid].complete()
discovery.advertiseType = BlockType.Block
await engine.start() # fire up advertise loop
await allFuturesThrowing(
allFinished(toSeq(advertised.values)))
await engine.start()
await sleepAsync(3.seconds)
await engine.stop()
test "Should not launch discovery if remote peer has block":
@ -165,7 +148,7 @@ asyncchecksuite "Block Advertising and Discovery":
proc(d: MockDiscovery, cid: Cid): Future[seq[SignedPeerRecord]] =
check false
await engine.start() # fire up discovery loop
await engine.start()
engine.pendingBlocks.resolve(blocks.mapIt(BlockDelivery(blk: it, address: it.address)))
await allFuturesThrowing(
@ -173,23 +156,33 @@ asyncchecksuite "Block Advertising and Discovery":
await engine.stop()
asyncchecksuite "E2E - Multiple Nodes Discovery":
let chunker = RandomChunker.new(Rng.instance(), size = 4096, chunkSize = 256)
proc asBlock(m: Manifest): bt.Block =
let mdata = m.encode().tryGet()
bt.Block.new(data = mdata, codec = ManifestCodec).tryGet()
asyncchecksuite "E2E - Multiple Nodes Discovery":
var
switch: seq[Switch]
blockexc: seq[NetworkStore]
blocks: seq[bt.Block]
manifests: seq[Manifest]
mBlocks: seq[bt.Block]
trees: seq[CodexTree]
setup:
while true:
let chunk = await chunker.getBytes()
if chunk.len <= 0:
break
blocks.add(bt.Block.new(chunk).tryGet())
for _ in 0..<4:
let chunker = RandomChunker.new(Rng.instance(), size = 4096, chunkSize = 256)
var blocks = newSeq[bt.Block]()
while true:
let chunk = await chunker.getBytes()
if chunk.len <= 0:
break
blocks.add(bt.Block.new(chunk).tryGet())
let (manifest, tree) = makeManifestAndTree(blocks).tryGet()
manifests.add(manifest)
mBlocks.add(manifest.asBlock())
trees.add(tree)
let
s = newStandardSwitch(transportFlags = {ServerFlags.ReuseAddr})
blockDiscovery = MockDiscovery.new()
@ -223,9 +216,12 @@ asyncchecksuite "E2E - Multiple Nodes Discovery":
teardown:
switch = @[]
blockexc = @[]
manifests = @[]
mBlocks = @[]
trees = @[]
test "E2E - Should advertise and discover blocks":
# Distribute the blocks amongst 1..3
# Distribute the manifests and trees amongst 1..3
# Ask 0 to download everything without connecting him beforehand
var advertised: Table[Cid, SignedPeerRecord]
@ -242,14 +238,14 @@ asyncchecksuite "E2E - Multiple Nodes Discovery":
.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} =
advertised[cid] = switch[3].peerInfo.signedPeerRecord
discard blocks[0..5].mapIt(blockexc[1].engine.pendingBlocks.getWantHandle(it.address))
await blockexc[1].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, blocks[0..5].mapIt(BlockDelivery(blk: it, address: it.address)))
discard blockexc[1].engine.pendingBlocks.getWantHandle(mBlocks[0].cid)
await blockexc[1].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, @[BlockDelivery(blk: mBlocks[0], address: BlockAddress(leaf: false, cid: mBlocks[0].cid))])
discard blocks[4..10].mapIt(blockexc[2].engine.pendingBlocks.getWantHandle(it.address))
await blockexc[2].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, blocks[4..10].mapIt(BlockDelivery(blk: it, address: it.address)))
discard blockexc[2].engine.pendingBlocks.getWantHandle(mBlocks[1].cid)
await blockexc[2].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, @[BlockDelivery(blk: mBlocks[1], address: BlockAddress(leaf: false, cid: mBlocks[1].cid))])
discard blocks[10..15].mapIt(blockexc[3].engine.pendingBlocks.getWantHandle(it.address))
await blockexc[3].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, blocks[10..15].mapIt(BlockDelivery(blk: it, address: it.address)))
discard blockexc[3].engine.pendingBlocks.getWantHandle(mBlocks[2].cid)
await blockexc[3].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, @[BlockDelivery(blk: mBlocks[2], address: BlockAddress(leaf: false, cid: mBlocks[2].cid))])
MockDiscovery(blockexc[0].engine.discovery.discovery)
.findBlockProvidersHandler = proc(d: MockDiscovery, cid: Cid):
@ -258,22 +254,22 @@ asyncchecksuite "E2E - Multiple Nodes Discovery":
result.add(advertised[cid])
let futs = collect(newSeq):
for b in blocks:
blockexc[0].engine.requestBlock(b.cid)
for m in mBlocks[0..2]:
blockexc[0].engine.requestBlock(m.cid)
await allFuturesThrowing(
switch.mapIt( it.start() ) &
blockexc.mapIt( it.engine.start() )).wait(10.seconds)
switch.mapIt(it.start()) &
blockexc.mapIt(it.engine.start())).wait(10.seconds)
await allFutures(futs).wait(10.seconds)
await allFuturesThrowing(
blockexc.mapIt( it.engine.stop() ) &
switch.mapIt( it.stop() )).wait(10.seconds)
blockexc.mapIt(it.engine.stop()) &
switch.mapIt(it.stop())).wait(10.seconds)
test "E2E - Should advertise and discover blocks with peers already connected":
# Distribute the blocks amongst 1..3
# Ask 0 to download everything without connecting him beforehand
# Ask 0 to download everything *WITH* connecting him beforehand
var advertised: Table[Cid, SignedPeerRecord]
@ -289,14 +285,14 @@ asyncchecksuite "E2E - Multiple Nodes Discovery":
.publishBlockProvideHandler = proc(d: MockDiscovery, cid: Cid): Future[void] {.async.} =
advertised[cid] = switch[3].peerInfo.signedPeerRecord
discard blocks[0..5].mapIt(blockexc[1].engine.pendingBlocks.getWantHandle(it.address))
await blockexc[1].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, blocks[0..5].mapIt(BlockDelivery(blk: it, address: it.address)))
discard blockexc[1].engine.pendingBlocks.getWantHandle(mBlocks[0].cid)
await blockexc[1].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, @[BlockDelivery(blk: mBlocks[0], address: BlockAddress(leaf: false, cid: mBlocks[0].cid))])
discard blocks[4..10].mapIt(blockexc[2].engine.pendingBlocks.getWantHandle(it.address))
await blockexc[2].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, blocks[4..10].mapIt(BlockDelivery(blk: it, address: it.address)))
discard blockexc[2].engine.pendingBlocks.getWantHandle(mBlocks[1].cid)
await blockexc[2].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, @[BlockDelivery(blk: mBlocks[1], address: BlockAddress(leaf: false, cid: mBlocks[1].cid))])
discard blocks[10..15].mapIt(blockexc[3].engine.pendingBlocks.getWantHandle(it.address))
await blockexc[3].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, blocks[10..15].mapIt(BlockDelivery(blk: it, address: it.address)))
discard blockexc[3].engine.pendingBlocks.getWantHandle(mBlocks[2].cid)
await blockexc[3].engine.blocksDeliveryHandler(switch[0].peerInfo.peerId, @[BlockDelivery(blk: mBlocks[2], address: BlockAddress(leaf: false, cid: mBlocks[2].cid))])
MockDiscovery(blockexc[0].engine.discovery.discovery)
.findBlockProvidersHandler = proc(d: MockDiscovery, cid: Cid):
@ -305,14 +301,14 @@ asyncchecksuite "E2E - Multiple Nodes Discovery":
return @[advertised[cid]]
let
futs = blocks.mapIt( blockexc[0].engine.requestBlock( it.cid ) )
futs = mBlocks[0..2].mapIt(blockexc[0].engine.requestBlock(it.cid))
await allFuturesThrowing(
switch.mapIt( it.start() ) &
blockexc.mapIt( it.engine.start() )).wait(10.seconds)
switch.mapIt(it.start()) &
blockexc.mapIt(it.engine.start())).wait(10.seconds)
await allFutures(futs).wait(10.seconds)
await allFuturesThrowing(
blockexc.mapIt( it.engine.stop() ) &
switch.mapIt( it.stop() )).wait(10.seconds)
blockexc.mapIt(it.engine.stop()) &
switch.mapIt(it.stop())).wait(10.seconds)

45
tests/codex/blockexchange/discovery/testdiscoveryengine.nim
View File

@ -10,17 +10,26 @@ import pkg/codex/blockexchange
import pkg/codex/chunker
import pkg/codex/blocktype as bt
import pkg/codex/blockexchange/engine
import pkg/codex/manifest
import pkg/codex/merkletree
import ../../../asynctest
import ../../helpers
import ../../helpers/mockdiscovery
import ../../examples
proc asBlock(m: Manifest): bt.Block =
let mdata = m.encode().tryGet()
bt.Block.new(data = mdata, codec = ManifestCodec).tryGet()
asyncchecksuite "Test Discovery Engine":
let chunker = RandomChunker.new(Rng.instance(), size = 4096, chunkSize = 256)
var
blocks: seq[bt.Block]
manifest: Manifest
tree: CodexTree
manifestBlock: bt.Block
switch: Switch
peerStore: PeerCtxStore
blockDiscovery: MockDiscovery
@ -35,6 +44,10 @@ asyncchecksuite "Test Discovery Engine":
blocks.add(bt.Block.new(chunk).tryGet())
(manifest, tree) = makeManifestAndTree(blocks).tryGet()
manifestBlock = manifest.asBlock()
blocks.add(manifestBlock)
switch = newStandardSwitch(transportFlags = {ServerFlags.ReuseAddr})
network = BlockExcNetwork.new(switch)
peerStore = PeerCtxStore.new()
@ -51,11 +64,11 @@ asyncchecksuite "Test Discovery Engine":
blockDiscovery,
pendingBlocks,
discoveryLoopSleep = 100.millis)
wants = blocks.mapIt( pendingBlocks.getWantHandle(it.cid) )
wants = blocks.mapIt(pendingBlocks.getWantHandle(it.cid) )
blockDiscovery.findBlockProvidersHandler =
proc(d: MockDiscovery, cid: Cid): Future[seq[SignedPeerRecord]] {.async, gcsafe.} =
pendingBlocks.resolve(blocks.filterIt( it.cid == cid).mapIt(BlockDelivery(blk: it, address: it.address)))
pendingBlocks.resolve(blocks.filterIt(it.cid == cid).mapIt(BlockDelivery(blk: it, address: it.address)))
await discoveryEngine.start()
await allFuturesThrowing(allFinished(wants)).wait(1.seconds)
@ -63,7 +76,7 @@ asyncchecksuite "Test Discovery Engine":
test "Should Advertise Haves":
var
localStore = CacheStore.new(blocks.mapIt( it ))
localStore = CacheStore.new(blocks.mapIt(it))
discoveryEngine = DiscoveryEngine.new(
localStore,
peerStore,
@ -72,8 +85,8 @@ asyncchecksuite "Test Discovery Engine":
pendingBlocks,
discoveryLoopSleep = 100.millis)
haves = collect(initTable):
for b in blocks:
{ b.cid: newFuture[void]() }
for cid in @[manifestBlock.cid, manifest.treeCid]:
{ cid: newFuture[void]() }
blockDiscovery.publishBlockProvideHandler =
proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} =
@ -108,28 +121,6 @@ asyncchecksuite "Test Discovery Engine":
await want.wait(1.seconds)
await discoveryEngine.stop()
test "Should queue advertise request":
var
localStore = CacheStore.new(@[blocks[0]])
discoveryEngine = DiscoveryEngine.new(
localStore,
peerStore,
network,
blockDiscovery,
pendingBlocks,
discoveryLoopSleep = 100.millis)
have = newFuture[void]()
blockDiscovery.publishBlockProvideHandler =
proc(d: MockDiscovery, cid: Cid) {.async, gcsafe.} =
check cid == blocks[0].cid
if not have.finished:
have.complete()
await discoveryEngine.start()
await have.wait(1.seconds)
await discoveryEngine.stop()
test "Should not request more than minPeersPerBlock":
var
localStore = CacheStore.new()

5
tests/codex/helpers/mockmarket.nim
View File

@ -20,6 +20,7 @@ type
activeSlots*: Table[Address, seq[SlotId]]
requested*: seq[StorageRequest]
requestEnds*: Table[RequestId, SecondsSince1970]
requestExpiry*: Table[RequestId, SecondsSince1970]
requestState*: Table[RequestId, RequestState]
slotState*: Table[SlotId, SlotState]
fulfilled*: seq[Fulfillment]
@ -165,6 +166,10 @@ method getRequestEnd*(market: MockMarket,
id: RequestId): Future[SecondsSince1970] {.async.} =
return market.requestEnds[id]
method requestExpiresAt*(market: MockMarket,
id: RequestId): Future[SecondsSince1970] {.async.} =
return market.requestExpiry[id]
method getHost*(market: MockMarket,
requestId: RequestId,
slotIndex: UInt256): Future[?Address] {.async.} =

10
tests/codex/sales/testsales.nim
View File

@ -473,6 +473,9 @@ asyncchecksuite "Sales":
check eventually (await reservations.all(Availability)).get == @[availability]
test "makes storage available again when request expires":
let expiry = getTime().toUnix() + 10
market.requestExpiry[request.id] = expiry
let origSize = availability.freeSize
sales.onStore = proc(request: StorageRequest,
slot: UInt256,
@ -486,11 +489,14 @@ asyncchecksuite "Sales":
# would otherwise not set the timeout early enough as it uses `clock.now` in the deadline calculation.
await sleepAsync(chronos.milliseconds(100))
market.requestState[request.id]=RequestState.Cancelled
clock.set(request.expiry.truncate(int64)+1)
clock.set(expiry + 1)
check eventually (await reservations.all(Availability)).get == @[availability]
check getAvailability().freeSize == origSize
test "verifies that request is indeed expired from onchain before firing onCancelled":
let expiry = getTime().toUnix() + 10
market.requestExpiry[request.id] = expiry
let origSize = availability.freeSize
sales.onStore = proc(request: StorageRequest,
slot: UInt256,
@ -504,7 +510,7 @@ asyncchecksuite "Sales":
# If we would not await, then the `clock.set` would run "too fast" as the `subscribeCancellation()`
# would otherwise not set the timeout early enough as it uses `clock.now` in the deadline calculation.
await sleepAsync(chronos.milliseconds(100))
clock.set(request.expiry.truncate(int64)+1)
clock.set(expiry + 1)
check getAvailability().freeSize == 0
market.requestState[request.id]=RequestState.Cancelled # Now "on-chain" is also expired

21
tests/codex/sales/testsalesagent.nim
View File

@ -41,19 +41,7 @@ method run*(state: MockErrorState, machine: Machine): Future[?State] {.async.} =
raise newException(ValueError, "failure")
asyncchecksuite "Sales agent":
var request = StorageRequest(
ask: StorageAsk(
slots: 4,
slotSize: 100.u256,
duration: 60.u256,
reward: 10.u256,
),
content: StorageContent(
cid: "some cid"
),
expiry: (getTime() + initDuration(hours=1)).toUnix.u256
)
let request = StorageRequest.example
var agent: SalesAgent
var context: SalesContext
var slotIndex: UInt256
@ -62,6 +50,7 @@ asyncchecksuite "Sales agent":
setup:
market = MockMarket.new()
market.requestExpiry[request.id] = getTime().toUnix() + request.expiry.truncate(int64)
clock = MockClock.new()
context = SalesContext(market: market, clock: clock)
slotIndex = 0.u256
@ -109,7 +98,7 @@ asyncchecksuite "Sales agent":
agent.start(MockState.new())
await agent.subscribe()
market.requestState[request.id] = RequestState.Cancelled
clock.set(request.expiry.truncate(int64) + 1)
clock.set(market.requestExpiry[request.id] + 1)
check eventually onCancelCalled
for requestState in {RequestState.New, Started, Finished, Failed}:
@ -117,7 +106,7 @@ asyncchecksuite "Sales agent":
agent.start(MockState.new())
await agent.subscribe()
market.requestState[request.id] = requestState
clock.set(request.expiry.truncate(int64) + 1)
clock.set(market.requestExpiry[request.id] + 1)
await sleepAsync(100.millis)
check not onCancelCalled
@ -126,7 +115,7 @@ asyncchecksuite "Sales agent":
agent.start(MockState.new())
await agent.subscribe()
market.requestState[request.id] = requestState
clock.set(request.expiry.truncate(int64) + 1)
clock.set(market.requestExpiry[request.id] + 1)
check eventually agent.data.cancelled.finished
test "cancelled future is finished (cancelled) when onFulfilled called":

43
tests/codex/testpurchasing.nim
View File

@ -19,7 +19,7 @@ asyncchecksuite "Purchasing":
var purchasing: Purchasing
var market: MockMarket
var clock: MockClock
var request: StorageRequest
var request, populatedRequest: StorageRequest
setup:
market = MockMarket.new()
@ -34,6 +34,12 @@ asyncchecksuite "Purchasing":
)
)
# We need request which has stable ID during the whole Purchasing pipeline
# for some tests (related to expiry). Because of Purchasing.populate() we need
# to do the steps bellow.
populatedRequest = StorageRequest.example
populatedRequest.client = await market.getSigner()
test "submits a storage request when asked":
discard await purchasing.purchase(request)
check eventually market.requested.len > 0
@ -63,23 +69,6 @@ asyncchecksuite "Purchasing":
check eventually market.requested.len > 0
check market.requested[0].ask.proofProbability == 42.u256
test "has a default value for request expiration interval":
check purchasing.requestExpiryInterval != 0.u256
test "can change default value for request expiration interval":
purchasing.requestExpiryInterval = 42.u256
let start = getTime().toUnix()
discard await purchasing.purchase(request)
check eventually market.requested.len > 0
check market.requested[0].expiry == (start + 42).u256
test "can override expiry time per request":
let expiry = (getTime().toUnix() + 42).u256
request.expiry = expiry
discard await purchasing.purchase(request)
check eventually market.requested.len > 0
check market.requested[0].expiry == expiry
test "includes a random nonce in every storage request":
discard await purchasing.purchase(request)
discard await purchasing.purchase(request)
@ -92,29 +81,37 @@ asyncchecksuite "Purchasing":
check market.requested[0].client == await market.getSigner()
test "succeeds when request is finished":
let purchase = await purchasing.purchase(request)
market.requestExpiry[populatedRequest.id] = getTime().toUnix() + 10
let purchase = await purchasing.purchase(populatedRequest)
check eventually market.requested.len > 0
let request = market.requested[0]
let requestEnd = getTime().toUnix() + 42
market.requestEnds[request.id] = requestEnd
market.emitRequestFulfilled(request.id)
clock.set(requestEnd + 1)
await purchase.wait()
check purchase.error.isNone
test "fails when request times out":
let purchase = await purchasing.purchase(request)
let expiry = getTime().toUnix() + 10
market.requestExpiry[populatedRequest.id] = expiry
let purchase = await purchasing.purchase(populatedRequest)
check eventually market.requested.len > 0
let request = market.requested[0]
clock.set(request.expiry.truncate(int64) + 1)
clock.set(expiry + 1)
expect PurchaseTimeout:
await purchase.wait()
test "checks that funds were withdrawn when purchase times out":
let purchase = await purchasing.purchase(request)
let expiry = getTime().toUnix() + 10
market.requestExpiry[populatedRequest.id] = expiry
let purchase = await purchasing.purchase(populatedRequest)
check eventually market.requested.len > 0
let request = market.requested[0]
clock.set(request.expiry.truncate(int64) + 1)
clock.set(expiry + 1)
expect PurchaseTimeout:
await purchase.wait()
check market.withdrawn == @[request.id]

3
tests/contracts/testContracts.nim
View File

@ -85,7 +85,8 @@ ethersuite "Marketplace contracts":
check endBalance == (startBalance + request.ask.duration * request.ask.reward + request.ask.collateral)
test "cannot mark proofs missing for cancelled request":
await ethProvider.advanceTimeTo(request.expiry + 1)
let expiry = await marketplace.requestExpiry(request.id)
await ethProvider.advanceTimeTo((expiry + 1).u256)
switchAccount(client)
let missingPeriod = periodicity.periodOf(await ethProvider.currentTime())
await ethProvider.advanceTime(periodicity.seconds)

26
tests/contracts/testMarket.nim
View File

@ -32,6 +32,10 @@ ethersuite "On-Chain Market":
let currentPeriod = periodicity.periodOf(await ethProvider.currentTime())
await ethProvider.advanceTimeTo(periodicity.periodEnd(currentPeriod) + 1)
proc advanceToCancelledRequest(request: StorageRequest) {.async.} =
let expiry = (await market.requestExpiresAt(request.id)) + 1
await ethProvider.advanceTimeTo(expiry.u256)
proc waitUntilProofRequired(slotId: SlotId) {.async.} =
await advanceToNextPeriod()
while not (
@ -70,22 +74,19 @@ ethersuite "On-Chain Market":
test "supports withdrawing of funds":
await market.requestStorage(request)
await ethProvider.advanceTimeTo(request.expiry + 1)
await advanceToCancelledRequest(request)
await market.withdrawFunds(request.id)
test "supports request subscriptions":
var receivedIds: seq[RequestId]
var receivedAsks: seq[StorageAsk]
var receivedExpirys: seq[UInt256]
proc onRequest(id: RequestId, ask: StorageAsk, expiry: UInt256) =
receivedIds.add(id)
receivedAsks.add(ask)
receivedExpirys.add(expiry)
let subscription = await market.subscribeRequests(onRequest)
await market.requestStorage(request)
check receivedIds == @[request.id]
check receivedAsks == @[request.ask]
check receivedExpirys == @[request.expiry]
await subscription.unsubscribe()
test "supports filling of slots":
@ -216,7 +217,7 @@ ethersuite "On-Chain Market":
receivedIds.add(id)
let subscription = await market.subscribeRequestCancelled(request.id, onRequestCancelled)
await ethProvider.advanceTimeTo(request.expiry + 1)
await advanceToCancelledRequest(request)
await market.withdrawFunds(request.id)
check receivedIds == @[request.id]
await subscription.unsubscribe()
@ -255,7 +256,7 @@ ethersuite "On-Chain Market":
receivedIds.add(requestId)
let subscription = await market.subscribeRequestCancelled(request.id, onRequestCancelled)
await ethProvider.advanceTimeTo(request.expiry + 1) # shares expiry with otherRequest
advanceToCancelledRequest(otherRequest) # shares expiry with otherRequest
await market.withdrawFunds(otherRequest.id)
check receivedIds.len == 0
await market.withdrawFunds(request.id)
@ -338,13 +339,12 @@ ethersuite "On-Chain Market":
# 6 blocks, we only need to check 5 "blocks ago". We don't need to check the
# `approve` for the first `requestStorage` call, so that's 1 less again = 4
# "blocks ago".
check eventually (
(await market.queryPastStorageRequests(5)) ==
@[
PastStorageRequest(requestId: request.id, ask: request.ask, expiry: request.expiry),
PastStorageRequest(requestId: request1.id, ask: request1.ask, expiry: request1.expiry),
PastStorageRequest(requestId: request2.id, ask: request2.ask, expiry: request2.expiry)
])
proc getsPastRequest(): Future[bool] {.async.} =
let reqs = await market.queryPastStorageRequests(5)
reqs.mapIt(it.requestId) == @[request.id, request1.id, request2.id]
check eventually await getsPastRequest()
test "past event query can specify negative `blocksAgo` parameter":
await market.requestStorage(request)

2
tests/examples.nim
View File

@ -59,7 +59,7 @@ proc example*(_: type StorageRequest): StorageRequest =
cid: "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob",
merkleRoot: array[32, byte].example
),
expiry: (getTime() + 1.hours).toUnix.u256,
expiry:(60 * 60).u256, # 1 hour ,
nonce: Nonce.example
)

6
tests/integration/codexclient.nim
View File

@ -70,7 +70,7 @@ proc requestStorageRaw*(
reward: UInt256,
proofProbability: UInt256,
collateral: UInt256,
expiry: UInt256 = 0.u256,
expiry: uint = 0,
nodes: uint = 1,
tolerance: uint = 0
): Response =
@ -88,7 +88,7 @@ proc requestStorageRaw*(
}
if expiry != 0:
json["expiry"] = %expiry
json["expiry"] = %($expiry)
return client.http.post(url, $json)
@ -98,7 +98,7 @@ proc requestStorage*(
duration: UInt256,
reward: UInt256,
proofProbability: UInt256,
expiry: UInt256,
expiry: uint,
collateral: UInt256,
nodes: uint = 1,
tolerance: uint = 0

4
tests/integration/marketplacesuite.nim
View File

@ -68,11 +68,9 @@ template marketplacesuite*(name: string, body: untyped) =
expiry: uint64 = 4.periods,
nodes = providers().len,
tolerance = 0): Future[PurchaseId] {.async.} =
let expiry = (await ethProvider.currentTime()) + expiry.u256
let id = client.requestStorage(
cid,
expiry=expiry,
expiry=expiry.uint,
duration=duration.u256,
proofProbability=proofProbability.u256,
collateral=collateral,

39
tests/integration/testIntegration.nim
View File

@ -112,10 +112,9 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
check availability in client1.getAvailabilities().get
test "node handles storage request":
let expiry = (await ethProvider.currentTime()) + 10
let cid = client1.upload("some file contents").get
let id1 = client1.requestStorage(cid, duration=100.u256, reward=2.u256, proofProbability=3.u256, expiry=expiry, collateral=200.u256).get
let id2 = client1.requestStorage(cid, duration=400.u256, reward=5.u256, proofProbability=6.u256, expiry=expiry, collateral=201.u256).get
let id1 = client1.requestStorage(cid, duration=100.u256, reward=2.u256, proofProbability=3.u256, expiry=10, collateral=200.u256).get
let id2 = client1.requestStorage(cid, duration=400.u256, reward=5.u256, proofProbability=6.u256, expiry=10, collateral=201.u256).get
check id1 != id2
test "node retrieves purchase status":
@ -124,13 +123,12 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
let chunker = RandomChunker.new(rng, size = DefaultBlockSize * 2, chunkSize = DefaultBlockSize * 2)
let data = await chunker.getBytes()
let cid = client1.upload(byteutils.toHex(data)).get
let expiry = (await ethProvider.currentTime()) + 30
let id = client1.requestStorage(
cid,
duration=100.u256,
reward=2.u256,
proofProbability=3.u256,
expiry=expiry,
expiry=30,
collateral=200.u256,
nodes=2,
tolerance=1).get
@ -139,16 +137,15 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
check request.ask.duration == 100.u256
check request.ask.reward == 2.u256
check request.ask.proofProbability == 3.u256
check request.expiry == expiry
check request.expiry == 30
check request.ask.collateral == 200.u256
check request.ask.slots == 2'u64
check request.ask.maxSlotLoss == 1'u64
# TODO: We currently do not support encoding single chunks
# test "node retrieves purchase status with 1 chunk":
# let expiry = (await ethProvider.currentTime()) + 30
# let cid = client1.upload("some file contents").get
# let id = client1.requestStorage(cid, duration=1.u256, reward=2.u256, proofProbability=3.u256, expiry=expiry, collateral=200.u256, nodes=2, tolerance=1).get
# let id = client1.requestStorage(cid, duration=1.u256, reward=2.u256, proofProbability=3.u256, expiry=30, collateral=200.u256, nodes=2, tolerance=1).get
# let request = client1.getPurchase(id).get.request.get
# check request.ask.duration == 1.u256
# check request.ask.reward == 2.u256
@ -159,13 +156,12 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
# check request.ask.maxSlotLoss == 1'u64
test "node remembers purchase status after restart":
let expiry = (await ethProvider.currentTime()) + 30
let cid = client1.upload("some file contents").get
let id = client1.requestStorage(cid,
duration=100.u256,
reward=2.u256,
proofProbability=3.u256,
expiry=expiry,
expiry=30,
collateral=200.u256).get
check eventually client1.purchaseStateIs(id, "submitted")
@ -177,7 +173,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
check request.ask.duration == 100.u256
check request.ask.reward == 2.u256
check request.ask.proofProbability == 3.u256
check request.expiry == expiry
check request.expiry == 30
check request.ask.collateral == 200.u256
check request.ask.slots == 1'u64
check request.ask.maxSlotLoss == 0'u64
@ -189,14 +185,13 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
let availability = client2.postAvailability(totalSize=size, duration=20*60.u256, minPrice=300.u256, maxCollateral=300.u256).get
# client 1 requests storage
let expiry = (await ethProvider.currentTime()) + 5*60
let cid = client1.upload(data).get
let id = client1.requestStorage(
cid,
duration=10*60.u256,
reward=400.u256,
proofProbability=3.u256,
expiry=expiry,
expiry=5*60,
collateral=200.u256,
nodes = 5,
tolerance = 2).get
@ -228,14 +223,13 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
discard client2.postAvailability(totalSize=size, duration=20*60.u256, minPrice=300.u256, maxCollateral=300.u256).get
# client 1 requests storage
let expiry = (await ethProvider.currentTime()) + 5*60
let cid = client1.upload(data).get
let id = client1.requestStorage(
cid,
duration=duration,
reward=reward,
proofProbability=3.u256,
expiry=expiry,
expiry=5*60,
collateral=200.u256,
nodes = nodes,
tolerance = 2).get
@ -253,12 +247,11 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
test "request storage fails if nodes and tolerance aren't correct":
let cid = client1.upload("some file contents").get
let expiry = (await ethProvider.currentTime()) + 30
let responseBefore = client1.requestStorageRaw(cid,
duration=100.u256,
reward=2.u256,
proofProbability=3.u256,
expiry=expiry,
expiry=30,
collateral=200.u256,
nodes=1,
tolerance=1)
@ -267,20 +260,15 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
check responseBefore.body == "Tolerance cannot be greater or equal than nodes (nodes - tolerance)"
test "node requires expiry and its value to be in future":
let currentTime = await ethProvider.currentTime()
let cid = client1.upload("some file contents").get
let responseMissing = client1.requestStorageRaw(cid, duration=1.u256, reward=2.u256, proofProbability=3.u256, collateral=200.u256)
check responseMissing.status == "400 Bad Request"
check responseMissing.body == "Expiry required"
let responsePast = client1.requestStorageRaw(cid, duration=1.u256, reward=2.u256, proofProbability=3.u256, collateral=200.u256, expiry=currentTime-10)
check responsePast.status == "400 Bad Request"
check "Expiry needs to be in future" in responsePast.body
let responseBefore = client1.requestStorageRaw(cid, duration=1.u256, reward=2.u256, proofProbability=3.u256, collateral=200.u256, expiry=currentTime+10)
let responseBefore = client1.requestStorageRaw(cid, duration=10.u256, reward=2.u256, proofProbability=3.u256, collateral=200.u256, expiry=10)
check responseBefore.status == "400 Bad Request"
check "Expiry has to be before the request's end (now + duration)" in responseBefore.body
check "Expiry needs value bigger then zero and smaller then the request's duration" in responseBefore.body
test "updating non-existing availability":
let nonExistingResponse = client1.patchAvailabilityRaw(AvailabilityId.example, duration=100.u256.some, minPrice=200.u256.some, maxCollateral=200.u256.some)
@ -317,14 +305,13 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
let availability = client1.postAvailability(totalSize=originalSize, duration=20*60.u256, minPrice=300.u256, maxCollateral=300.u256).get
# Lets create storage request that will utilize some of the availability's space
let expiry = (await ethProvider.currentTime()) + 5*60
let cid = client2.upload(data).get
let id = client2.requestStorage(
cid,
duration=10*60.u256,
reward=400.u256,
proofProbability=3.u256,
expiry=expiry,
expiry=5*60,
collateral=200.u256,
nodes = 5,
tolerance = 2).get

2
vendor/codex-contracts-eth vendored

@ -1 +1 @@
Subproject commit a58427e496088b904aed070e92f1c479c45fd852
Subproject commit 57e8cd5013325f05e16833a5320b575d32a403f3