From c05eec422c8d6835f9606c2fdb35f645cc20c35f Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 6 Feb 2025 16:21:12 +0100 Subject: [PATCH] fix dataset and slot size calculations in integration tests (#1095) * fixes datasetSize and slotSize helpers (and also RandomChunker.example) * adds overload for <> for seq[byte] * changes RandomChunker.example to return seq[byte] * fixes restapi tests after correcting RandomChunker.example * review: use string.fromBytes from nim-stew to convert seq[byte] to string --- tests/examples.nim | 5 ++--- tests/integration/codexclient.nim | 3 +++ tests/integration/marketplacesuite.nim | 10 +++++++--- tests/integration/testmarketplace.nim | 4 ++-- tests/integration/testproofs.nim | 4 +--- tests/integration/testrestapi.nim | 26 ++++++++++++++------------ tests/integration/testupdownload.nim | 2 +- 7 files changed, 30 insertions(+), 24 deletions(-) diff --git a/tests/examples.nim b/tests/examples.nim index bfb34cff..c96fefd6 100644 --- a/tests/examples.nim +++ b/tests/examples.nim @@ -86,8 +86,7 @@ proc example(_: type G2Point): G2Point = proc example*(_: type Groth16Proof): Groth16Proof = Groth16Proof(a: G1Point.example, b: G2Point.example, c: G1Point.example) -proc example*(_: type RandomChunker, blocks: int): Future[string] {.async.} = - # doAssert blocks >= 3, "must be more than 3 blocks" +proc example*(_: type RandomChunker, blocks: int): Future[seq[byte]] {.async.} = let rng = Rng.instance() let chunker = RandomChunker.new( rng, size = DefaultBlockSize * blocks.NBytes, chunkSize = DefaultBlockSize @@ -95,7 +94,7 @@ proc example*(_: type RandomChunker, blocks: int): Future[string] {.async.} = var data: seq[byte] while (let moar = await chunker.getBytes(); moar != []): data.add moar - return byteutils.toHex(data) + return data proc example*(_: type RandomChunker): Future[string] {.async.} = await RandomChunker.example(3) diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index 7826b151..d1191fb9 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -44,6 +44,9 @@ proc upload*(client: CodexClient, contents: string): ?!Cid = assert response.status == "200 OK" Cid.init(response.body).mapFailure +proc upload*(client: CodexClient, bytes: seq[byte]): ?!Cid = + client.upload(string.fromBytes(bytes)) + proc download*(client: CodexClient, cid: Cid, local = false): ?!string = let response = client.http.get( client.baseurl & "/data/" & $cid & (if local: "" else: "/network/stream") diff --git a/tests/integration/marketplacesuite.nim b/tests/integration/marketplacesuite.nim index 4d155186..68283ad1 100644 --- a/tests/integration/marketplacesuite.nim +++ b/tests/integration/marketplacesuite.nim @@ -4,6 +4,7 @@ from pkg/libp2p import Cid import pkg/codex/contracts/marketplace as mp import pkg/codex/periods import pkg/codex/utils/json +from pkg/codex/utils import roundUp, divUp import ./multinodes import ../contracts/time import ../contracts/deployment @@ -45,11 +46,14 @@ template marketplacesuite*(name: string, body: untyped) = proc periods(p: int): uint64 = p.uint64 * period - proc slotSize(blocks: int): UInt256 = - (DefaultBlockSize * blocks.NBytes).Natural.u256 + proc slotSize(blocks, nodes, tolerance: int): UInt256 = + let ecK = nodes - tolerance + let blocksRounded = roundUp(blocks, ecK) + let blocksPerSlot = divUp(blocksRounded, ecK) + (DefaultBlockSize * blocksPerSlot.NBytes).Natural.u256 proc datasetSize(blocks, nodes, tolerance: int): UInt256 = - (nodes + tolerance).u256 * slotSize(blocks) + return nodes.u256 * slotSize(blocks, nodes, tolerance) proc createAvailabilities( datasetSize: UInt256, diff --git a/tests/integration/testmarketplace.nim b/tests/integration/testmarketplace.nim index bc030a1d..7813485b 100644 --- a/tests/integration/testmarketplace.nim +++ b/tests/integration/testmarketplace.nim @@ -112,7 +112,7 @@ marketplacesuite "Marketplace": await ethProvider.advanceTime(duration) # Checking that the hosting node received reward for at least the time between - let slotSize = slotSize(blocks) + let slotSize = slotSize(blocks, ecNodes, ecTolerance) let pricePerSlotPerSecond = minPricePerBytePerSecond * slotSize check eventually (await token.balanceOf(hostAccount)) - startBalanceHost >= (duration - 5 * 60) * pricePerSlotPerSecond * ecNodes.u256 @@ -197,7 +197,7 @@ marketplacesuite "Marketplace payouts": await advanceToNextPeriod() - let slotSize = slotSize(blocks) + let slotSize = slotSize(blocks, ecNodes, ecTolerance) let pricePerSlotPerSecond = minPricePerBytePerSecond * slotSize check eventually ( diff --git a/tests/integration/testproofs.nim b/tests/integration/testproofs.nim index b25643ad..a547890b 100644 --- a/tests/integration/testproofs.nim +++ b/tests/integration/testproofs.nim @@ -1,7 +1,6 @@ from std/times import inMilliseconds import pkg/questionable import pkg/codex/logutils -import pkg/stew/byteutils import ../contracts/time import ../contracts/deployment import ../codex/helpers @@ -60,8 +59,7 @@ marketplacesuite "Hosts submit regular proofs": let purchase = client0.getPurchase(purchaseId).get check purchase.error == none string - let request = purchase.request.get - let slotSize = request.ask.slotSize + let slotSize = slotSize(blocks, ecNodes, ecTolerance) check eventually( client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000 diff --git a/tests/integration/testrestapi.nim b/tests/integration/testrestapi.nim index f1f2299f..8cbe9817 100644 --- a/tests/integration/testrestapi.nim +++ b/tests/integration/testrestapi.nim @@ -1,5 +1,6 @@ import std/httpclient import std/sequtils +import std/strformat from pkg/libp2p import `==` import pkg/codex/units import ./twonodes @@ -144,18 +145,19 @@ twonodessuite "REST API": check responseBefore.body == "Invalid parameters: `tolerance` cannot be greater than `nodes`" - test "request storage succeeds if nodes and tolerance within range", twoNodesConfig: - let data = await RandomChunker.example(blocks = 2) - let cid = client1.upload(data).get - let duration = 100.u256 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 3.u256 - let expiry = 30.uint - let collateralPerByte = 1.u256 - let ecParams = @[(3, 1), (5, 2)] - - for ecParam in ecParams: - let (nodes, tolerance) = ecParam + for ecParams in @[ + (minBlocks: 2, nodes: 3, tolerance: 1), (minBlocks: 3, nodes: 5, tolerance: 2) + ]: + let (minBlocks, nodes, tolerance) = ecParams + test "request storage succeeds if nodes and tolerance within range " & + fmt"({minBlocks=}, {nodes=}, {tolerance=})", twoNodesConfig: + let data = await RandomChunker.example(blocks = minBlocks) + let cid = client1.upload(data).get + let duration = 100.u256 + let pricePerBytePerSecond = 1.u256 + let proofProbability = 3.u256 + let expiry = 30.uint + let collateralPerByte = 1.u256 var responseBefore = client1.requestStorageRaw( cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, diff --git a/tests/integration/testupdownload.nim b/tests/integration/testupdownload.nim index 74bee8c7..05d3a496 100644 --- a/tests/integration/testupdownload.nim +++ b/tests/integration/testupdownload.nim @@ -88,7 +88,7 @@ twonodessuite "Uploads and downloads": let cid = a.upload(data).get let response = b.download(cid).get check: - response == data + @response.mapIt(it.byte) == data for run in 0 .. 10: await transferTest(client1, client2)