fix dataset and slot size calculations in integration tests (#1095)

* fixes datasetSize and slotSize helpers (and also RandomChunker.example)

* adds overload for <<upload>> for seq[byte]

* changes RandomChunker.example to return seq[byte]

* fixes restapi tests after correcting RandomChunker.example

* review: use string.fromBytes from nim-stew to convert seq[byte] to string

tests/examples.nim

@@ -86,8 +86,7 @@ proc example(_: type G2Point): G2Point =
 proc example*(_: type Groth16Proof): Groth16Proof =
   Groth16Proof(a: G1Point.example, b: G2Point.example, c: G1Point.example)
 
-proc example*(_: type RandomChunker, blocks: int): Future[string] {.async.} =
+proc example*(_: type RandomChunker, blocks: int): Future[seq[byte]] {.async.} =
-  # doAssert blocks >= 3, "must be more than 3 blocks"
   let rng = Rng.instance()
   let chunker = RandomChunker.new(
     rng, size = DefaultBlockSize * blocks.NBytes, chunkSize = DefaultBlockSize
@@ -95,7 +94,7 @@ proc example*(_: type RandomChunker, blocks: int): Future[string] {.async.} =
   var data: seq[byte]
   while (let moar = await chunker.getBytes(); moar != []):
     data.add moar
-  return byteutils.toHex(data)
+  return data
 
 proc example*(_: type RandomChunker): Future[string] {.async.} =
   await RandomChunker.example(3)

tests/integration/codexclient.nim

@@ -44,6 +44,9 @@ proc upload*(client: CodexClient, contents: string): ?!Cid =
   assert response.status == "200 OK"
   Cid.init(response.body).mapFailure
 
+proc upload*(client: CodexClient, bytes: seq[byte]): ?!Cid =
+  client.upload(string.fromBytes(bytes))
+
 proc download*(client: CodexClient, cid: Cid, local = false): ?!string =
   let response = client.http.get(
     client.baseurl & "/data/" & $cid & (if local: "" else: "/network/stream")

tests/integration/marketplacesuite.nim

@@ -4,6 +4,7 @@ from pkg/libp2p import Cid
 import pkg/codex/contracts/marketplace as mp
 import pkg/codex/periods
 import pkg/codex/utils/json
+from pkg/codex/utils import roundUp, divUp
 import ./multinodes
 import ../contracts/time
 import ../contracts/deployment
@@ -45,11 +46,14 @@ template marketplacesuite*(name: string, body: untyped) =
     proc periods(p: int): uint64 =
       p.uint64 * period
 
-    proc slotSize(blocks: int): UInt256 =
+    proc slotSize(blocks, nodes, tolerance: int): UInt256 =
-      (DefaultBlockSize * blocks.NBytes).Natural.u256
+      let ecK = nodes - tolerance
+      let blocksRounded = roundUp(blocks, ecK)
+      let blocksPerSlot = divUp(blocksRounded, ecK)
+      (DefaultBlockSize * blocksPerSlot.NBytes).Natural.u256
 
     proc datasetSize(blocks, nodes, tolerance: int): UInt256 =
-      (nodes + tolerance).u256 * slotSize(blocks)
+      return nodes.u256 * slotSize(blocks, nodes, tolerance)
 
     proc createAvailabilities(
         datasetSize: UInt256,

tests/integration/testmarketplace.nim

@@ -112,7 +112,7 @@ marketplacesuite "Marketplace":
     await ethProvider.advanceTime(duration)
 
     # Checking that the hosting node received reward for at least the time between <expiry;end>
-    let slotSize = slotSize(blocks)
+    let slotSize = slotSize(blocks, ecNodes, ecTolerance)
     let pricePerSlotPerSecond = minPricePerBytePerSecond * slotSize
     check eventually (await token.balanceOf(hostAccount)) - startBalanceHost >=
       (duration - 5 * 60) * pricePerSlotPerSecond * ecNodes.u256
@@ -197,7 +197,7 @@ marketplacesuite "Marketplace payouts":
 
     await advanceToNextPeriod()
 
-    let slotSize = slotSize(blocks)
+    let slotSize = slotSize(blocks, ecNodes, ecTolerance)
     let pricePerSlotPerSecond = minPricePerBytePerSecond * slotSize
 
     check eventually (

tests/integration/testproofs.nim

@@ -1,7 +1,6 @@
 from std/times import inMilliseconds
 import pkg/questionable
 import pkg/codex/logutils
-import pkg/stew/byteutils
 import ../contracts/time
 import ../contracts/deployment
 import ../codex/helpers
@@ -60,8 +59,7 @@ marketplacesuite "Hosts submit regular proofs":
     let purchase = client0.getPurchase(purchaseId).get
     check purchase.error == none string
 
-    let request = purchase.request.get
+    let slotSize = slotSize(blocks, ecNodes, ecTolerance)
-    let slotSize = request.ask.slotSize
 
     check eventually(
       client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000

tests/integration/testrestapi.nim

@@ -1,5 +1,6 @@
 import std/httpclient
 import std/sequtils
+import std/strformat
 from pkg/libp2p import `==`
 import pkg/codex/units
 import ./twonodes
@@ -144,18 +145,19 @@ twonodessuite "REST API":
       check responseBefore.body ==
         "Invalid parameters: `tolerance` cannot be greater than `nodes`"
 
-  test "request storage succeeds if nodes and tolerance within range", twoNodesConfig:
+  for ecParams in @[
-    let data = await RandomChunker.example(blocks = 2)
+    (minBlocks: 2, nodes: 3, tolerance: 1), (minBlocks: 3, nodes: 5, tolerance: 2)
-    let cid = client1.upload(data).get
+  ]:
-    let duration = 100.u256
+    let (minBlocks, nodes, tolerance) = ecParams
-    let pricePerBytePerSecond = 1.u256
+    test "request storage succeeds if nodes and tolerance within range " &
-    let proofProbability = 3.u256
+      fmt"({minBlocks=}, {nodes=}, {tolerance=})", twoNodesConfig:
-    let expiry = 30.uint
+      let data = await RandomChunker.example(blocks = minBlocks)
-    let collateralPerByte = 1.u256
+      let cid = client1.upload(data).get
-    let ecParams = @[(3, 1), (5, 2)]
+      let duration = 100.u256
-
+      let pricePerBytePerSecond = 1.u256
-    for ecParam in ecParams:
+      let proofProbability = 3.u256
-      let (nodes, tolerance) = ecParam
+      let expiry = 30.uint
+      let collateralPerByte = 1.u256
 
       var responseBefore = client1.requestStorageRaw(
         cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte,

tests/integration/testupdownload.nim

@@ -88,7 +88,7 @@ twonodessuite "Uploads and downloads":
       let cid = a.upload(data).get
       let response = b.download(cid).get
       check:
-        response == data
+        @response.mapIt(it.byte) == data
 
     for run in 0 .. 10:
       await transferTest(client1, client2)