WIP: test takes 10 mins during erasure coding

codex/erasure/erasure.nim

@@ -85,12 +85,13 @@ proc encode*(
   without var encoded =? Manifest.new(manifest, blocks, parity), error:
     trace "Unable to create manifest", msg = error.msg
     return error.failure
-
   logScope:
     steps           = encoded.steps
     rounded_blocks  = encoded.rounded
     new_manifest    = encoded.len
 
+  trace "Manifest encoded"
+
   var
     encoder = self.encoderProvider(manifest.blockSize.int, blocks, parity)
 

codex/node.nim

@@ -268,7 +268,7 @@ proc requestStorage*(
   ## - Run the PoR setup on the erasure dataset
   ## - Call into the marketplace and purchasing contracts
   ##
-  trace "Received a request for storage!", cid, duration, nodes, tolerance, reward
+  trace "Received a request for storage!", cid, duration, proofProbability, nodes, tolerance, reward, collateral, expiry
 
   without contracts =? self.contracts.client:
     trace "Purchasing not available"
@@ -278,6 +278,7 @@ proc requestStorage*(
     trace "Unable to fetch manifest for cid", cid
     raise error
 
+  trace "manifest created successfully", blockSize = manifest.blockSize
   # Erasure code the dataset according to provided parameters
   without encoded =? (await self.erasure.encode(manifest, nodes.int, tolerance.int)), error:
     trace "Unable to erasure code dataset", cid

codex/rest/api.nim

@@ -241,7 +241,10 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
 
       try:
         without cid =? (
-          await node.store(AsyncStreamWrapper.new(reader = AsyncStreamReader(reader)))), error:
+          await node.store(
+            AsyncStreamWrapper.new(reader = AsyncStreamReader(reader)),
+            blockSize = 1024'nb) # FIXME: workaround for slow erasure coding with blocktype.DefaultBlockSize = 65,472 NBytes
+        ), error:
           trace "Error uploading file", exc = error.msg
           return RestApiResponse.error(Http500, error.msg)
 

codex/rest/json.nim

@@ -39,8 +39,10 @@ proc fromJson*(
   let reward = ?catch UInt256.fromDecimal(json["reward"].getStr)
   let collateral = ?catch UInt256.fromDecimal(json["collateral"].getStr)
   let expiry = UInt256.fromDecimal(json["expiry"].getStr).catch.option
-  let nodes = parseUInt(json["nodes"].getStr).catch.option
+  let nodes = json{"nodes"}.getBiggestInt.uint.option
   let tolerance = parseUInt(json["tolerance"].getStr).catch.option
+  echo "[json.fromJson] nodes: ", nodes
+  echo "[json.fromJson] tolerance: ", tolerance
   success StorageRequestParams(
     duration: duration,
     proofProbability: proofProbability,

tests/integration/codexclient.nim

@@ -33,7 +33,9 @@ proc requestStorage*(
     reward: uint64,
     proofProbability: uint64,
     expiry: UInt256,
-    collateral: uint64
+    collateral: uint64,
+    nodes: uint = 1,
+    tolerance: uint = 0
 ): string =
   ## Call request storage REST endpoint
   ## 
@@ -44,6 +46,8 @@ proc requestStorage*(
     "proofProbability": $proofProbability,
     "expiry": $expiry,
     "collateral": $collateral,
+    "nodes": nodes,
+    "tolerance": $tolerance
   }
   let response = client.http.post(url, $json)
   assert response.status == "200 OK"

tests/integration/testIntegration.nim

@@ -13,7 +13,7 @@ import ./twonodes
 # You can also pass a string in same format like for the `--log-level` parameter
 # to enable custom logging levels for specific topics like: debug2 = "INFO; TRACE: marketplace"
 
-twonodessuite "Integration tests", debug1 = false, debug2 = false:
+twonodessuite "Integration tests", debug1 = "INFO;TRACE:node,restapi,erasure", debug2 = false:
   setup:
     # Our Hardhat configuration does use automine, which means that time tracked by `provider.currentTime()` is not
     # advanced until blocks are mined and that happens only when transaction is submitted.
@@ -59,7 +59,7 @@ twonodessuite "Integration tests", debug1 = false, debug2 = false:
   test "node remembers purchase status after restart":
     let expiry = (await provider.currentTime()) + 30
     let cid = client1.upload("some file contents")
-    let id = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry, collateral=200)
+    let id = client1.requestStorage(cid, duration=1, reward=2, proofProbability=3, expiry=expiry, collateral=200, nodes=2, tolerance=1)
     check eventually client1.getPurchase(id){"state"}.getStr() == "submitted"
 
     node1.restart()