Merge branch 'master' into feature/ceremony-files

codex/manifest/manifest.nim

@@ -322,7 +322,7 @@ func new*(
     protected: true,
     ecK: manifest.ecK,
     ecM: manifest.ecM,
-    originalTreeCid: manifest.treeCid,
+    originalTreeCid: manifest.originalTreeCid,
     originalDatasetSize: manifest.originalDatasetSize,
     protectedStrategy: manifest.protectedStrategy,
     verifiable: true,

codex/node.nim

@@ -257,7 +257,10 @@ proc streamEntireDataset(
             leoDecoderProvider,
             self.taskpool)
         without _ =? (await erasure.decode(manifest)), error:
-          trace "Unable to erasure decode manifest", manifestCid, exc = error.msg
+          error "Unable to erasure decode manifest", manifestCid, exc = error.msg
+          return failure(error)
+
+        return success()
       # --------------------------------------------------------------------------
       # FIXME this is a HACK so that the node does not crash during the workshop.
       #   We should NOT catch Defect.

tests/codex/testerasure.nim

@@ -17,6 +17,7 @@ import pkg/taskpools
 
 import ../asynctest
 import ./helpers
+import ./examples
 
 suite "Erasure encode/decode":
   const BlockSize = 1024'nb
@@ -232,3 +233,22 @@ suite "Erasure encode/decode":
     let encoded = await encode(buffers, parity)
 
     discard (await erasure.decode(encoded)).tryGet()
+
+  test "Should handle verifiable manifests":
+    const
+      buffers = 20
+      parity = 10
+
+    let
+      encoded = await encode(buffers, parity)
+      slotCids = collect(newSeq):
+        for i in 0..<encoded.numSlots: Cid.example
+
+      verifiable = Manifest.new(encoded, Cid.example, slotCids).tryGet()
+
+      decoded = (await erasure.decode(verifiable)).tryGet()
+
+    check:
+      decoded.treeCid == manifest.treeCid
+      decoded.treeCid == verifiable.originalTreeCid
+      decoded.blocksCount == verifiable.originalBlocksCount

tests/integration/codexclient.nim

@@ -4,6 +4,7 @@ import std/strutils
 from pkg/libp2p import Cid, `$`, init
 import pkg/stint
 import pkg/questionable/results
+import pkg/chronos/apps/http/[httpserver, shttpserver, httpclient]
 import pkg/codex/logutils
 import pkg/codex/rest/json
 import pkg/codex/purchasing
@@ -15,9 +16,16 @@ export purchasing
 type CodexClient* = ref object
   http: HttpClient
   baseurl: string
+  session: HttpSessionRef
+
+type CodexClientError* = object of CatchableError
 
 proc new*(_: type CodexClient, baseurl: string): CodexClient =
-  CodexClient(http: newHttpClient(), baseurl: baseurl)
+  CodexClient(
+    http: newHttpClient(),
+    baseurl: baseurl,
+    session: HttpSessionRef.new({HttpClientFlag.Http11Pipeline})
+  )
 
 proc info*(client: CodexClient): ?!JsonNode =
   let url = client.baseurl & "/debug/info"
@@ -45,6 +53,23 @@ proc download*(client: CodexClient, cid: Cid, local = false): ?!string =
 
   success response.body
 
+proc downloadBytes*(
+  client: CodexClient,
+  cid: Cid,
+  local = false): Future[?!seq[byte]] {.async.} =
+
+  let uri = parseUri(
+    client.baseurl & "/data/" & $cid &
+    (if local: "" else: "/network")
+  )
+
+  let (status, bytes) = await client.session.fetch(uri)
+
+  if status != 200:
+    return failure("fetch failed with status " & $status)
+
+  success bytes
+
 proc list*(client: CodexClient): ?!RestContentList =
   let url = client.baseurl & "/data"
   let response = client.http.get(url)

tests/integration/testecbug.nim

@@ -0,0 +1,60 @@
+from pkg/libp2p import Cid, init
+import ../examples
+import ./marketplacesuite
+import ./nodeconfigs
+import ./hardhatconfig
+
+marketplacesuite "Bug #821 - node crashes during erasure coding":
+
+  test "should be able to create storage request and download dataset",
+    NodeConfigs(
+      clients:
+        CodexConfigs.init(nodes=1)
+          # .debug() # uncomment to enable console log output.debug()
+          .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
+          .withLogTopics("node", "erasure", "marketplace", )
+          .some,
+
+      providers:
+        CodexConfigs.init(nodes=0)
+          # .debug() # uncomment to enable console log output
+          # .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
+          # .withLogTopics("node", "marketplace", "sales", "reservations", "node", "proving", "clock")
+          .some,
+  ):
+    let reward = 400.u256
+    let duration = 10.periods
+    let collateral = 200.u256
+    let expiry = 5.periods
+    let data = await RandomChunker.example(blocks=8)
+    let client = clients()[0]
+    let clientApi = client.client
+
+    let cid = clientApi.upload(data).get
+
+    var requestId = none RequestId
+    proc onStorageRequested(event: StorageRequested) {.raises:[].} =
+      requestId = event.requestId.some
+
+    let subscription = await marketplace.subscribe(StorageRequested, onStorageRequested)
+
+    # client requests storage but requires multiple slots to host the content
+    let id = await clientApi.requestStorage(
+      cid,
+      duration=duration,
+      reward=reward,
+      expiry=expiry,
+      collateral=collateral,
+      nodes=3,
+      tolerance=1
+    )
+
+    check eventually(requestId.isSome, timeout=expiry.int * 1000)
+
+    let request = await marketplace.getRequest(requestId.get)
+    let cidFromRequest = Cid.init(request.content.cid).get()
+    let downloaded = await clientApi.downloadBytes(cidFromRequest, local = true)
+    check downloaded.isOk
+    check downloaded.get.toHex == data.toHex
+
+    await subscription.unsubscribe()

tests/testIntegration.nim

@@ -6,5 +6,6 @@ import ./integration/testpurchasing
 import ./integration/testblockexpiration
 import ./integration/testmarketplace
 import ./integration/testproofs
+import ./integration/testecbug
 
 {.warning[UnusedImport]:off.}

vendor/nim-leopard

@@ -1 +1 @@
-Subproject commit 1a6f2ab7252426a6ac01482a68b75d0c3b134cf0
+Subproject commit 895ff24ca6615d577acfb11811cdd5465f596c97