Fix verifiable manifest constructor (#844)

* Fix verifiable manifest constructor

* Add integration test for verifiable manifest download

Add integration test for testing download of verifiable dataset after creating request for storage

* add missing import

* add testecbug to integration suite

* Remove hardhat instance from integration test

* change description, drop echo

---------

Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com>
Co-authored-by: gmega <giuliano.mega@gmail.com>
This commit is contained in:
Tomasz Bekas 2024-06-26 22:02:39 +02:00 committed by GitHub
parent 5b131611ac
commit 11eaebfd77
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 108 additions and 2 deletions

View File

@ -322,7 +322,7 @@ func new*(
protected: true,
ecK: manifest.ecK,
ecM: manifest.ecM,
originalTreeCid: manifest.treeCid,
originalTreeCid: manifest.originalTreeCid,
originalDatasetSize: manifest.originalDatasetSize,
protectedStrategy: manifest.protectedStrategy,
verifiable: true,

View File

@ -17,6 +17,7 @@ import pkg/taskpools
import ../asynctest
import ./helpers
import ./examples
suite "Erasure encode/decode":
const BlockSize = 1024'nb
@ -232,3 +233,22 @@ suite "Erasure encode/decode":
let encoded = await encode(buffers, parity)
discard (await erasure.decode(encoded)).tryGet()
test "Should handle verifiable manifests":
const
buffers = 20
parity = 10
let
encoded = await encode(buffers, parity)
slotCids = collect(newSeq):
for i in 0..<encoded.numSlots: Cid.example
verifiable = Manifest.new(encoded, Cid.example, slotCids).tryGet()
decoded = (await erasure.decode(verifiable)).tryGet()
check:
decoded.treeCid == manifest.treeCid
decoded.treeCid == verifiable.originalTreeCid
decoded.blocksCount == verifiable.originalBlocksCount

View File

@ -4,6 +4,7 @@ import std/strutils
from pkg/libp2p import Cid, `$`, init
import pkg/stint
import pkg/questionable/results
import pkg/chronos/apps/http/[httpserver, shttpserver, httpclient]
import pkg/codex/logutils
import pkg/codex/rest/json
import pkg/codex/purchasing
@ -15,9 +16,16 @@ export purchasing
type CodexClient* = ref object
http: HttpClient
baseurl: string
session: HttpSessionRef
type CodexClientError* = object of CatchableError
proc new*(_: type CodexClient, baseurl: string): CodexClient =
CodexClient(http: newHttpClient(), baseurl: baseurl)
CodexClient(
http: newHttpClient(),
baseurl: baseurl,
session: HttpSessionRef.new({HttpClientFlag.Http11Pipeline})
)
proc info*(client: CodexClient): ?!JsonNode =
let url = client.baseurl & "/debug/info"
@ -45,6 +53,23 @@ proc download*(client: CodexClient, cid: Cid, local = false): ?!string =
success response.body
proc downloadBytes*(
client: CodexClient,
cid: Cid,
local = false): Future[?!seq[byte]] {.async.} =
let uri = parseUri(
client.baseurl & "/data/" & $cid &
(if local: "" else: "/network")
)
let (status, bytes) = await client.session.fetch(uri)
if status != 200:
return failure("fetch failed with status " & $status)
success bytes
proc list*(client: CodexClient): ?!RestContentList =
let url = client.baseurl & "/data"
let response = client.http.get(url)

View File

@ -0,0 +1,60 @@
from pkg/libp2p import Cid, init
import ../examples
import ./marketplacesuite
import ./nodeconfigs
import ./hardhatconfig
marketplacesuite "Bug #821 - node crashes during erasure coding":
test "should be able to create storage request and download dataset",
NodeConfigs(
clients:
CodexConfigs.init(nodes=1)
# .debug() # uncomment to enable console log output.debug()
.withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
.withLogTopics("node", "erasure", "marketplace", )
.some,
providers:
CodexConfigs.init(nodes=0)
# .debug() # uncomment to enable console log output
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
# .withLogTopics("node", "marketplace", "sales", "reservations", "node", "proving", "clock")
.some,
):
let reward = 400.u256
let duration = 10.periods
let collateral = 200.u256
let expiry = 5.periods
let data = await RandomChunker.example(blocks=8)
let client = clients()[0]
let clientApi = client.client
let cid = clientApi.upload(data).get
var requestId = none RequestId
proc onStorageRequested(event: StorageRequested) {.raises:[].} =
requestId = event.requestId.some
let subscription = await marketplace.subscribe(StorageRequested, onStorageRequested)
# client requests storage but requires multiple slots to host the content
let id = await clientApi.requestStorage(
cid,
duration=duration,
reward=reward,
expiry=expiry,
collateral=collateral,
nodes=3,
tolerance=1
)
check eventually(requestId.isSome, timeout=expiry.int * 1000)
let request = await marketplace.getRequest(requestId.get)
let cidFromRequest = Cid.init(request.content.cid).get()
let downloaded = await clientApi.downloadBytes(cidFromRequest, local = true)
check downloaded.isOk
check downloaded.get.toHex == data.toHex
await subscription.unsubscribe()

View File

@ -6,5 +6,6 @@ import ./integration/testpurchasing
import ./integration/testblockexpiration
import ./integration/testmarketplace
import ./integration/testproofs
import ./integration/testecbug
{.warning[UnusedImport]:off.}