From 54177e9fbfd143534b51131d7893459fe7469f4b Mon Sep 17 00:00:00 2001 From: Giuliano Mega Date: Mon, 17 Mar 2025 17:08:24 -0300 Subject: [PATCH 01/24] feat(integration): use async client instead of standard Nim HTTP client (#1159) * WiP: migrating CodexClient to chronos http client * fix(api): fixes #1163 * feat: fully working API integration tests * convert most of the tests in testupdownload * feat: working updownload tests on async client * feat: make testsales work with async codexclient * feat: make testpurchasing work with async codexclient * feat: make testblockexpiration work with async codexclient * feat: make marketplacesuite work with async codexclient * make testproofs work with async codexclient * chore: refactor client to express higher level in terms of lower level operations * fix: set correct content-length for erasure-coded datasets * feat: make testecbug work with async client * feat: make testvalidator work with async client * refactor: simplify request aliases, add close operation * wire back client.close at node shutdown * refactor: remove unused exception * fix: use await instead of waitFor on async call sites --- codex/rest/api.nim | 11 +- tests/integration/codexclient.nim | 357 ++++++++++++++-------- tests/integration/codexprocess.nim | 2 +- tests/integration/marketplacesuite.nim | 26 +- tests/integration/multinodes.nim | 10 +- tests/integration/testblockexpiration.nim | 10 +- tests/integration/testecbug.nim | 31 +- tests/integration/testmarketplace.nim | 52 ++-- tests/integration/testproofs.nim | 38 +-- tests/integration/testpurchasing.nim | 104 ++++--- tests/integration/testrestapi.nim | 302 +++++++++--------- tests/integration/testsales.nim | 153 ++++++---- tests/integration/testupdownload.nim | 41 +-- tests/integration/testvalidator.nim | 16 +- tests/testTaiko.nim | 2 +- 15 files changed, 656 insertions(+), 499 deletions(-) diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 054e1c2b..553cb91c 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -114,9 +114,14 @@ proc retrieveCid( else: resp.setHeader("Content-Disposition", "attachment") - resp.setHeader("Content-Length", $manifest.datasetSize.int) + # For erasure-coded datasets, we need to return the _original_ length; i.e., + # the length of the non-erasure-coded dataset, as that's what we will be + # returning to the client. + let contentLength = + if manifest.protected: manifest.originalDatasetSize else: manifest.datasetSize + resp.setHeader("Content-Length", $(contentLength.int)) - await resp.prepareChunked() + await resp.prepare(HttpResponseStreamType.Plain) while not stream.atEof: var @@ -129,7 +134,7 @@ proc retrieveCid( bytes += buff.len - await resp.sendChunk(addr buff[0], buff.len) + await resp.send(addr buff[0], buff.len) await resp.finish() codex_api_downloads.inc() except CancelledError as exc: diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index 4a106253..ef76b577 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -4,119 +4,216 @@ import std/strutils from pkg/libp2p import Cid, `$`, init import pkg/stint import pkg/questionable/results -import pkg/chronos/apps/http/[httpserver, shttpserver, httpclient] +import pkg/chronos/apps/http/[httpserver, shttpserver, httpclient, httptable] import pkg/codex/logutils import pkg/codex/rest/json import pkg/codex/purchasing import pkg/codex/errors import pkg/codex/sales/reservations -export purchasing +export purchasing, httptable, httpclient type CodexClient* = ref object baseurl: string - httpClients: seq[HttpClient] - -type CodexClientError* = object of CatchableError - -const HttpClientTimeoutMs = 60 * 1000 + session: HttpSessionRef proc new*(_: type CodexClient, baseurl: string): CodexClient = - CodexClient(baseurl: baseurl, httpClients: newSeq[HttpClient]()) + CodexClient(session: HttpSessionRef.new(), baseurl: baseurl) -proc http*(client: CodexClient): HttpClient = - let httpClient = newHttpClient(timeout = HttpClientTimeoutMs) - client.httpClients.insert(httpClient) - return httpClient +proc close*(self: CodexClient): Future[void] {.async: (raises: []).} = + await self.session.closeWait() -proc close*(client: CodexClient): void = - for httpClient in client.httpClients: - httpClient.close() +proc request( + self: CodexClient, + httpMethod: httputils.HttpMethod, + url: string, + body: openArray[char] = [], + headers: openArray[HttpHeaderTuple] = [], +): Future[HttpClientResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]) +.} = + HttpClientRequestRef + .new( + self.session, + url, + httpMethod, + version = HttpVersion11, + flags = {}, + maxResponseHeadersSize = HttpMaxHeadersSize, + headers = headers, + body = body.toOpenArrayByte(0, len(body) - 1), + ).get + .send() -proc info*(client: CodexClient): ?!JsonNode = - let url = client.baseurl & "/debug/info" - JsonNode.parse(client.http().getContent(url)) +proc post( + self: CodexClient, + url: string, + body: string = "", + headers: seq[HttpHeaderTuple] = @[], +): Future[HttpClientResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]) +.} = + return self.request(MethodPost, url, headers = headers, body = body) -proc setLogLevel*(client: CodexClient, level: string) = - let url = client.baseurl & "/debug/chronicles/loglevel?level=" & level - let headers = newHttpHeaders({"Content-Type": "text/plain"}) - let response = client.http().request(url, httpMethod = HttpPost, headers = headers) - assert response.status == "200 OK" +proc get( + self: CodexClient, url: string, headers: seq[HttpHeaderTuple] = @[] +): Future[HttpClientResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]) +.} = + return self.request(MethodGet, url, headers = headers) -proc upload*(client: CodexClient, contents: string): ?!Cid = - let response = client.http().post(client.baseurl & "/data", contents) - assert response.status == "200 OK" - Cid.init(response.body).mapFailure +proc delete( + self: CodexClient, url: string, headers: seq[HttpHeaderTuple] = @[] +): Future[HttpClientResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]) +.} = + return self.request(MethodDelete, url, headers = headers) -proc upload*(client: CodexClient, bytes: seq[byte]): ?!Cid = - client.upload(string.fromBytes(bytes)) +proc patch( + self: CodexClient, + url: string, + body: string = "", + headers: seq[HttpHeaderTuple] = @[], +): Future[HttpClientResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]) +.} = + return self.request(MethodPatch, url, headers = headers, body = body) -proc download*(client: CodexClient, cid: Cid, local = false): ?!string = - let response = client.http().get( - client.baseurl & "/data/" & $cid & (if local: "" else: "/network/stream") - ) +proc body*( + response: HttpClientResponseRef +): Future[string] {.async: (raises: [CancelledError, HttpError]).} = + return bytesToString (await response.getBodyBytes()) - if response.status != "200 OK": - return failure(response.status) +proc getContent( + client: CodexClient, url: string, headers: seq[HttpHeaderTuple] = @[] +): Future[string] {.async: (raises: [CancelledError, HttpError]).} = + let response = await client.get(url, headers) + return await response.body - success response.body +proc info*( + client: CodexClient +): Future[?!JsonNode] {.async: (raises: [CancelledError, HttpError]).} = + let response = await client.get(client.baseurl & "/debug/info") + return JsonNode.parse(await response.body) -proc downloadManifestOnly*(client: CodexClient, cid: Cid): ?!string = - let response = - client.http().get(client.baseurl & "/data/" & $cid & "/network/manifest") +proc setLogLevel*( + client: CodexClient, level: string +): Future[void] {.async: (raises: [CancelledError, HttpError]).} = + let + url = client.baseurl & "/debug/chronicles/loglevel?level=" & level + headers = @[("Content-Type", "text/plain")] + response = await client.post(url, headers = headers, body = "") + assert response.status == 200 - if response.status != "200 OK": - return failure(response.status) +proc uploadRaw*( + client: CodexClient, contents: string, headers: seq[HttpHeaderTuple] = @[] +): Future[HttpClientResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]) +.} = + return client.post(client.baseurl & "/data", body = contents, headers = headers) - success response.body +proc upload*( + client: CodexClient, contents: string +): Future[?!Cid] {.async: (raises: [CancelledError, HttpError]).} = + let response = await client.uploadRaw(contents) + assert response.status == 200 + Cid.init(await response.body).mapFailure -proc downloadNoStream*(client: CodexClient, cid: Cid): ?!string = - let response = client.http().post(client.baseurl & "/data/" & $cid & "/network") +proc upload*( + client: CodexClient, bytes: seq[byte] +): Future[?!Cid] {.async: (raw: true).} = + return client.upload(string.fromBytes(bytes)) - if response.status != "200 OK": - return failure(response.status) - - success response.body +proc downloadRaw*( + client: CodexClient, cid: string, local = false +): Future[HttpClientResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]) +.} = + return + client.get(client.baseurl & "/data/" & cid & (if local: "" else: "/network/stream")) proc downloadBytes*( client: CodexClient, cid: Cid, local = false -): Future[?!seq[byte]] {.async.} = - let uri = client.baseurl & "/data/" & $cid & (if local: "" else: "/network/stream") +): Future[?!seq[byte]] {.async: (raises: [CancelledError, HttpError]).} = + let response = await client.downloadRaw($cid, local = local) - let response = client.http().get(uri) + if response.status != 200: + return failure($response.status) - if response.status != "200 OK": - return failure("fetch failed with status " & $response.status) + success await response.getBodyBytes() - success response.body.toBytes +proc download*( + client: CodexClient, cid: Cid, local = false +): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} = + without response =? await client.downloadBytes(cid, local = local), err: + return failure(err) + return success bytesToString(response) -proc delete*(client: CodexClient, cid: Cid): ?!void = - let - url = client.baseurl & "/data/" & $cid - response = client.http().delete(url) +proc downloadNoStream*( + client: CodexClient, cid: Cid +): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} = + let response = await client.post(client.baseurl & "/data/" & $cid & "/network") - if response.status != "204 No Content": - return failure(response.status) + if response.status != 200: + return failure($response.status) + + success await response.body + +proc downloadManifestOnly*( + client: CodexClient, cid: Cid +): Future[?!string] {.async: (raises: [CancelledError, HttpError]).} = + let response = + await client.get(client.baseurl & "/data/" & $cid & "/network/manifest") + + if response.status != 200: + return failure($response.status) + + success await response.body + +proc deleteRaw*( + client: CodexClient, cid: string +): Future[HttpClientResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]) +.} = + return client.delete(client.baseurl & "/data/" & cid) + +proc delete*( + client: CodexClient, cid: Cid +): Future[?!void] {.async: (raises: [CancelledError, HttpError]).} = + let response = await client.deleteRaw($cid) + + if response.status != 204: + return failure($response.status) success() -proc list*(client: CodexClient): ?!RestContentList = - let url = client.baseurl & "/data" - let response = client.http().get(url) +proc listRaw*( + client: CodexClient +): Future[HttpClientResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]) +.} = + return client.get(client.baseurl & "/data") - if response.status != "200 OK": - return failure(response.status) +proc list*( + client: CodexClient +): Future[?!RestContentList] {.async: (raises: [CancelledError, HttpError]).} = + let response = await client.listRaw() - RestContentList.fromJson(response.body) + if response.status != 200: + return failure($response.status) -proc space*(client: CodexClient): ?!RestRepoStore = + RestContentList.fromJson(await response.body) + +proc space*( + client: CodexClient +): Future[?!RestRepoStore] {.async: (raises: [CancelledError, HttpError]).} = let url = client.baseurl & "/space" - let response = client.http().get(url) + let response = await client.get(url) - if response.status != "200 OK": - return failure(response.status) + if response.status != 200: + return failure($response.status) - RestRepoStore.fromJson(response.body) + RestRepoStore.fromJson(await response.body) proc requestStorageRaw*( client: CodexClient, @@ -128,7 +225,9 @@ proc requestStorageRaw*( expiry: uint64 = 0, nodes: uint = 3, tolerance: uint = 1, -): Response = +): Future[HttpClientResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]) +.} = ## Call request storage REST endpoint ## let url = client.baseurl & "/storage/request/" & $cid @@ -145,7 +244,7 @@ proc requestStorageRaw*( if expiry != 0: json["expiry"] = %($expiry) - return client.http().post(url, $json) + return client.post(url, $json) proc requestStorage*( client: CodexClient, @@ -157,43 +256,45 @@ proc requestStorage*( collateralPerByte: UInt256, nodes: uint = 3, tolerance: uint = 1, -): ?!PurchaseId = +): Future[?!PurchaseId] {.async: (raises: [CancelledError, HttpError]).} = ## Call request storage REST endpoint ## - let response = client.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, - nodes, tolerance, - ) - if response.status != "200 OK": - doAssert(false, response.body) - PurchaseId.fromHex(response.body).catch + let + response = await client.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, + nodes, tolerance, + ) + body = await response.body -proc getPurchase*(client: CodexClient, purchaseId: PurchaseId): ?!RestPurchase = + if response.status != 200: + doAssert(false, body) + PurchaseId.fromHex(body).catch + +proc getPurchase*( + client: CodexClient, purchaseId: PurchaseId +): Future[?!RestPurchase] {.async: (raises: [CancelledError, HttpError]).} = let url = client.baseurl & "/storage/purchases/" & purchaseId.toHex try: - let body = client.http().getContent(url) + let body = await client.getContent(url) return RestPurchase.fromJson(body) except CatchableError as e: return failure e.msg -proc getSalesAgent*(client: CodexClient, slotId: SlotId): ?!RestSalesAgent = +proc getSalesAgent*( + client: CodexClient, slotId: SlotId +): Future[?!RestSalesAgent] {.async: (raises: [CancelledError, HttpError]).} = let url = client.baseurl & "/sales/slots/" & slotId.toHex try: - let body = client.http().getContent(url) + let body = await client.getContent(url) return RestSalesAgent.fromJson(body) except CatchableError as e: return failure e.msg -proc getSlots*(client: CodexClient): ?!seq[Slot] = - let url = client.baseurl & "/sales/slots" - let body = client.http().getContent(url) - seq[Slot].fromJson(body) - proc postAvailability*( client: CodexClient, totalSize, duration: uint64, minPricePerBytePerSecond, totalCollateral: UInt256, -): ?!Availability = +): Future[?!Availability] {.async: (raises: [CancelledError, HttpError]).} = ## Post sales availability endpoint ## let url = client.baseurl & "/sales/availability" @@ -204,17 +305,21 @@ proc postAvailability*( "minPricePerBytePerSecond": minPricePerBytePerSecond, "totalCollateral": totalCollateral, } - let response = client.http().post(url, $json) - doAssert response.status == "201 Created", - "expected 201 Created, got " & response.status & ", body: " & response.body - Availability.fromJson(response.body) + let response = await client.post(url, $json) + let body = await response.body + + doAssert response.status == 201, + "expected 201 Created, got " & $response.status & ", body: " & body + Availability.fromJson(body) proc patchAvailabilityRaw*( client: CodexClient, availabilityId: AvailabilityId, totalSize, freeSize, duration: ?uint64 = uint64.none, minPricePerBytePerSecond, totalCollateral: ?UInt256 = UInt256.none, -): Response = +): Future[HttpClientResponseRef] {. + async: (raw: true, raises: [CancelledError, HttpError]) +.} = ## Updates availability ## let url = client.baseurl & "/sales/availability/" & $availabilityId @@ -237,66 +342,50 @@ proc patchAvailabilityRaw*( if totalCollateral =? totalCollateral: json["totalCollateral"] = %totalCollateral - client.http().patch(url, $json) + client.patch(url, $json) proc patchAvailability*( client: CodexClient, availabilityId: AvailabilityId, totalSize, duration: ?uint64 = uint64.none, minPricePerBytePerSecond, totalCollateral: ?UInt256 = UInt256.none, -): void = - let response = client.patchAvailabilityRaw( +): Future[void] {.async: (raises: [CancelledError, HttpError]).} = + let response = await client.patchAvailabilityRaw( availabilityId, totalSize = totalSize, duration = duration, minPricePerBytePerSecond = minPricePerBytePerSecond, totalCollateral = totalCollateral, ) - doAssert response.status == "200 OK", "expected 200 OK, got " & response.status + doAssert response.status == 200, "expected 200 OK, got " & $response.status -proc getAvailabilities*(client: CodexClient): ?!seq[Availability] = +proc getAvailabilities*( + client: CodexClient +): Future[?!seq[Availability]] {.async: (raises: [CancelledError, HttpError]).} = ## Call sales availability REST endpoint let url = client.baseurl & "/sales/availability" - let body = client.http().getContent(url) + let body = await client.getContent(url) seq[Availability].fromJson(body) proc getAvailabilityReservations*( client: CodexClient, availabilityId: AvailabilityId -): ?!seq[Reservation] = +): Future[?!seq[Reservation]] {.async: (raises: [CancelledError, HttpError]).} = ## Retrieves Availability's Reservations let url = client.baseurl & "/sales/availability/" & $availabilityId & "/reservations" - let body = client.http().getContent(url) + let body = await client.getContent(url) seq[Reservation].fromJson(body) -proc purchaseStateIs*(client: CodexClient, id: PurchaseId, state: string): bool = - client.getPurchase(id).option .? state == some state +proc purchaseStateIs*( + client: CodexClient, id: PurchaseId, state: string +): Future[bool] {.async: (raises: [CancelledError, HttpError]).} = + (await client.getPurchase(id)).option .? state == some state -proc saleStateIs*(client: CodexClient, id: SlotId, state: string): bool = - client.getSalesAgent(id).option .? state == some state +proc saleStateIs*( + client: CodexClient, id: SlotId, state: string +): Future[bool] {.async: (raises: [CancelledError, HttpError]).} = + (await client.getSalesAgent(id)).option .? state == some state -proc requestId*(client: CodexClient, id: PurchaseId): ?RequestId = - return client.getPurchase(id).option .? requestId - -proc uploadRaw*( - client: CodexClient, contents: string, headers = newHttpHeaders() -): Response = - return client.http().request( - client.baseurl & "/data", - body = contents, - httpMethod = HttpPost, - headers = headers, - ) - -proc listRaw*(client: CodexClient): Response = - return client.http().request(client.baseurl & "/data", httpMethod = HttpGet) - -proc downloadRaw*( - client: CodexClient, cid: string, local = false, httpClient = client.http() -): Response = - return httpClient.request( - client.baseurl & "/data/" & cid & (if local: "" else: "/network/stream"), - httpMethod = HttpGet, - ) - -proc deleteRaw*(client: CodexClient, cid: string): Response = - return client.http().request(client.baseurl & "/data/" & cid, httpMethod = HttpDelete) +proc requestId*( + client: CodexClient, id: PurchaseId +): Future[?RequestId] {.async: (raises: [CancelledError, HttpError]).} = + return (await client.getPurchase(id)).option .? requestId diff --git a/tests/integration/codexprocess.nim b/tests/integration/codexprocess.nim index 79d4b040..3eca5b04 100644 --- a/tests/integration/codexprocess.nim +++ b/tests/integration/codexprocess.nim @@ -68,7 +68,7 @@ method stop*(node: CodexProcess) {.async.} = trace "stopping codex client" if client =? node.client: - client.close() + await client.close() node.client = none CodexClient method removeDataDir*(node: CodexProcess) = diff --git a/tests/integration/marketplacesuite.nim b/tests/integration/marketplacesuite.nim index d7502bf4..1e09963b 100644 --- a/tests/integration/marketplacesuite.nim +++ b/tests/integration/marketplacesuite.nim @@ -60,13 +60,13 @@ template marketplacesuite*(name: string, body: untyped) = duration: uint64, collateralPerByte: UInt256, minPricePerBytePerSecond: UInt256, - ) = + ): Future[void] {.async: (raises: [CancelledError, HttpError, ConfigurationError]).} = let totalCollateral = datasetSize.u256 * collateralPerByte # post availability to each provider for i in 0 ..< providers().len: let provider = providers()[i].client - discard provider.postAvailability( + discard await provider.postAvailability( totalSize = datasetSize, duration = duration.uint64, minPricePerBytePerSecond = minPricePerBytePerSecond, @@ -83,16 +83,18 @@ template marketplacesuite*(name: string, body: untyped) = expiry: uint64 = 4.periods, nodes = providers().len, tolerance = 0, - ): Future[PurchaseId] {.async.} = - let id = client.requestStorage( - cid, - expiry = expiry, - duration = duration, - proofProbability = proofProbability, - collateralPerByte = collateralPerByte, - pricePerBytePerSecond = pricePerBytePerSecond, - nodes = nodes.uint, - tolerance = tolerance.uint, + ): Future[PurchaseId] {.async: (raises: [CancelledError, HttpError]).} = + let id = ( + await client.requestStorage( + cid, + expiry = expiry, + duration = duration, + proofProbability = proofProbability, + collateralPerByte = collateralPerByte, + pricePerBytePerSecond = pricePerBytePerSecond, + nodes = nodes.uint, + tolerance = tolerance.uint, + ) ).get return id diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index bade6899..0003b216 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -275,8 +275,10 @@ template multinodesuite*(name: string, body: untyped) = fail() quit(1) - proc updateBootstrapNodes(node: CodexProcess) = - without ninfo =? node.client.info(): + proc updateBootstrapNodes( + node: CodexProcess + ): Future[void] {.async: (raises: [CatchableError]).} = + without ninfo =? await node.client.info(): # raise CatchableError instead of Defect (with .get or !) so we # can gracefully shutdown and prevent zombies raiseMultiNodeSuiteError "Failed to get node info" @@ -315,14 +317,14 @@ template multinodesuite*(name: string, body: untyped) = for config in clients.configs: let node = await startClientNode(config) running.add RunningNode(role: Role.Client, node: node) - CodexProcess(node).updateBootstrapNodes() + await CodexProcess(node).updateBootstrapNodes() if var providers =? nodeConfigs.providers: failAndTeardownOnError "failed to start provider nodes": for config in providers.configs.mitems: let node = await startProviderNode(config) running.add RunningNode(role: Role.Provider, node: node) - CodexProcess(node).updateBootstrapNodes() + await CodexProcess(node).updateBootstrapNodes() if var validators =? nodeConfigs.validators: failAndTeardownOnError "failed to start validator nodes": diff --git a/tests/integration/testblockexpiration.nim b/tests/integration/testblockexpiration.nim index 7e742c2a..6a33f3c6 100644 --- a/tests/integration/testblockexpiration.nim +++ b/tests/integration/testblockexpiration.nim @@ -18,11 +18,11 @@ multinodesuite "Node block expiration tests": let client = clients()[0] let clientApi = client.client - let contentId = clientApi.upload(content).get + let contentId = (await clientApi.upload(content)).get await sleepAsync(2.seconds) - let download = clientApi.download(contentId, local = true) + let download = await clientApi.download(contentId, local = true) check: download.isOk @@ -39,12 +39,12 @@ multinodesuite "Node block expiration tests": let client = clients()[0] let clientApi = client.client - let contentId = clientApi.upload(content).get + let contentId = (await clientApi.upload(content)).get await sleepAsync(3.seconds) - let download = clientApi.download(contentId, local = true) + let download = await clientApi.download(contentId, local = true) check: download.isFailure - download.error.msg == "404 Not Found" + download.error.msg == "404" diff --git a/tests/integration/testecbug.nim b/tests/integration/testecbug.nim index 29a3bc6f..6b86fd29 100644 --- a/tests/integration/testecbug.nim +++ b/tests/integration/testecbug.nim @@ -13,21 +13,18 @@ marketplacesuite "Bug #821 - node crashes during erasure coding": .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log .withLogTopics("node", "erasure", "marketplace").some, - providers: CodexConfigs.init(nodes = 0) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("node", "marketplace", "sales", "reservations", "node", "proving", "clock") - .some, + providers: CodexConfigs.init(nodes = 0).some, ): - let pricePerBytePerSecond = 1.u256 - let duration = 20.periods - let collateralPerByte = 1.u256 - let expiry = 10.periods - let data = await RandomChunker.example(blocks = 8) - let client = clients()[0] - let clientApi = client.client + let + pricePerBytePerSecond = 1.u256 + duration = 20.periods + collateralPerByte = 1.u256 + expiry = 10.periods + data = await RandomChunker.example(blocks = 8) + client = clients()[0] + clientApi = client.client - let cid = clientApi.upload(data).get + let cid = (await clientApi.upload(data)).get var requestId = none RequestId proc onStorageRequested(eventResult: ?!StorageRequested) = @@ -49,9 +46,11 @@ marketplacesuite "Bug #821 - node crashes during erasure coding": check eventually(requestId.isSome, timeout = expiry.int * 1000) - let request = await marketplace.getRequest(requestId.get) - let cidFromRequest = request.content.cid - let downloaded = await clientApi.downloadBytes(cidFromRequest, local = true) + let + request = await marketplace.getRequest(requestId.get) + cidFromRequest = request.content.cid + downloaded = await clientApi.downloadBytes(cidFromRequest, local = true) + check downloaded.isOk check downloaded.get.toHex == data.toHex diff --git a/tests/integration/testmarketplace.nim b/tests/integration/testmarketplace.nim index 727f3fad..dee3645e 100644 --- a/tests/integration/testmarketplace.nim +++ b/tests/integration/testmarketplace.nim @@ -37,15 +37,17 @@ marketplacesuite "Marketplace": let size = 0xFFFFFF.uint64 let data = await RandomChunker.example(blocks = blocks) # host makes storage available - let availability = host.postAvailability( - totalSize = size, - duration = 20 * 60.uint64, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = size.u256 * minPricePerBytePerSecond, + let availability = ( + await host.postAvailability( + totalSize = size, + duration = 20 * 60.uint64, + minPricePerBytePerSecond = minPricePerBytePerSecond, + totalCollateral = size.u256 * minPricePerBytePerSecond, + ) ).get # client requests storage - let cid = client.upload(data).get + let cid = (await client.upload(data)).get let id = await client.requestStorage( cid, duration = 20 * 60.uint64, @@ -57,15 +59,17 @@ marketplacesuite "Marketplace": tolerance = ecTolerance, ) - check eventually(client.purchaseStateIs(id, "started"), timeout = 10 * 60 * 1000) - let purchase = client.getPurchase(id).get + check eventually( + await client.purchaseStateIs(id, "started"), timeout = 10 * 60 * 1000 + ) + let purchase = (await client.getPurchase(id)).get check purchase.error == none string - let availabilities = host.getAvailabilities().get + let availabilities = (await host.getAvailabilities()).get check availabilities.len == 1 let newSize = availabilities[0].freeSize check newSize > 0 and newSize < size - let reservations = host.getAvailabilityReservations(availability.id).get + let reservations = (await host.getAvailabilityReservations(availability.id)).get check reservations.len == 3 check reservations[0].requestId == purchase.requestId @@ -80,15 +84,17 @@ marketplacesuite "Marketplace": # host makes storage available let startBalanceHost = await token.balanceOf(hostAccount) - discard host.postAvailability( - totalSize = size, - duration = 20 * 60.uint64, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = size.u256 * minPricePerBytePerSecond, + discard ( + await host.postAvailability( + totalSize = size, + duration = 20 * 60.uint64, + minPricePerBytePerSecond = minPricePerBytePerSecond, + totalCollateral = size.u256 * minPricePerBytePerSecond, + ) ).get # client requests storage - let cid = client.upload(data).get + let cid = (await client.upload(data)).get let id = await client.requestStorage( cid, duration = duration, @@ -100,8 +106,10 @@ marketplacesuite "Marketplace": tolerance = ecTolerance, ) - check eventually(client.purchaseStateIs(id, "started"), timeout = 10 * 60 * 1000) - let purchase = client.getPurchase(id).get + check eventually( + await client.purchaseStateIs(id, "started"), timeout = 10 * 60 * 1000 + ) + let purchase = (await client.getPurchase(id)).get check purchase.error == none string let clientBalanceBeforeFinished = await token.balanceOf(clientAccount) @@ -158,7 +166,7 @@ marketplacesuite "Marketplace payouts": # provider makes storage available let datasetSize = datasetSize(blocks, ecNodes, ecTolerance) let totalAvailabilitySize = (datasetSize div 2).truncate(uint64) - discard providerApi.postAvailability( + discard await providerApi.postAvailability( # make availability size small enough that we can't fill all the slots, # thus causing a cancellation totalSize = totalAvailabilitySize, @@ -167,7 +175,7 @@ marketplacesuite "Marketplace payouts": totalCollateral = collateralPerByte * totalAvailabilitySize.u256, ) - let cid = clientApi.upload(data).get + let cid = (await clientApi.upload(data)).get var slotIdxFilled = none uint64 proc onSlotFilled(eventResult: ?!SlotFilled) = @@ -189,11 +197,11 @@ marketplacesuite "Marketplace payouts": # wait until one slot is filled check eventually(slotIdxFilled.isSome, timeout = expiry.int * 1000) - let slotId = slotId(!clientApi.requestId(id), !slotIdxFilled) + let slotId = slotId(!(await clientApi.requestId(id)), !slotIdxFilled) # wait until sale is cancelled await ethProvider.advanceTime(expiry.u256) - check eventually providerApi.saleStateIs(slotId, "SaleCancelled") + check eventually await providerApi.saleStateIs(slotId, "SaleCancelled") await advanceToNextPeriod() diff --git a/tests/integration/testproofs.nim b/tests/integration/testproofs.nim index ab29ca4e..b0ede765 100644 --- a/tests/integration/testproofs.nim +++ b/tests/integration/testproofs.nim @@ -42,14 +42,14 @@ marketplacesuite "Hosts submit regular proofs": let data = await RandomChunker.example(blocks = blocks) let datasetSize = datasetSize(blocks = blocks, nodes = ecNodes, tolerance = ecTolerance) - createAvailabilities( + await createAvailabilities( datasetSize.truncate(uint64), duration, collateralPerByte, minPricePerBytePerSecond, ) - let cid = client0.upload(data).get + let cid = (await client0.upload(data)).get let purchaseId = await client0.requestStorage( cid, @@ -59,13 +59,13 @@ marketplacesuite "Hosts submit regular proofs": tolerance = ecTolerance, ) - let purchase = client0.getPurchase(purchaseId).get + let purchase = (await client0.getPurchase(purchaseId)).get check purchase.error == none string let slotSize = slotSize(blocks, ecNodes, ecTolerance) check eventually( - client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000 + await client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000 ) var proofWasSubmitted = false @@ -119,27 +119,29 @@ marketplacesuite "Simulate invalid proofs": let data = await RandomChunker.example(blocks = blocks) let datasetSize = datasetSize(blocks = blocks, nodes = ecNodes, tolerance = ecTolerance) - createAvailabilities( + await createAvailabilities( datasetSize.truncate(uint64), duration, collateralPerByte, minPricePerBytePerSecond, ) - let cid = client0.upload(data).get + let cid = (await client0.upload(data)).get - let purchaseId = await client0.requestStorage( - cid, - expiry = expiry, - duration = duration, - nodes = ecNodes, - tolerance = ecTolerance, - proofProbability = 1.u256, + let purchaseId = ( + await client0.requestStorage( + cid, + expiry = expiry, + duration = duration, + nodes = ecNodes, + tolerance = ecTolerance, + proofProbability = 1.u256, + ) ) - let requestId = client0.requestId(purchaseId).get + let requestId = (await client0.requestId(purchaseId)).get check eventually( - client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000 + await client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000 ) var slotWasFreed = false @@ -182,14 +184,14 @@ marketplacesuite "Simulate invalid proofs": let data = await RandomChunker.example(blocks = blocks) let datasetSize = datasetSize(blocks = blocks, nodes = ecNodes, tolerance = ecTolerance) - createAvailabilities( + await createAvailabilities( datasetSize.truncate(uint64), duration, collateralPerByte, minPricePerBytePerSecond, ) - let cid = client0.upload(data).get + let cid = (await client0.upload(data)).get let purchaseId = await client0.requestStorage( cid, @@ -199,7 +201,7 @@ marketplacesuite "Simulate invalid proofs": tolerance = ecTolerance, proofProbability = 1.u256, ) - let requestId = client0.requestId(purchaseId).get + let requestId = (await client0.requestId(purchaseId)).get var slotWasFilled = false proc onSlotFilled(eventResult: ?!SlotFilled) = diff --git a/tests/integration/testpurchasing.nim b/tests/integration/testpurchasing.nim index 4eb5c775..e5adebe2 100644 --- a/tests/integration/testpurchasing.nim +++ b/tests/integration/testpurchasing.nim @@ -8,22 +8,26 @@ import ../examples twonodessuite "Purchasing": test "node handles storage request", twoNodesConfig: let data = await RandomChunker.example(blocks = 2) - let cid = client1.upload(data).get - let id1 = client1.requestStorage( - cid, - duration = 100.uint64, - pricePerBytePerSecond = 1.u256, - proofProbability = 3.u256, - expiry = 10.uint64, - collateralPerByte = 1.u256, + let cid = (await client1.upload(data)).get + let id1 = ( + await client1.requestStorage( + cid, + duration = 100.uint64, + pricePerBytePerSecond = 1.u256, + proofProbability = 3.u256, + expiry = 10.uint64, + collateralPerByte = 1.u256, + ) ).get - let id2 = client1.requestStorage( - cid, - duration = 400.uint64, - pricePerBytePerSecond = 2.u256, - proofProbability = 6.u256, - expiry = 10.uint64, - collateralPerByte = 2.u256, + let id2 = ( + await client1.requestStorage( + cid, + duration = 400.uint64, + pricePerBytePerSecond = 2.u256, + proofProbability = 6.u256, + expiry = 10.uint64, + collateralPerByte = 2.u256, + ) ).get check id1 != id2 @@ -34,19 +38,21 @@ twonodessuite "Purchasing": rng, size = DefaultBlockSize * 2, chunkSize = DefaultBlockSize * 2 ) let data = await chunker.getBytes() - let cid = client1.upload(byteutils.toHex(data)).get - let id = client1.requestStorage( - cid, - duration = 100.uint64, - pricePerBytePerSecond = 1.u256, - proofProbability = 3.u256, - expiry = 30.uint64, - collateralPerByte = 1.u256, - nodes = 3, - tolerance = 1, + let cid = (await client1.upload(byteutils.toHex(data))).get + let id = ( + await client1.requestStorage( + cid, + duration = 100.uint64, + pricePerBytePerSecond = 1.u256, + proofProbability = 3.u256, + expiry = 30.uint64, + collateralPerByte = 1.u256, + nodes = 3, + tolerance = 1, + ) ).get - let request = client1.getPurchase(id).get.request.get + let request = (await client1.getPurchase(id)).get.request.get check request.content.cid.data.buffer.len > 0 check request.ask.duration == 100.uint64 @@ -75,23 +81,29 @@ twonodessuite "Purchasing": test "node remembers purchase status after restart", twoNodesConfig: let data = await RandomChunker.example(blocks = 2) - let cid = client1.upload(data).get - let id = client1.requestStorage( - cid, - duration = 10 * 60.uint64, - pricePerBytePerSecond = 1.u256, - proofProbability = 3.u256, - expiry = 5 * 60.uint64, - collateralPerByte = 1.u256, - nodes = 3.uint, - tolerance = 1.uint, + let cid = (await client1.upload(data)).get + let id = ( + await client1.requestStorage( + cid, + duration = 10 * 60.uint64, + pricePerBytePerSecond = 1.u256, + proofProbability = 3.u256, + expiry = 5 * 60.uint64, + collateralPerByte = 1.u256, + nodes = 3.uint, + tolerance = 1.uint, + ) ).get - check eventually(client1.purchaseStateIs(id, "submitted"), timeout = 3 * 60 * 1000) + check eventually( + await client1.purchaseStateIs(id, "submitted"), timeout = 3 * 60 * 1000 + ) await node1.restart() - check eventually(client1.purchaseStateIs(id, "submitted"), timeout = 3 * 60 * 1000) - let request = client1.getPurchase(id).get.request.get + check eventually( + await client1.purchaseStateIs(id, "submitted"), timeout = 3 * 60 * 1000 + ) + let request = (await client1.getPurchase(id)).get.request.get check request.ask.duration == (10 * 60).uint64 check request.ask.pricePerBytePerSecond == 1.u256 check request.ask.proofProbability == 3.u256 @@ -102,19 +114,19 @@ twonodessuite "Purchasing": test "node requires expiry and its value to be in future", twoNodesConfig: let data = await RandomChunker.example(blocks = 2) - let cid = client1.upload(data).get + let cid = (await client1.upload(data)).get - let responseMissing = client1.requestStorageRaw( + let responseMissing = await client1.requestStorageRaw( cid, duration = 1.uint64, pricePerBytePerSecond = 1.u256, proofProbability = 3.u256, collateralPerByte = 1.u256, ) - check responseMissing.status == "400 Bad Request" - check responseMissing.body == "Expiry required" + check responseMissing.status == 400 + check (await responseMissing.body) == "Expiry required" - let responseBefore = client1.requestStorageRaw( + let responseBefore = await client1.requestStorageRaw( cid, duration = 10.uint64, pricePerBytePerSecond = 1.u256, @@ -122,6 +134,6 @@ twonodessuite "Purchasing": collateralPerByte = 1.u256, expiry = 10.uint64, ) - check responseBefore.status == "400 Bad Request" + check responseBefore.status == 400 check "Expiry needs value bigger then zero and smaller then the request's duration" in - responseBefore.body + (await responseBefore.body) diff --git a/tests/integration/testrestapi.nim b/tests/integration/testrestapi.nim index 7164372b..761eda31 100644 --- a/tests/integration/testrestapi.nim +++ b/tests/integration/testrestapi.nim @@ -1,4 +1,3 @@ -import std/httpclient import std/importutils import std/net import std/sequtils @@ -14,29 +13,31 @@ import json twonodessuite "REST API": test "nodes can print their peer information", twoNodesConfig: - check !client1.info() != !client2.info() + check !(await client1.info()) != !(await client2.info()) test "nodes can set chronicles log level", twoNodesConfig: - client1.setLogLevel("DEBUG;TRACE:codex") + await client1.setLogLevel("DEBUG;TRACE:codex") test "node accepts file uploads", twoNodesConfig: - let cid1 = client1.upload("some file contents").get - let cid2 = client1.upload("some other contents").get + let cid1 = (await client1.upload("some file contents")).get + let cid2 = (await client1.upload("some other contents")).get check cid1 != cid2 test "node shows used and available space", twoNodesConfig: - discard client1.upload("some file contents").get + discard (await client1.upload("some file contents")).get let totalSize = 12.uint64 let minPricePerBytePerSecond = 1.u256 let totalCollateral = totalSize.u256 * minPricePerBytePerSecond - discard client1.postAvailability( - totalSize = totalSize, - duration = 2.uint64, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = totalCollateral, + discard ( + await client1.postAvailability( + totalSize = totalSize, + duration = 2.uint64, + minPricePerBytePerSecond = minPricePerBytePerSecond, + totalCollateral = totalCollateral, + ) ).get - let space = client1.space().tryGet() + let space = (await client1.space()).tryGet() check: space.totalBlocks == 2 space.quotaMaxBytes == 21474836480.NBytes @@ -47,48 +48,52 @@ twonodessuite "REST API": let content1 = "some file contents" let content2 = "some other contents" - let cid1 = client1.upload(content1).get - let cid2 = client1.upload(content2).get - let list = client1.list().get + let cid1 = (await client1.upload(content1)).get + let cid2 = (await client1.upload(content2)).get + let list = (await client1.list()).get check: [cid1, cid2].allIt(it in list.content.mapIt(it.cid)) test "request storage fails for datasets that are too small", twoNodesConfig: - let cid = client1.upload("some file contents").get - let response = client1.requestStorageRaw( - cid, - duration = 10.uint64, - pricePerBytePerSecond = 1.u256, - proofProbability = 3.u256, - collateralPerByte = 1.u256, - expiry = 9.uint64, + let cid = (await client1.upload("some file contents")).get + let response = ( + await client1.requestStorageRaw( + cid, + duration = 10.uint64, + pricePerBytePerSecond = 1.u256, + proofProbability = 3.u256, + collateralPerByte = 1.u256, + expiry = 9.uint64, + ) ) check: - response.status == "400 Bad Request" - response.body == + response.status == 400 + (await response.body) == "Dataset too small for erasure parameters, need at least " & $(2 * DefaultBlockSize.int) & " bytes" test "request storage succeeds for sufficiently sized datasets", twoNodesConfig: let data = await RandomChunker.example(blocks = 2) - let cid = client1.upload(data).get - let response = client1.requestStorageRaw( - cid, - duration = 10.uint64, - pricePerBytePerSecond = 1.u256, - proofProbability = 3.u256, - collateralPerByte = 1.u256, - expiry = 9.uint64, + let cid = (await client1.upload(data)).get + let response = ( + await client1.requestStorageRaw( + cid, + duration = 10.uint64, + pricePerBytePerSecond = 1.u256, + proofProbability = 3.u256, + collateralPerByte = 1.u256, + expiry = 9.uint64, + ) ) check: - response.status == "200 OK" + response.status == 200 test "request storage fails if tolerance is zero", twoNodesConfig: let data = await RandomChunker.example(blocks = 2) - let cid = client1.upload(data).get + let cid = (await client1.upload(data)).get let duration = 100.uint64 let pricePerBytePerSecond = 1.u256 let proofProbability = 3.u256 @@ -97,17 +102,19 @@ twonodessuite "REST API": let nodes = 3 let tolerance = 0 - var responseBefore = client1.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, - nodes.uint, tolerance.uint, + var responseBefore = ( + await client1.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, + expiry, nodes.uint, tolerance.uint, + ) ) - check responseBefore.status == "400 Bad Request" - check responseBefore.body == "Tolerance needs to be bigger then zero" + check responseBefore.status == 400 + check (await responseBefore.body) == "Tolerance needs to be bigger then zero" test "request storage fails if duration exceeds limit", twoNodesConfig: let data = await RandomChunker.example(blocks = 2) - let cid = client1.upload(data).get + let cid = (await client1.upload(data)).get let duration = (31 * 24 * 60 * 60).uint64 # 31 days TODO: this should not be hardcoded, but waits for https://github.com/codex-storage/nim-codex/issues/1056 let proofProbability = 3.u256 @@ -117,17 +124,19 @@ twonodessuite "REST API": let tolerance = 2 let pricePerBytePerSecond = 1.u256 - var responseBefore = client1.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, - nodes.uint, tolerance.uint, + var responseBefore = ( + await client1.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, + expiry, nodes.uint, tolerance.uint, + ) ) - check responseBefore.status == "400 Bad Request" - check "Duration exceeds limit of" in responseBefore.body + check responseBefore.status == 400 + check "Duration exceeds limit of" in (await responseBefore.body) test "request storage fails if nodes and tolerance aren't correct", twoNodesConfig: let data = await RandomChunker.example(blocks = 2) - let cid = client1.upload(data).get + let cid = (await client1.upload(data)).get let duration = 100.uint64 let pricePerBytePerSecond = 1.u256 let proofProbability = 3.u256 @@ -138,19 +147,21 @@ twonodessuite "REST API": for ecParam in ecParams: let (nodes, tolerance) = ecParam - var responseBefore = client1.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, - expiry, nodes.uint, tolerance.uint, + var responseBefore = ( + await client1.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, + expiry, nodes.uint, tolerance.uint, + ) ) - check responseBefore.status == "400 Bad Request" - check responseBefore.body == + check responseBefore.status == 400 + check (await responseBefore.body) == "Invalid parameters: parameters must satify `1 < (nodes - tolerance) ≥ tolerance`" test "request storage fails if tolerance > nodes (underflow protection)", twoNodesConfig: let data = await RandomChunker.example(blocks = 2) - let cid = client1.upload(data).get + let cid = (await client1.upload(data)).get let duration = 100.uint64 let pricePerBytePerSecond = 1.u256 let proofProbability = 3.u256 @@ -161,13 +172,15 @@ twonodessuite "REST API": for ecParam in ecParams: let (nodes, tolerance) = ecParam - var responseBefore = client1.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, - expiry, nodes.uint, tolerance.uint, + var responseBefore = ( + await client1.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, + expiry, nodes.uint, tolerance.uint, + ) ) - check responseBefore.status == "400 Bad Request" - check responseBefore.body == + check responseBefore.status == 400 + check (await responseBefore.body) == "Invalid parameters: `tolerance` cannot be greater than `nodes`" for ecParams in @[ @@ -177,70 +190,69 @@ twonodessuite "REST API": test "request storage succeeds if nodes and tolerance within range " & fmt"({minBlocks=}, {nodes=}, {tolerance=})", twoNodesConfig: let data = await RandomChunker.example(blocks = minBlocks) - let cid = client1.upload(data).get + let cid = (await client1.upload(data)).get let duration = 100.uint64 let pricePerBytePerSecond = 1.u256 let proofProbability = 3.u256 let expiry = 30.uint64 let collateralPerByte = 1.u256 - var responseBefore = client1.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, - expiry, nodes.uint, tolerance.uint, + var responseBefore = ( + await client1.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, + expiry, nodes.uint, tolerance.uint, + ) ) - check responseBefore.status == "200 OK" + check responseBefore.status == 200 test "node accepts file uploads with content type", twoNodesConfig: - let headers = newHttpHeaders({"Content-Type": "text/plain"}) - let response = client1.uploadRaw("some file contents", headers) + let headers = @[("Content-Type", "text/plain")] + let response = await client1.uploadRaw("some file contents", headers) - check response.status == "200 OK" - check response.body != "" + check response.status == 200 + check (await response.body) != "" test "node accepts file uploads with content disposition", twoNodesConfig: - let headers = - newHttpHeaders({"Content-Disposition": "attachment; filename=\"example.txt\""}) - let response = client1.uploadRaw("some file contents", headers) + let headers = @[("Content-Disposition", "attachment; filename=\"example.txt\"")] + let response = await client1.uploadRaw("some file contents", headers) - check response.status == "200 OK" - check response.body != "" + check response.status == 200 + check (await response.body) != "" test "node accepts file uploads with content disposition without filename", twoNodesConfig: - let headers = newHttpHeaders({"Content-Disposition": "attachment"}) - let response = client1.uploadRaw("some file contents", headers) + let headers = @[("Content-Disposition", "attachment")] + let response = await client1.uploadRaw("some file contents", headers) - check response.status == "200 OK" - check response.body != "" + check response.status == 200 + check (await response.body) != "" test "upload fails if content disposition contains bad filename", twoNodesConfig: - let headers = - newHttpHeaders({"Content-Disposition": "attachment; filename=\"exam*ple.txt\""}) - let response = client1.uploadRaw("some file contents", headers) + let headers = @[("Content-Disposition", "attachment; filename=\"exam*ple.txt\"")] + let response = await client1.uploadRaw("some file contents", headers) - check response.status == "422 Unprocessable Entity" - check response.body == "The filename is not valid." + check response.status == 422 + check (await response.body) == "The filename is not valid." test "upload fails if content type is invalid", twoNodesConfig: - let headers = newHttpHeaders({"Content-Type": "hello/world"}) - let response = client1.uploadRaw("some file contents", headers) + let headers = @[("Content-Type", "hello/world")] + let response = await client1.uploadRaw("some file contents", headers) - check response.status == "422 Unprocessable Entity" - check response.body == "The MIME type 'hello/world' is not valid." + check response.status == 422 + check (await response.body) == "The MIME type 'hello/world' is not valid." test "node retrieve the metadata", twoNodesConfig: - let headers = newHttpHeaders( - { - "Content-Type": "text/plain", - "Content-Disposition": "attachment; filename=\"example.txt\"", - } - ) - let uploadResponse = client1.uploadRaw("some file contents", headers) - let cid = uploadResponse.body - let listResponse = client1.listRaw() + let headers = + @[ + ("Content-Type", "text/plain"), + ("Content-Disposition", "attachment; filename=\"example.txt\""), + ] + let uploadResponse = await client1.uploadRaw("some file contents", headers) + let cid = await uploadResponse.body + let listResponse = await client1.listRaw() - let jsonData = parseJson(listResponse.body) + let jsonData = parseJson(await listResponse.body) check jsonData.hasKey("content") == true @@ -256,83 +268,79 @@ twonodessuite "REST API": check manifest["mimetype"].getStr() == "text/plain" test "node set the headers when for download", twoNodesConfig: - let headers = newHttpHeaders( - { - "Content-Disposition": "attachment; filename=\"example.txt\"", - "Content-Type": "text/plain", - } - ) + let headers = + @[ + ("Content-Disposition", "attachment; filename=\"example.txt\""), + ("Content-Type", "text/plain"), + ] - let uploadResponse = client1.uploadRaw("some file contents", headers) - let cid = uploadResponse.body + let uploadResponse = await client1.uploadRaw("some file contents", headers) + let cid = await uploadResponse.body - check uploadResponse.status == "200 OK" + check uploadResponse.status == 200 - let response = client1.downloadRaw(cid) + let response = await client1.downloadRaw(cid) - check response.status == "200 OK" - check response.headers.hasKey("Content-Type") == true - check response.headers["Content-Type"] == "text/plain" - check response.headers.hasKey("Content-Disposition") == true - check response.headers["Content-Disposition"] == + check response.status == 200 + check "Content-Type" in response.headers + check response.headers.getString("Content-Type") == "text/plain" + check "Content-Disposition" in response.headers + check response.headers.getString("Content-Disposition") == "attachment; filename=\"example.txt\"" let local = true - let localResponse = client1.downloadRaw(cid, local) + let localResponse = await client1.downloadRaw(cid, local) - check localResponse.status == "200 OK" - check localResponse.headers.hasKey("Content-Type") == true - check localResponse.headers["Content-Type"] == "text/plain" - check localResponse.headers.hasKey("Content-Disposition") == true - check localResponse.headers["Content-Disposition"] == + check localResponse.status == 200 + check "Content-Type" in localResponse.headers + check localResponse.headers.getString("Content-Type") == "text/plain" + check "Content-Disposition" in localResponse.headers + check localResponse.headers.getString("Content-Disposition") == "attachment; filename=\"example.txt\"" test "should delete a dataset when requested", twoNodesConfig: - let cid = client1.upload("some file contents").get + let cid = (await client1.upload("some file contents")).get - var response = client1.downloadRaw($cid, local = true) - check response.body == "some file contents" + var response = await client1.downloadRaw($cid, local = true) + check (await response.body) == "some file contents" - client1.delete(cid).get + (await client1.delete(cid)).get - response = client1.downloadRaw($cid, local = true) - check response.status == "404 Not Found" + response = await client1.downloadRaw($cid, local = true) + check response.status == 404 test "should return 200 when attempting delete of non-existing block", twoNodesConfig: - let response = client1.deleteRaw($(Cid.example())) - check response.status == "204 No Content" + let response = await client1.deleteRaw($(Cid.example())) + check response.status == 204 test "should return 200 when attempting delete of non-existing dataset", twoNodesConfig: let cid = Manifest.example().makeManifestBlock().get.cid - let response = client1.deleteRaw($cid) - check response.status == "204 No Content" + let response = await client1.deleteRaw($cid) + check response.status == 204 test "should not crash if the download stream is closed before download completes", twoNodesConfig: - privateAccess(client1.type) - privateAccess(client1.http.type) + # FIXME this is not a good test. For some reason, to get this to fail, I have to + # store content that is several times the default stream buffer size, otherwise + # the test will succeed even when the bug is present. Since this is probably some + # setting that is internal to chronos, it might change in future versions, + # invalidating this test. Works on Chronos 4.0.3. - let cid = client1.upload(repeat("some file contents", 1000)).get - let httpClient = client1.http() + let + contents = repeat("b", DefaultStreamBufferSize * 10) + cid = (await client1.upload(contents)).get + response = await client1.downloadRaw($cid) - try: - # Sadly, there's no high level API for preventing the client from - # consuming the whole response, and we need to close the socket - # before that happens if we want to trigger the bug, so we need to - # resort to this. - httpClient.getBody = false - let response = client1.downloadRaw($cid, httpClient = httpClient) + let reader = response.getBodyReader() - # Read 4 bytes from the stream just to make sure we actually - # receive some data. - let data = httpClient.socket.recv(4) - check data.len == 4 + # Read 4 bytes from the stream just to make sure we actually + # receive some data. + check (bytesToString await reader.read(4)) == "bbbb" - # Prematurely closes the connection. - httpClient.close() - finally: - httpClient.getBody = true + # Abruptly closes the stream (we have to dig all the way to the transport + # or Chronos will close things "nicely"). + response.connection.reader.tsource.close() - let response = client1.downloadRaw($cid, httpClient = httpClient) - check response.body == repeat("some file contents", 1000) + let response2 = await client1.downloadRaw($cid) + check (await response2.body) == contents diff --git a/tests/integration/testsales.nim b/tests/integration/testsales.nim index 6c5c30d5..2d7a199c 100644 --- a/tests/integration/testsales.nim +++ b/tests/integration/testsales.nim @@ -30,54 +30,63 @@ multinodesuite "Sales": client = clients()[0].client test "node handles new storage availability", salesConfig: - let availability1 = host.postAvailability( - totalSize = 1.uint64, - duration = 2.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 4.u256, + let availability1 = ( + await host.postAvailability( + totalSize = 1.uint64, + duration = 2.uint64, + minPricePerBytePerSecond = 3.u256, + totalCollateral = 4.u256, + ) ).get - let availability2 = host.postAvailability( - totalSize = 4.uint64, - duration = 5.uint64, - minPricePerBytePerSecond = 6.u256, - totalCollateral = 7.u256, + let availability2 = ( + await host.postAvailability( + totalSize = 4.uint64, + duration = 5.uint64, + minPricePerBytePerSecond = 6.u256, + totalCollateral = 7.u256, + ) ).get check availability1 != availability2 test "node lists storage that is for sale", salesConfig: - let availability = host.postAvailability( - totalSize = 1.uint64, - duration = 2.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 4.u256, + let availability = ( + await host.postAvailability( + totalSize = 1.uint64, + duration = 2.uint64, + minPricePerBytePerSecond = 3.u256, + totalCollateral = 4.u256, + ) ).get - check availability in host.getAvailabilities().get + check availability in (await host.getAvailabilities()).get test "updating non-existing availability", salesConfig: - let nonExistingResponse = host.patchAvailabilityRaw( + let nonExistingResponse = await host.patchAvailabilityRaw( AvailabilityId.example, duration = 100.uint64.some, minPricePerBytePerSecond = 2.u256.some, totalCollateral = 200.u256.some, ) - check nonExistingResponse.status == "404 Not Found" + check nonExistingResponse.status == 404 test "updating availability", salesConfig: - let availability = host.postAvailability( - totalSize = 140000.uint64, - duration = 200.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 300.u256, + let availability = ( + await host.postAvailability( + totalSize = 140000.uint64, + duration = 200.uint64, + minPricePerBytePerSecond = 3.u256, + totalCollateral = 300.u256, + ) ).get - host.patchAvailability( + await host.patchAvailability( availability.id, duration = 100.uint64.some, minPricePerBytePerSecond = 2.u256.some, totalCollateral = 200.u256.some, ) - let updatedAvailability = (host.getAvailabilities().get).findItem(availability).get + let updatedAvailability = + ((await host.getAvailabilities()).get).findItem(availability).get check updatedAvailability.duration == 100.uint64 check updatedAvailability.minPricePerBytePerSecond == 2 check updatedAvailability.totalCollateral == 200 @@ -85,26 +94,31 @@ multinodesuite "Sales": check updatedAvailability.freeSize == 140000.uint64 test "updating availability - freeSize is not allowed to be changed", salesConfig: - let availability = host.postAvailability( - totalSize = 140000.uint64, - duration = 200.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 300.u256, + let availability = ( + await host.postAvailability( + totalSize = 140000.uint64, + duration = 200.uint64, + minPricePerBytePerSecond = 3.u256, + totalCollateral = 300.u256, + ) ).get let freeSizeResponse = - host.patchAvailabilityRaw(availability.id, freeSize = 110000.uint64.some) - check freeSizeResponse.status == "400 Bad Request" - check "not allowed" in freeSizeResponse.body + await host.patchAvailabilityRaw(availability.id, freeSize = 110000.uint64.some) + check freeSizeResponse.status == 400 + check "not allowed" in (await freeSizeResponse.body) test "updating availability - updating totalSize", salesConfig: - let availability = host.postAvailability( - totalSize = 140000.uint64, - duration = 200.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 300.u256, + let availability = ( + await host.postAvailability( + totalSize = 140000.uint64, + duration = 200.uint64, + minPricePerBytePerSecond = 3.u256, + totalCollateral = 300.u256, + ) ).get - host.patchAvailability(availability.id, totalSize = 100000.uint64.some) - let updatedAvailability = (host.getAvailabilities().get).findItem(availability).get + await host.patchAvailability(availability.id, totalSize = 100000.uint64.some) + let updatedAvailability = + ((await host.getAvailabilities()).get).findItem(availability).get check updatedAvailability.totalSize == 100000 check updatedAvailability.freeSize == 100000 @@ -115,38 +129,51 @@ multinodesuite "Sales": let minPricePerBytePerSecond = 3.u256 let collateralPerByte = 1.u256 let totalCollateral = originalSize.u256 * collateralPerByte - let availability = host.postAvailability( - totalSize = originalSize, - duration = 20 * 60.uint64, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = totalCollateral, + let availability = ( + await host.postAvailability( + totalSize = originalSize, + duration = 20 * 60.uint64, + minPricePerBytePerSecond = minPricePerBytePerSecond, + totalCollateral = totalCollateral, + ) ).get # Lets create storage request that will utilize some of the availability's space - let cid = client.upload(data).get - let id = client.requestStorage( - cid, - duration = 20 * 60.uint64, - pricePerBytePerSecond = minPricePerBytePerSecond, - proofProbability = 3.u256, - expiry = (10 * 60).uint64, - collateralPerByte = collateralPerByte, - nodes = 3, - tolerance = 1, + let cid = (await client.upload(data)).get + let id = ( + await client.requestStorage( + cid, + duration = 20 * 60.uint64, + pricePerBytePerSecond = minPricePerBytePerSecond, + proofProbability = 3.u256, + expiry = (10 * 60).uint64, + collateralPerByte = collateralPerByte, + nodes = 3, + tolerance = 1, + ) ).get - check eventually(client.purchaseStateIs(id, "started"), timeout = 10 * 60 * 1000) - let updatedAvailability = (host.getAvailabilities().get).findItem(availability).get + check eventually( + await client.purchaseStateIs(id, "started"), timeout = 10 * 60 * 1000 + ) + let updatedAvailability = + ((await host.getAvailabilities()).get).findItem(availability).get check updatedAvailability.totalSize != updatedAvailability.freeSize let utilizedSize = updatedAvailability.totalSize - updatedAvailability.freeSize - let totalSizeResponse = - host.patchAvailabilityRaw(availability.id, totalSize = (utilizedSize - 1).some) - check totalSizeResponse.status == "400 Bad Request" - check "totalSize must be larger then current totalSize" in totalSizeResponse.body + let totalSizeResponse = ( + await host.patchAvailabilityRaw( + availability.id, totalSize = (utilizedSize - 1).some + ) + ) + check totalSizeResponse.status == 400 + check "totalSize must be larger then current totalSize" in + (await totalSizeResponse.body) - host.patchAvailability(availability.id, totalSize = (originalSize + 20000).some) + await host.patchAvailability( + availability.id, totalSize = (originalSize + 20000).some + ) let newUpdatedAvailability = - (host.getAvailabilities().get).findItem(availability).get + ((await host.getAvailabilities()).get).findItem(availability).get check newUpdatedAvailability.totalSize == originalSize + 20000 check newUpdatedAvailability.freeSize - updatedAvailability.freeSize == 20000 diff --git a/tests/integration/testupdownload.nim b/tests/integration/testupdownload.nim index 05d3a496..24e6039c 100644 --- a/tests/integration/testupdownload.nim +++ b/tests/integration/testupdownload.nim @@ -9,11 +9,11 @@ twonodessuite "Uploads and downloads": let content1 = "some file contents" let content2 = "some other contents" - let cid1 = client1.upload(content1).get - let cid2 = client2.upload(content2).get + let cid1 = (await client1.upload(content1)).get + let cid2 = (await client2.upload(content2)).get - let resp1 = client1.download(cid1, local = true).get - let resp2 = client2.download(cid2, local = true).get + let resp1 = (await client1.download(cid1, local = true)).get + let resp2 = (await client2.download(cid2, local = true)).get check: content1 == resp1 @@ -23,11 +23,11 @@ twonodessuite "Uploads and downloads": let content1 = "some file contents" let content2 = "some other contents" - let cid1 = client1.upload(content1).get - let cid2 = client2.upload(content2).get + let cid1 = (await client1.upload(content1)).get + let cid2 = (await client2.upload(content2)).get - let resp2 = client1.download(cid2, local = false).get - let resp1 = client2.download(cid1, local = false).get + let resp2 = (await client1.download(cid2, local = false)).get + let resp1 = (await client2.download(cid1, local = false)).get check: content1 == resp1 @@ -35,11 +35,12 @@ twonodessuite "Uploads and downloads": test "node fails retrieving non-existing local file", twoNodesConfig: let content1 = "some file contents" - let cid1 = client1.upload(content1).get # upload to first node - let resp2 = client2.download(cid1, local = true) # try retrieving from second node + let cid1 = (await client1.upload(content1)).get # upload to first node + let resp2 = + await client2.download(cid1, local = true) # try retrieving from second node check: - resp2.error.msg == "404 Not Found" + resp2.error.msg == "404" proc checkRestContent(cid: Cid, content: ?!string) = let c = content.tryGet() @@ -67,26 +68,28 @@ twonodessuite "Uploads and downloads": test "node allows downloading only manifest", twoNodesConfig: let content1 = "some file contents" - let cid1 = client1.upload(content1).get + let cid1 = (await client1.upload(content1)).get - let resp2 = client1.downloadManifestOnly(cid1) + let resp2 = await client1.downloadManifestOnly(cid1) checkRestContent(cid1, resp2) test "node allows downloading content without stream", twoNodesConfig: - let content1 = "some file contents" - let cid1 = client1.upload(content1).get + let + content1 = "some file contents" + cid1 = (await client1.upload(content1)).get + resp1 = await client2.downloadNoStream(cid1) - let resp1 = client2.downloadNoStream(cid1) checkRestContent(cid1, resp1) - let resp2 = client2.download(cid1, local = true).get + + let resp2 = (await client2.download(cid1, local = true)).get check: content1 == resp2 test "reliable transfer test", twoNodesConfig: proc transferTest(a: CodexClient, b: CodexClient) {.async.} = let data = await RandomChunker.example(blocks = 8) - let cid = a.upload(data).get - let response = b.download(cid).get + let cid = (await a.upload(data)).get + let response = (await b.download(cid)).get check: @response.mapIt(it.byte) == data diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 7f4bc851..0d1a50e8 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -99,14 +99,14 @@ marketplacesuite "Validation": let data = await RandomChunker.example(blocks = blocks) let datasetSize = datasetSize(blocks = blocks, nodes = ecNodes, tolerance = ecTolerance) - createAvailabilities( + await createAvailabilities( datasetSize.truncate(uint64), duration, collateralPerByte, minPricePerBytePerSecond, ) - let cid = client0.upload(data).get + let cid = (await client0.upload(data)).get let purchaseId = await client0.requestStorage( cid, expiry = expiry, @@ -115,12 +115,12 @@ marketplacesuite "Validation": tolerance = ecTolerance, proofProbability = proofProbability, ) - let requestId = client0.requestId(purchaseId).get + let requestId = (await client0.requestId(purchaseId)).get debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId if not eventuallyS( - client0.purchaseStateIs(purchaseId, "started"), + await client0.purchaseStateIs(purchaseId, "started"), timeout = (expiry + 60).int, step = 5, ): @@ -169,14 +169,14 @@ marketplacesuite "Validation": let data = await RandomChunker.example(blocks = blocks) let datasetSize = datasetSize(blocks = blocks, nodes = ecNodes, tolerance = ecTolerance) - createAvailabilities( + await createAvailabilities( datasetSize.truncate(uint64), duration, collateralPerByte, minPricePerBytePerSecond, ) - let cid = client0.upload(data).get + let cid = (await client0.upload(data)).get let purchaseId = await client0.requestStorage( cid, expiry = expiry, @@ -185,12 +185,12 @@ marketplacesuite "Validation": tolerance = ecTolerance, proofProbability = proofProbability, ) - let requestId = client0.requestId(purchaseId).get + let requestId = (await client0.requestId(purchaseId)).get debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId if not eventuallyS( - client0.purchaseStateIs(purchaseId, "started"), + await client0.purchaseStateIs(purchaseId, "started"), timeout = (expiry + 60).int, step = 5, ): diff --git a/tests/testTaiko.nim b/tests/testTaiko.nim index 8036e8a3..b1555bfb 100644 --- a/tests/testTaiko.nim +++ b/tests/testTaiko.nim @@ -24,7 +24,7 @@ suite "Taiko L2 Integration Tests": ) node1.waitUntilStarted() - let bootstrap = (!node1.client.info())["spr"].getStr() + let bootstrap = (!(await node1.client.info()))["spr"].getStr() node2 = startNode( [ From 9d7b521519329766cee675ece4013614adc7c6ad Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 18 Mar 2025 08:06:46 +0100 Subject: [PATCH 02/24] chore: add missing custom errors (#1134) * Add missing custom errors * Separate mock state errors * Remove the Option in the error setters * Wrap the contract errors in MarketError * Remove async raises (needs to address it in another PR) * Wrap contract errors into specific error types * Rename SlotNotFreeError to SlotStateMismatchError --- codex/contracts/market.nim | 36 +++++++++++------ codex/contracts/marketplace.nim | 1 + codex/market.nim | 2 + codex/sales/states/filling.nim | 12 +++--- codex/sales/states/slotreserving.nim | 9 ++--- tests/codex/helpers/mockmarket.nim | 23 +++++++++-- tests/codex/sales/states/testfilling.nim | 40 ++++++++++++++++++- .../codex/sales/states/testslotreserving.nim | 9 +++-- 8 files changed, 100 insertions(+), 32 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 58495b45..0b846099 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -249,10 +249,16 @@ method fillSlot( requestId slotIndex - await market.approveFunds(collateral) - trace "calling fillSlot on contract" - discard await market.contract.fillSlot(requestId, slotIndex, proof).confirm(1) - trace "fillSlot transaction completed" + try: + await market.approveFunds(collateral) + trace "calling fillSlot on contract" + discard await market.contract.fillSlot(requestId, slotIndex, proof).confirm(1) + trace "fillSlot transaction completed" + except Marketplace_SlotNotFree as parent: + raise newException( + SlotStateMismatchError, "Failed to fill slot because the slot is not free", + parent, + ) method freeSlot*(market: OnChainMarket, slotId: SlotId) {.async.} = convertEthersError("Failed to free slot"): @@ -327,14 +333,20 @@ method reserveSlot*( market: OnChainMarket, requestId: RequestId, slotIndex: uint64 ) {.async.} = convertEthersError("Failed to reserve slot"): - discard await market.contract - .reserveSlot( - requestId, - slotIndex, - # reserveSlot runs out of gas for unknown reason, but 100k gas covers it - TransactionOverrides(gasLimit: some 100000.u256), - ) - .confirm(1) + try: + discard await market.contract + .reserveSlot( + requestId, + slotIndex, + # reserveSlot runs out of gas for unknown reason, but 100k gas covers it + TransactionOverrides(gasLimit: some 100000.u256), + ) + .confirm(1) + except SlotReservations_ReservationNotAllowed: + raise newException( + SlotReservationNotAllowedError, + "Failed to reserve slot because reservation is not allowed", + ) method canReserveSlot*( market: OnChainMarket, requestId: RequestId, slotIndex: uint64 diff --git a/codex/contracts/marketplace.nim b/codex/contracts/marketplace.nim index 761caada..686414fb 100644 --- a/codex/contracts/marketplace.nim +++ b/codex/contracts/marketplace.nim @@ -53,6 +53,7 @@ type Proofs_ProofAlreadyMarkedMissing* = object of SolidityError Proofs_InvalidProbability* = object of SolidityError Periods_InvalidSecondsPerPeriod* = object of SolidityError + SlotReservations_ReservationNotAllowed* = object of SolidityError proc configuration*(marketplace: Marketplace): MarketplaceConfig {.contract, view.} proc token*(marketplace: Marketplace): Address {.contract, view.} diff --git a/codex/market.nim b/codex/market.nim index c5177aeb..dd8e14ba 100644 --- a/codex/market.nim +++ b/codex/market.nim @@ -18,6 +18,8 @@ export periods type Market* = ref object of RootObj MarketError* = object of CodexError + SlotStateMismatchError* = object of MarketError + SlotReservationNotAllowedError* = object of MarketError Subscription* = ref object of RootObj OnRequest* = proc(id: RequestId, ask: StorageAsk, expiry: uint64) {.gcsafe, upraises: [].} diff --git a/codex/sales/states/filling.nim b/codex/sales/states/filling.nim index 03e2ef2b..13644223 100644 --- a/codex/sales/states/filling.nim +++ b/codex/sales/states/filling.nim @@ -30,6 +30,7 @@ method run*( ): Future[?State] {.async: (raises: []).} = let data = SalesAgent(machine).data let market = SalesAgent(machine).context.market + without (request =? data.request): raiseAssert "Request not set" @@ -42,17 +43,16 @@ method run*( err: error "Failure attempting to fill slot: unable to calculate collateral", error = err.msg - return + return some State(SaleErrored(error: err)) debug "Filling slot" try: await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral) + except SlotStateMismatchError as e: + debug "Slot is already filled, ignoring slot" + return some State(SaleIgnored(reprocessSlot: false, returnBytes: true)) except MarketError as e: - if e.msg.contains "Slot is not free": - debug "Slot is already filled, ignoring slot" - return some State(SaleIgnored(reprocessSlot: false, returnBytes: true)) - else: - return some State(SaleErrored(error: e)) + return some State(SaleErrored(error: e)) # other CatchableErrors are handled "automatically" by the SaleState return some State(SaleFilled()) diff --git a/codex/sales/states/slotreserving.nim b/codex/sales/states/slotreserving.nim index a67c51a0..e9ac8dcd 100644 --- a/codex/sales/states/slotreserving.nim +++ b/codex/sales/states/slotreserving.nim @@ -44,12 +44,11 @@ method run*( try: trace "Reserving slot" await market.reserveSlot(data.requestId, data.slotIndex) + except SlotReservationNotAllowedError as e: + debug "Slot cannot be reserved, ignoring", error = e.msg + return some State(SaleIgnored(reprocessSlot: false, returnBytes: true)) except MarketError as e: - if e.msg.contains "SlotReservations_ReservationNotAllowed": - debug "Slot cannot be reserved, ignoring", error = e.msg - return some State(SaleIgnored(reprocessSlot: false, returnBytes: true)) - else: - return some State(SaleErrored(error: e)) + return some State(SaleErrored(error: e)) # other CatchableErrors are handled "automatically" by the SaleState trace "Slot successfully reserved" diff --git a/tests/codex/helpers/mockmarket.nim b/tests/codex/helpers/mockmarket.nim index 16806cb2..edf8a62d 100644 --- a/tests/codex/helpers/mockmarket.nim +++ b/tests/codex/helpers/mockmarket.nim @@ -46,7 +46,8 @@ type subscriptions: Subscriptions config*: MarketplaceConfig canReserveSlot*: bool - reserveSlotThrowError*: ?(ref MarketError) + errorOnReserveSlot*: ?(ref MarketError) + errorOnFillSlot*: ?(ref CatchableError) clock: ?Clock Fulfillment* = object @@ -289,6 +290,9 @@ proc fillSlot*( host: Address, collateral = 0.u256, ) = + if error =? market.errorOnFillSlot: + raise error + let slot = MockSlot( requestId: requestId, slotIndex: slotIndex, @@ -370,7 +374,7 @@ method canProofBeMarkedAsMissing*( method reserveSlot*( market: MockMarket, requestId: RequestId, slotIndex: uint64 ) {.async.} = - if error =? market.reserveSlotThrowError: + if error =? market.errorOnReserveSlot: raise error method canReserveSlot*( @@ -381,8 +385,19 @@ method canReserveSlot*( func setCanReserveSlot*(market: MockMarket, canReserveSlot: bool) = market.canReserveSlot = canReserveSlot -func setReserveSlotThrowError*(market: MockMarket, error: ?(ref MarketError)) = - market.reserveSlotThrowError = error +func setErrorOnReserveSlot*(market: MockMarket, error: ref MarketError) = + market.errorOnReserveSlot = + if error.isNil: + none (ref MarketError) + else: + some error + +func setErrorOnFillSlot*(market: MockMarket, error: ref CatchableError) = + market.errorOnFillSlot = + if error.isNil: + none (ref CatchableError) + else: + some error method subscribeRequests*( market: MockMarket, callback: OnRequest diff --git a/tests/codex/sales/states/testfilling.nim b/tests/codex/sales/states/testfilling.nim index 1a26753d..f746b5a8 100644 --- a/tests/codex/sales/states/testfilling.nim +++ b/tests/codex/sales/states/testfilling.nim @@ -1,18 +1,31 @@ -import pkg/unittest2 import pkg/questionable import pkg/codex/contracts/requests import pkg/codex/sales/states/filling import pkg/codex/sales/states/cancelled import pkg/codex/sales/states/failed +import pkg/codex/sales/states/ignored +import pkg/codex/sales/states/errored +import pkg/codex/sales/salesagent +import pkg/codex/sales/salescontext +import ../../../asynctest import ../../examples import ../../helpers +import ../../helpers/mockmarket +import ../../helpers/mockclock suite "sales state 'filling'": let request = StorageRequest.example let slotIndex = request.ask.slots div 2 var state: SaleFilling + var market: MockMarket + var clock: MockClock + var agent: SalesAgent setup: + clock = MockClock.new() + market = MockMarket.new() + let context = SalesContext(market: market, clock: clock) + agent = newSalesAgent(context, request.id, slotIndex, request.some) state = SaleFilling.new() test "switches to cancelled state when request expires": @@ -22,3 +35,28 @@ suite "sales state 'filling'": test "switches to failed state when request fails": let next = state.onFailed(request) check !next of SaleFailed + + test "run switches to ignored when slot is not free": + let error = newException( + SlotStateMismatchError, "Failed to fill slot because the slot is not free" + ) + market.setErrorOnFillSlot(error) + market.requested.add(request) + market.slotState[request.slotId(slotIndex)] = SlotState.Filled + + let next = !(await state.run(agent)) + check next of SaleIgnored + check SaleIgnored(next).reprocessSlot == false + check SaleIgnored(next).returnBytes + + test "run switches to errored with other error ": + let error = newException(MarketError, "some error") + market.setErrorOnFillSlot(error) + market.requested.add(request) + market.slotState[request.slotId(slotIndex)] = SlotState.Filled + + let next = !(await state.run(agent)) + check next of SaleErrored + + let errored = SaleErrored(next) + check errored.error == error diff --git a/tests/codex/sales/states/testslotreserving.nim b/tests/codex/sales/states/testslotreserving.nim index d9ecdfc8..0e2e2cc7 100644 --- a/tests/codex/sales/states/testslotreserving.nim +++ b/tests/codex/sales/states/testslotreserving.nim @@ -54,15 +54,16 @@ asyncchecksuite "sales state 'SlotReserving'": test "run switches to errored when slot reservation errors": let error = newException(MarketError, "some error") - market.setReserveSlotThrowError(some error) + market.setErrorOnReserveSlot(error) let next = !(await state.run(agent)) check next of SaleErrored let errored = SaleErrored(next) check errored.error == error - test "catches reservation not allowed error": - let error = newException(MarketError, "SlotReservations_ReservationNotAllowed") - market.setReserveSlotThrowError(some error) + test "run switches to ignored when reservation is not allowed": + let error = + newException(SlotReservationNotAllowedError, "Reservation is not allowed") + market.setErrorOnReserveSlot(error) let next = !(await state.run(agent)) check next of SaleIgnored check SaleIgnored(next).reprocessSlot == false From 3a312596bf1b7cc6842047112777488bc9f0e4f8 Mon Sep 17 00:00:00 2001 From: munna0908 <88337208+munna0908@users.noreply.github.com> Date: Fri, 21 Mar 2025 07:41:00 +0530 Subject: [PATCH 03/24] deps: upgrade libp2p & constantine (#1167) * upgrade libp2p and constantine * fix libp2p update issues * add missing vendor package * add missing vendor package --- .gitmodules | 10 ++++++++++ codex/blockexchange/engine/engine.nim | 4 +++- codex/blockexchange/network/network.nim | 8 ++++++-- codex/blockexchange/protobuf/message.nim | 10 ++++------ codex/merkletree/codex/coders.nim | 12 ++++++------ vendor/constantine | 2 +- vendor/nim-codex-dht | 2 +- vendor/nim-libp2p | 2 +- vendor/nim-ngtcp2 | 1 + vendor/nim-quic | 1 + 10 files changed, 34 insertions(+), 18 deletions(-) create mode 160000 vendor/nim-ngtcp2 create mode 160000 vendor/nim-quic diff --git a/.gitmodules b/.gitmodules index ece88749..5cc2bfab 100644 --- a/.gitmodules +++ b/.gitmodules @@ -221,3 +221,13 @@ [submodule "vendor/nph"] path = vendor/nph url = https://github.com/arnetheduck/nph.git +[submodule "vendor/nim-quic"] + path = vendor/nim-quic + url = https://github.com/vacp2p/nim-quic.git + ignore = untracked + branch = master +[submodule "vendor/nim-ngtcp2"] + path = vendor/nim-ngtcp2 + url = https://github.com/vacp2p/nim-ngtcp2.git + ignore = untracked + branch = master diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index befb8ae9..35785cfe 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -678,7 +678,9 @@ proc new*( advertiser: advertiser, ) - proc peerEventHandler(peerId: PeerId, event: PeerEvent) {.async.} = + proc peerEventHandler( + peerId: PeerId, event: PeerEvent + ): Future[void] {.gcsafe, async: (raises: [CancelledError]).} = if event.kind == PeerEventKind.Joined: await self.setupPeer(peerId) else: diff --git a/codex/blockexchange/network/network.nim b/codex/blockexchange/network/network.nim index 26c07445..d4754110 100644 --- a/codex/blockexchange/network/network.nim +++ b/codex/blockexchange/network/network.nim @@ -323,7 +323,9 @@ method init*(self: BlockExcNetwork) = ## Perform protocol initialization ## - proc peerEventHandler(peerId: PeerId, event: PeerEvent) {.async.} = + proc peerEventHandler( + peerId: PeerId, event: PeerEvent + ): Future[void] {.gcsafe, async: (raises: [CancelledError]).} = if event.kind == PeerEventKind.Joined: self.setupPeer(peerId) else: @@ -332,7 +334,9 @@ method init*(self: BlockExcNetwork) = self.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Joined) self.switch.addPeerEventHandler(peerEventHandler, PeerEventKind.Left) - proc handler(conn: Connection, proto: string) {.async.} = + proc handler( + conn: Connection, proto: string + ): Future[void] {.async: (raises: [CancelledError]).} = let peerId = conn.peerId let blockexcPeer = self.getOrCreatePeer(peerId) await blockexcPeer.readLoop(conn) # attach read loop diff --git a/codex/blockexchange/protobuf/message.nim b/codex/blockexchange/protobuf/message.nim index 73cb60f1..4db89729 100644 --- a/codex/blockexchange/protobuf/message.nim +++ b/codex/blockexchange/protobuf/message.nim @@ -97,7 +97,7 @@ proc write*(pb: var ProtoBuffer, field: int, value: WantList) = pb.write(field, ipb) proc write*(pb: var ProtoBuffer, field: int, value: BlockDelivery) = - var ipb = initProtoBuffer(maxSize = MaxBlockSize) + var ipb = initProtoBuffer() ipb.write(1, value.blk.cid.data.buffer) ipb.write(2, value.blk.data) ipb.write(3, value.address) @@ -128,7 +128,7 @@ proc write*(pb: var ProtoBuffer, field: int, value: StateChannelUpdate) = pb.write(field, ipb) proc protobufEncode*(value: Message): seq[byte] = - var ipb = initProtoBuffer(maxSize = MaxMessageSize) + var ipb = initProtoBuffer() ipb.write(1, value.wantList) for v in value.payload: ipb.write(3, v) @@ -254,16 +254,14 @@ proc decode*( proc protobufDecode*(_: type Message, msg: seq[byte]): ProtoResult[Message] = var value = Message() - pb = initProtoBuffer(msg, maxSize = MaxMessageSize) + pb = initProtoBuffer(msg) ipb: ProtoBuffer sublist: seq[seq[byte]] if ?pb.getField(1, ipb): value.wantList = ?WantList.decode(ipb) if ?pb.getRepeatedField(3, sublist): for item in sublist: - value.payload.add( - ?BlockDelivery.decode(initProtoBuffer(item, maxSize = MaxBlockSize)) - ) + value.payload.add(?BlockDelivery.decode(initProtoBuffer(item))) if ?pb.getRepeatedField(4, sublist): for item in sublist: value.blockPresences.add(?BlockPresence.decode(initProtoBuffer(item))) diff --git a/codex/merkletree/codex/coders.nim b/codex/merkletree/codex/coders.nim index b8209991..1d50707c 100644 --- a/codex/merkletree/codex/coders.nim +++ b/codex/merkletree/codex/coders.nim @@ -27,11 +27,11 @@ const MaxMerkleTreeSize = 100.MiBs.uint const MaxMerkleProofSize = 1.MiBs.uint proc encode*(self: CodexTree): seq[byte] = - var pb = initProtoBuffer(maxSize = MaxMerkleTreeSize) + var pb = initProtoBuffer() pb.write(1, self.mcodec.uint64) pb.write(2, self.leavesCount.uint64) for node in self.nodes: - var nodesPb = initProtoBuffer(maxSize = MaxMerkleTreeSize) + var nodesPb = initProtoBuffer() nodesPb.write(1, node) nodesPb.finish() pb.write(3, nodesPb) @@ -40,7 +40,7 @@ proc encode*(self: CodexTree): seq[byte] = pb.buffer proc decode*(_: type CodexTree, data: seq[byte]): ?!CodexTree = - var pb = initProtoBuffer(data, maxSize = MaxMerkleTreeSize) + var pb = initProtoBuffer(data) var mcodecCode: uint64 var leavesCount: uint64 discard ?pb.getField(1, mcodecCode).mapFailure @@ -63,13 +63,13 @@ proc decode*(_: type CodexTree, data: seq[byte]): ?!CodexTree = CodexTree.fromNodes(mcodec, nodes, leavesCount.int) proc encode*(self: CodexProof): seq[byte] = - var pb = initProtoBuffer(maxSize = MaxMerkleProofSize) + var pb = initProtoBuffer() pb.write(1, self.mcodec.uint64) pb.write(2, self.index.uint64) pb.write(3, self.nleaves.uint64) for node in self.path: - var nodesPb = initProtoBuffer(maxSize = MaxMerkleTreeSize) + var nodesPb = initProtoBuffer() nodesPb.write(1, node) nodesPb.finish() pb.write(4, nodesPb) @@ -78,7 +78,7 @@ proc encode*(self: CodexProof): seq[byte] = pb.buffer proc decode*(_: type CodexProof, data: seq[byte]): ?!CodexProof = - var pb = initProtoBuffer(data, maxSize = MaxMerkleProofSize) + var pb = initProtoBuffer(data) var mcodecCode: uint64 var index: uint64 var nleaves: uint64 diff --git a/vendor/constantine b/vendor/constantine index bc3845aa..8d6a6a38 160000 --- a/vendor/constantine +++ b/vendor/constantine @@ -1 +1 @@ -Subproject commit bc3845aa492b52f7fef047503b1592e830d1a774 +Subproject commit 8d6a6a38b90fb8ee3ec2230839773e69aab36d80 diff --git a/vendor/nim-codex-dht b/vendor/nim-codex-dht index 4bd3a39e..f6eef1ac 160000 --- a/vendor/nim-codex-dht +++ b/vendor/nim-codex-dht @@ -1 +1 @@ -Subproject commit 4bd3a39e0030f8ee269ef217344b6b59ec2be6dc +Subproject commit f6eef1ac95c70053b2518f1e3909c909ed8701a6 diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index 036e110a..c08d8073 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit 036e110a6080fba1a1662c58cfd8c21f9a548021 +Subproject commit c08d80734989b028b3d1705f2188d783a343aac0 diff --git a/vendor/nim-ngtcp2 b/vendor/nim-ngtcp2 new file mode 160000 index 00000000..6834f475 --- /dev/null +++ b/vendor/nim-ngtcp2 @@ -0,0 +1 @@ +Subproject commit 6834f4756b6af58356ac9c4fef3d71db3c3ae5fe diff --git a/vendor/nim-quic b/vendor/nim-quic new file mode 160000 index 00000000..ddcb31ff --- /dev/null +++ b/vendor/nim-quic @@ -0,0 +1 @@ +Subproject commit ddcb31ffb74b5460ab37fd13547eca90594248bc From 110147d8efbb1411fa4cb393125ada1b2e461be1 Mon Sep 17 00:00:00 2001 From: Dmitriy Ryajov Date: Fri, 21 Mar 2025 11:23:07 -0600 Subject: [PATCH 04/24] monitor background tasks on streaming dataset (#1164) --- codex/node.nim | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/codex/node.nim b/codex/node.nim index 203e034a..9932deb6 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -271,6 +271,8 @@ proc streamEntireDataset( ## trace "Retrieving blocks from manifest", manifestCid + var jobs: seq[Future[void]] + let stream = LPStream(StoreStream.new(self.networkStore, manifest, pad = false)) if manifest.protected: # Retrieve, decode and save to the local store all EС groups proc erasureJob(): Future[void] {.async: (raises: []).} = @@ -284,14 +286,25 @@ proc streamEntireDataset( except CatchableError as exc: trace "Error erasure decoding manifest", manifestCid, exc = exc.msg - self.trackedFutures.track(erasureJob()) + jobs.add(erasureJob()) - self.trackedFutures.track(self.fetchDatasetAsync(manifest, fetchLocal = false)) - # prefetch task should not fetch from local store + jobs.add(self.fetchDatasetAsync(manifest)) + + # Monitor stream completion and cancel background jobs when done + proc monitorStream() {.async: (raises: []).} = + try: + await stream.join() + except CatchableError as exc: + warn "Stream failed", exc = exc.msg + finally: + await noCancel allFutures(jobs.mapIt(it.cancelAndWait)) + + self.trackedFutures.track(monitorStream()) # Retrieve all blocks of the dataset sequentially from the local store or network trace "Creating store stream for manifest", manifestCid - LPStream(StoreStream.new(self.networkStore, manifest, pad = false)).success + + stream.success proc retrieve*( self: CodexNodeRef, cid: Cid, local: bool = true From 709a8648fd9854253b5fc0c55adef1c5177c5ae3 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 24 Mar 2025 12:53:34 +0100 Subject: [PATCH 05/24] chore: add request validations (#1144) * Add request validations * Define expiry as required field in storage request params and fix tests * Fix error messages * Enable logs to figure out the issue with recurring failing test on macos * Add custom errors raised by contract * Remove custom error non existing anymore * Update asynctest module * Update timer tests after updating asynctest --- codex/contracts/marketplace.nim | 5 +- codex/rest/api.nim | 46 ++++++++++----- codex/rest/json.nim | 2 +- openapi.yaml | 2 + tests/codex/utils/testtimer.nim | 10 ++-- tests/integration/testpurchasing.nim | 9 +-- tests/integration/testrestapi.nim | 88 ++++++++++++++++++++++++++-- tests/integration/testsales.nim | 14 ++++- vendor/asynctest | 2 +- 9 files changed, 143 insertions(+), 35 deletions(-) diff --git a/codex/contracts/marketplace.nim b/codex/contracts/marketplace.nim index 686414fb..11eca5be 100644 --- a/codex/contracts/marketplace.nim +++ b/codex/contracts/marketplace.nim @@ -51,7 +51,6 @@ type Proofs_ProofNotMissing* = object of SolidityError Proofs_ProofNotRequired* = object of SolidityError Proofs_ProofAlreadyMarkedMissing* = object of SolidityError - Proofs_InvalidProbability* = object of SolidityError Periods_InvalidSecondsPerPeriod* = object of SolidityError SlotReservations_ReservationNotAllowed* = object of SolidityError @@ -68,7 +67,9 @@ proc requestStorage*( errors: [ Marketplace_InvalidClientAddress, Marketplace_RequestAlreadyExists, Marketplace_InvalidExpiry, Marketplace_InsufficientSlots, - Marketplace_InvalidMaxSlotLoss, + Marketplace_InvalidMaxSlotLoss, Marketplace_InsufficientDuration, + Marketplace_InsufficientProofProbability, Marketplace_InsufficientCollateral, + Marketplace_InsufficientReward, Marketplace_InvalidCid, ] .} diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 553cb91c..5d813188 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -652,10 +652,36 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = without params =? StorageRequestParams.fromJson(body), error: return RestApiResponse.error(Http400, error.msg, headers = headers) + let expiry = params.expiry + + if expiry <= 0 or expiry >= params.duration: + return RestApiResponse.error( + Http422, + "Expiry must be greater than zero and less than the request's duration", + headers = headers, + ) + + if params.proofProbability <= 0: + return RestApiResponse.error( + Http422, "Proof probability must be greater than zero", headers = headers + ) + + if params.collateralPerByte <= 0: + return RestApiResponse.error( + Http422, "Collateral per byte must be greater than zero", headers = headers + ) + + if params.pricePerBytePerSecond <= 0: + return RestApiResponse.error( + Http422, + "Price per byte per second must be greater than zero", + headers = headers, + ) + let requestDurationLimit = await contracts.purchasing.market.requestDurationLimit if params.duration > requestDurationLimit: return RestApiResponse.error( - Http400, + Http422, "Duration exceeds limit of " & $requestDurationLimit & " seconds", headers = headers, ) @@ -665,13 +691,13 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = if tolerance == 0: return RestApiResponse.error( - Http400, "Tolerance needs to be bigger then zero", headers = headers + Http422, "Tolerance needs to be bigger then zero", headers = headers ) # prevent underflow if tolerance > nodes: return RestApiResponse.error( - Http400, + Http422, "Invalid parameters: `tolerance` cannot be greater than `nodes`", headers = headers, ) @@ -682,21 +708,11 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = # ensure leopard constrainst of 1 < K ≥ M if ecK <= 1 or ecK < ecM: return RestApiResponse.error( - Http400, + Http422, "Invalid parameters: parameters must satify `1 < (nodes - tolerance) ≥ tolerance`", headers = headers, ) - without expiry =? params.expiry: - return RestApiResponse.error(Http400, "Expiry required", headers = headers) - - if expiry <= 0 or expiry >= params.duration: - return RestApiResponse.error( - Http400, - "Expiry needs value bigger then zero and smaller then the request's duration", - headers = headers, - ) - without purchaseId =? await node.requestStorage( cid, params.duration, params.proofProbability, nodes, tolerance, @@ -704,7 +720,7 @@ proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = ), error: if error of InsufficientBlocksError: return RestApiResponse.error( - Http400, + Http422, "Dataset too small for erasure parameters, need at least " & $(ref InsufficientBlocksError)(error).minSize.int & " bytes", headers = headers, diff --git a/codex/rest/json.nim b/codex/rest/json.nim index c221ba73..50c8b514 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -17,7 +17,7 @@ type proofProbability* {.serialize.}: UInt256 pricePerBytePerSecond* {.serialize.}: UInt256 collateralPerByte* {.serialize.}: UInt256 - expiry* {.serialize.}: ?uint64 + expiry* {.serialize.}: uint64 nodes* {.serialize.}: ?uint tolerance* {.serialize.}: ?uint diff --git a/openapi.yaml b/openapi.yaml index 53a908a3..ad1b166b 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -800,6 +800,8 @@ paths: type: string "400": description: Invalid or missing Request ID + "422": + description: The storage request parameters are not valid "404": description: Request ID not found "503": diff --git a/tests/codex/utils/testtimer.nim b/tests/codex/utils/testtimer.nim index 303c43fb..2f356df9 100644 --- a/tests/codex/utils/testtimer.nim +++ b/tests/codex/utils/testtimer.nim @@ -52,21 +52,21 @@ asyncchecksuite "Timer": test "Start timer1 should execute callback": startNumbersTimer() - check eventually output == "0" + check eventually(output == "0", pollInterval = 10) test "Start timer1 should execute callback multiple times": startNumbersTimer() - check eventually output == "012" + check eventually(output == "012", pollInterval = 10) test "Starting timer1 multiple times has no impact": startNumbersTimer() startNumbersTimer() startNumbersTimer() - check eventually output == "01234" + check eventually(output == "01234", pollInterval = 10) test "Stop timer1 should stop execution of the callback": startNumbersTimer() - check eventually output == "012" + check eventually(output == "012", pollInterval = 10) await timer1.stop() await sleepAsync(30.milliseconds) let stoppedOutput = output @@ -81,4 +81,4 @@ asyncchecksuite "Timer": test "Starting both timers should execute callbacks sequentially": startNumbersTimer() startLettersTimer() - check eventually output == "0a1b2c3d4e" + check eventually(output == "0a1b2c3d4e", pollInterval = 10) diff --git a/tests/integration/testpurchasing.nim b/tests/integration/testpurchasing.nim index e5adebe2..ba8dd190 100644 --- a/tests/integration/testpurchasing.nim +++ b/tests/integration/testpurchasing.nim @@ -123,8 +123,9 @@ twonodessuite "Purchasing": proofProbability = 3.u256, collateralPerByte = 1.u256, ) - check responseMissing.status == 400 - check (await responseMissing.body) == "Expiry required" + check responseMissing.status == 422 + check (await responseMissing.body) == + "Expiry must be greater than zero and less than the request's duration" let responseBefore = await client1.requestStorageRaw( cid, @@ -134,6 +135,6 @@ twonodessuite "Purchasing": collateralPerByte = 1.u256, expiry = 10.uint64, ) - check responseBefore.status == 400 - check "Expiry needs value bigger then zero and smaller then the request's duration" in + check responseBefore.status == 422 + check "Expiry must be greater than zero and less than the request's duration" in (await responseBefore.body) diff --git a/tests/integration/testrestapi.nim b/tests/integration/testrestapi.nim index 761eda31..e7e185b8 100644 --- a/tests/integration/testrestapi.nim +++ b/tests/integration/testrestapi.nim @@ -69,7 +69,7 @@ twonodessuite "REST API": ) check: - response.status == 400 + response.status == 422 (await response.body) == "Dataset too small for erasure parameters, need at least " & $(2 * DefaultBlockSize.int) & " bytes" @@ -109,7 +109,7 @@ twonodessuite "REST API": ) ) - check responseBefore.status == 400 + check responseBefore.status == 422 check (await responseBefore.body) == "Tolerance needs to be bigger then zero" test "request storage fails if duration exceeds limit", twoNodesConfig: @@ -131,7 +131,7 @@ twonodessuite "REST API": ) ) - check responseBefore.status == 400 + check responseBefore.status == 422 check "Duration exceeds limit of" in (await responseBefore.body) test "request storage fails if nodes and tolerance aren't correct", twoNodesConfig: @@ -154,7 +154,7 @@ twonodessuite "REST API": ) ) - check responseBefore.status == 400 + check responseBefore.status == 422 check (await responseBefore.body) == "Invalid parameters: parameters must satify `1 < (nodes - tolerance) ≥ tolerance`" @@ -179,10 +179,88 @@ twonodessuite "REST API": ) ) - check responseBefore.status == 400 + check responseBefore.status == 422 check (await responseBefore.body) == "Invalid parameters: `tolerance` cannot be greater than `nodes`" + test "request storage fails if expiry is zero", twoNodesConfig: + let data = await RandomChunker.example(blocks = 2) + let cid = (await client1.upload(data)).get + let duration = 100.uint64 + let pricePerBytePerSecond = 1.u256 + let proofProbability = 3.u256 + let expiry = 0.uint64 + let collateralPerByte = 1.u256 + let nodes = 3 + let tolerance = 1 + + var responseBefore = await client1.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, + nodes.uint, tolerance.uint, + ) + + check responseBefore.status == 422 + check (await responseBefore.body) == + "Expiry must be greater than zero and less than the request's duration" + + test "request storage fails if proof probability is zero", twoNodesConfig: + let data = await RandomChunker.example(blocks = 2) + let cid = (await client1.upload(data)).get + let duration = 100.uint64 + let pricePerBytePerSecond = 1.u256 + let proofProbability = 0.u256 + let expiry = 30.uint64 + let collateralPerByte = 1.u256 + let nodes = 3 + let tolerance = 1 + + var responseBefore = await client1.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, + nodes.uint, tolerance.uint, + ) + + check responseBefore.status == 422 + check (await responseBefore.body) == "Proof probability must be greater than zero" + + test "request storage fails if collareral per byte is zero", twoNodesConfig: + let data = await RandomChunker.example(blocks = 2) + let cid = (await client1.upload(data)).get + let duration = 100.uint64 + let pricePerBytePerSecond = 1.u256 + let proofProbability = 3.u256 + let expiry = 30.uint64 + let collateralPerByte = 0.u256 + let nodes = 3 + let tolerance = 1 + + var responseBefore = await client1.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, + nodes.uint, tolerance.uint, + ) + + check responseBefore.status == 422 + check (await responseBefore.body) == "Collateral per byte must be greater than zero" + + test "request storage fails if price per byte per second is zero", twoNodesConfig: + let data = await RandomChunker.example(blocks = 2) + let cid = (await client1.upload(data)).get + let duration = 100.uint64 + let pricePerBytePerSecond = 0.u256 + let proofProbability = 3.u256 + let expiry = 30.uint64 + let collateralPerByte = 1.u256 + let nodes = 3 + let tolerance = 1 + + var responseBefore = await client1.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, + nodes.uint, tolerance.uint, + ) + + check responseBefore.status == 422 + check (await responseBefore.body) == + "Price per byte per second must be greater than zero" + for ecParams in @[ (minBlocks: 2, nodes: 3, tolerance: 1), (minBlocks: 3, nodes: 5, tolerance: 2) ]: diff --git a/tests/integration/testsales.nim b/tests/integration/testsales.nim index 2d7a199c..9bf8a97c 100644 --- a/tests/integration/testsales.nim +++ b/tests/integration/testsales.nim @@ -16,8 +16,18 @@ proc findItem[T](items: seq[T], item: T): ?!T = multinodesuite "Sales": let salesConfig = NodeConfigs( - clients: CodexConfigs.init(nodes = 1).some, - providers: CodexConfigs.init(nodes = 1).some, + clients: CodexConfigs + .init(nodes = 1) + .withLogFile() + .withLogTopics( + "node", "marketplace", "sales", "reservations", "node", "proving", "clock" + ).some, + providers: CodexConfigs + .init(nodes = 1) + .withLogFile() + .withLogTopics( + "node", "marketplace", "sales", "reservations", "node", "proving", "clock" + ).some, ) let minPricePerBytePerSecond = 1.u256 diff --git a/vendor/asynctest b/vendor/asynctest index 5154c0d7..73c08f77 160000 --- a/vendor/asynctest +++ b/vendor/asynctest @@ -1 +1 @@ -Subproject commit 5154c0d79dd8bb086ab418cc659e923330ac24f2 +Subproject commit 73c08f77afc5cc2a5628d00f915b97bf72f70c9b From a0d6fbaf0248c36782a14bf95a6771861ae1034e Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 24 Mar 2025 16:47:05 +0100 Subject: [PATCH 06/24] chore(marketplace) - fix the http error codes when validating the availability requests (#1104) * Use 422 http code when there is a validation error * Update the open api description * Fix typo * Add more tests for total size * Catch CancelledError because TrackedFuture raise no error * Split rest api validation test to a new file * Change the way of testing negative numbers * Rename client variable and fix test status code * Try to reduce the number of requests in CI when asserting in tests * Fix rebase and remove safeEventually --- codex/rest/api.nim | 12 +- codex/sales.nim | 8 +- openapi.yaml | 4 +- tests/helpers.nim | 2 +- tests/integration/codexclient.nim | 23 +- tests/integration/multinodes.nim | 17 +- tests/integration/testrestapi.nim | 203 ----------- tests/integration/testrestapivalidation.nim | 368 ++++++++++++++++++++ tests/integration/testsales.nim | 26 +- tests/testIntegration.nim | 1 + 10 files changed, 418 insertions(+), 246 deletions(-) create mode 100644 tests/integration/testrestapivalidation.nim diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 5d813188..243d4ed6 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -475,7 +475,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = if restAv.totalSize == 0: return RestApiResponse.error( - Http400, "Total size must be larger then zero", headers = headers + Http422, "Total size must be larger then zero", headers = headers ) if not reservations.hasAvailable(restAv.totalSize): @@ -548,17 +548,23 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = return RestApiResponse.error(Http500, error.msg) if isSome restAv.freeSize: - return RestApiResponse.error(Http400, "Updating freeSize is not allowed") + return RestApiResponse.error(Http422, "Updating freeSize is not allowed") if size =? restAv.totalSize: + if size == 0: + return RestApiResponse.error(Http422, "Total size must be larger then zero") + # we don't allow lowering the totalSize bellow currently utilized size if size < (availability.totalSize - availability.freeSize): return RestApiResponse.error( - Http400, + Http422, "New totalSize must be larger then current totalSize - freeSize, which is currently: " & $(availability.totalSize - availability.freeSize), ) + if not reservations.hasAvailable(size): + return RestApiResponse.error(Http422, "Not enough storage quota") + availability.freeSize += size - availability.totalSize availability.totalSize = size diff --git a/codex/sales.nim b/codex/sales.nim index 998a2967..a4a174c1 100644 --- a/codex/sales.nim +++ b/codex/sales.nim @@ -374,13 +374,13 @@ proc onSlotFreed(sales: Sales, requestId: RequestId, slotIndex: uint64) = if err =? queue.push(slotQueueItem).errorOption: if err of SlotQueueItemExistsError: - error "Failed to push item to queue becaue it already exists", + error "Failed to push item to queue because it already exists", error = err.msgDetail elif err of QueueNotRunningError: - warn "Failed to push item to queue becaue queue is not running", + warn "Failed to push item to queue because queue is not running", error = err.msgDetail - except CatchableError as e: - warn "Failed to add slot to queue", error = e.msg + except CancelledError as e: + trace "sales.addSlotToQueue was cancelled" # We could get rid of this by adding the storage ask in the SlotFreed event, # so we would not need to call getRequest to get the collateralPerSlot. diff --git a/openapi.yaml b/openapi.yaml index ad1b166b..c2088cc5 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -704,7 +704,7 @@ paths: "400": description: Invalid data input "422": - description: Not enough node's storage quota available + description: The provided parameters did not pass validation "500": description: Error reserving availability "503": @@ -737,7 +737,7 @@ paths: "404": description: Availability not found "422": - description: Not enough node's storage quota available + description: The provided parameters did not pass validation "500": description: Error reserving availability "503": diff --git a/tests/helpers.nim b/tests/helpers.nim index 82b544f1..b48b787e 100644 --- a/tests/helpers.nim +++ b/tests/helpers.nim @@ -1,7 +1,7 @@ import helpers/multisetup import helpers/trackers import helpers/templeveldb - +import std/times import std/sequtils, chronos export multisetup, trackers, templeveldb diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index ef76b577..5d5f0cc2 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -45,7 +45,7 @@ proc request( ).get .send() -proc post( +proc post*( self: CodexClient, url: string, body: string = "", @@ -69,7 +69,7 @@ proc delete( .} = return self.request(MethodDelete, url, headers = headers) -proc patch( +proc patch*( self: CodexClient, url: string, body: string = "", @@ -290,11 +290,11 @@ proc getSalesAgent*( except CatchableError as e: return failure e.msg -proc postAvailability*( +proc postAvailabilityRaw*( client: CodexClient, totalSize, duration: uint64, minPricePerBytePerSecond, totalCollateral: UInt256, -): Future[?!Availability] {.async: (raises: [CancelledError, HttpError]).} = +): Future[HttpClientResponseRef] {.async: (raises: [CancelledError, HttpError]).} = ## Post sales availability endpoint ## let url = client.baseurl & "/sales/availability" @@ -305,7 +305,17 @@ proc postAvailability*( "minPricePerBytePerSecond": minPricePerBytePerSecond, "totalCollateral": totalCollateral, } - let response = await client.post(url, $json) + + return await client.post(url, $json) + +proc postAvailability*( + client: CodexClient, + totalSize, duration: uint64, + minPricePerBytePerSecond, totalCollateral: UInt256, +): Future[?!Availability] {.async: (raises: [CancelledError, HttpError]).} = + let response = await client.postAvailabilityRaw( + totalSize, duration, minPricePerBytePerSecond, totalCollateral + ) let body = await response.body doAssert response.status == 201, @@ -389,3 +399,6 @@ proc requestId*( client: CodexClient, id: PurchaseId ): Future[?RequestId] {.async: (raises: [CancelledError, HttpError]).} = return (await client.getPurchase(id)).option .? requestId + +proc buildUrl*(client: CodexClient, path: string): string = + return client.baseurl & path diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 0003b216..4b183674 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -37,10 +37,12 @@ type MultiNodeSuiteError = object of CatchableError +const jsonRpcProviderUrl* = "http://127.0.0.1:8545" + proc raiseMultiNodeSuiteError(msg: string) = raise newException(MultiNodeSuiteError, msg) -proc nextFreePort(startPort: int): Future[int] {.async.} = +proc nextFreePort*(startPort: int): Future[int] {.async.} = proc client(server: StreamServer, transp: StreamTransport) {.async.} = await transp.closeWait() @@ -60,6 +62,15 @@ proc nextFreePort(startPort: int): Future[int] {.async.} = trace "port is not free", port inc port +proc sanitize(pathSegment: string): string = + var sanitized = pathSegment + for invalid in invalidFilenameChars.items: + sanitized = sanitized.replace(invalid, '_').replace(' ', '_') + sanitized + +proc getTempDirName*(starttime: string, role: Role, roleIdx: int): string = + getTempDir() / "Codex" / sanitize($starttime) / sanitize($role & "_" & $roleIdx) + template multinodesuite*(name: string, body: untyped) = asyncchecksuite name: # Following the problem described here: @@ -82,7 +93,6 @@ template multinodesuite*(name: string, body: untyped) = # .withEthProvider("ws://localhost:8545") # .some, # ... - let jsonRpcProviderUrl = "http://127.0.0.1:8545" var running {.inject, used.}: seq[RunningNode] var bootstrapNodes: seq[string] let starttime = now().format("yyyy-MM-dd'_'HH:mm:ss") @@ -148,8 +158,7 @@ template multinodesuite*(name: string, body: untyped) = raiseMultiNodeSuiteError "Cannot start node at nodeIdx " & $nodeIdx & ", not enough eth accounts." - let datadir = - getTempDir() / "Codex" / sanitize($starttime) / sanitize($role & "_" & $roleIdx) + let datadir = getTempDirName(starttime, role, roleIdx) try: if config.logFile.isSome: diff --git a/tests/integration/testrestapi.nim b/tests/integration/testrestapi.nim index e7e185b8..415658c1 100644 --- a/tests/integration/testrestapi.nim +++ b/tests/integration/testrestapi.nim @@ -55,25 +55,6 @@ twonodessuite "REST API": check: [cid1, cid2].allIt(it in list.content.mapIt(it.cid)) - test "request storage fails for datasets that are too small", twoNodesConfig: - let cid = (await client1.upload("some file contents")).get - let response = ( - await client1.requestStorageRaw( - cid, - duration = 10.uint64, - pricePerBytePerSecond = 1.u256, - proofProbability = 3.u256, - collateralPerByte = 1.u256, - expiry = 9.uint64, - ) - ) - - check: - response.status == 422 - (await response.body) == - "Dataset too small for erasure parameters, need at least " & - $(2 * DefaultBlockSize.int) & " bytes" - test "request storage succeeds for sufficiently sized datasets", twoNodesConfig: let data = await RandomChunker.example(blocks = 2) let cid = (await client1.upload(data)).get @@ -91,176 +72,6 @@ twonodessuite "REST API": check: response.status == 200 - test "request storage fails if tolerance is zero", twoNodesConfig: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client1.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 3.u256 - let expiry = 30.uint64 - let collateralPerByte = 1.u256 - let nodes = 3 - let tolerance = 0 - - var responseBefore = ( - await client1.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, - expiry, nodes.uint, tolerance.uint, - ) - ) - - check responseBefore.status == 422 - check (await responseBefore.body) == "Tolerance needs to be bigger then zero" - - test "request storage fails if duration exceeds limit", twoNodesConfig: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client1.upload(data)).get - let duration = (31 * 24 * 60 * 60).uint64 - # 31 days TODO: this should not be hardcoded, but waits for https://github.com/codex-storage/nim-codex/issues/1056 - let proofProbability = 3.u256 - let expiry = 30.uint - let collateralPerByte = 1.u256 - let nodes = 3 - let tolerance = 2 - let pricePerBytePerSecond = 1.u256 - - var responseBefore = ( - await client1.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, - expiry, nodes.uint, tolerance.uint, - ) - ) - - check responseBefore.status == 422 - check "Duration exceeds limit of" in (await responseBefore.body) - - test "request storage fails if nodes and tolerance aren't correct", twoNodesConfig: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client1.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 3.u256 - let expiry = 30.uint64 - let collateralPerByte = 1.u256 - let ecParams = @[(1, 1), (2, 1), (3, 2), (3, 3)] - - for ecParam in ecParams: - let (nodes, tolerance) = ecParam - - var responseBefore = ( - await client1.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, - expiry, nodes.uint, tolerance.uint, - ) - ) - - check responseBefore.status == 422 - check (await responseBefore.body) == - "Invalid parameters: parameters must satify `1 < (nodes - tolerance) ≥ tolerance`" - - test "request storage fails if tolerance > nodes (underflow protection)", - twoNodesConfig: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client1.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 3.u256 - let expiry = 30.uint64 - let collateralPerByte = 1.u256 - let ecParams = @[(0, 1), (1, 2), (2, 3)] - - for ecParam in ecParams: - let (nodes, tolerance) = ecParam - - var responseBefore = ( - await client1.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, - expiry, nodes.uint, tolerance.uint, - ) - ) - - check responseBefore.status == 422 - check (await responseBefore.body) == - "Invalid parameters: `tolerance` cannot be greater than `nodes`" - - test "request storage fails if expiry is zero", twoNodesConfig: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client1.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 3.u256 - let expiry = 0.uint64 - let collateralPerByte = 1.u256 - let nodes = 3 - let tolerance = 1 - - var responseBefore = await client1.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, - nodes.uint, tolerance.uint, - ) - - check responseBefore.status == 422 - check (await responseBefore.body) == - "Expiry must be greater than zero and less than the request's duration" - - test "request storage fails if proof probability is zero", twoNodesConfig: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client1.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 0.u256 - let expiry = 30.uint64 - let collateralPerByte = 1.u256 - let nodes = 3 - let tolerance = 1 - - var responseBefore = await client1.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, - nodes.uint, tolerance.uint, - ) - - check responseBefore.status == 422 - check (await responseBefore.body) == "Proof probability must be greater than zero" - - test "request storage fails if collareral per byte is zero", twoNodesConfig: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client1.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 3.u256 - let expiry = 30.uint64 - let collateralPerByte = 0.u256 - let nodes = 3 - let tolerance = 1 - - var responseBefore = await client1.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, - nodes.uint, tolerance.uint, - ) - - check responseBefore.status == 422 - check (await responseBefore.body) == "Collateral per byte must be greater than zero" - - test "request storage fails if price per byte per second is zero", twoNodesConfig: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client1.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 0.u256 - let proofProbability = 3.u256 - let expiry = 30.uint64 - let collateralPerByte = 1.u256 - let nodes = 3 - let tolerance = 1 - - var responseBefore = await client1.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, - nodes.uint, tolerance.uint, - ) - - check responseBefore.status == 422 - check (await responseBefore.body) == - "Price per byte per second must be greater than zero" - for ecParams in @[ (minBlocks: 2, nodes: 3, tolerance: 1), (minBlocks: 3, nodes: 5, tolerance: 2) ]: @@ -306,20 +117,6 @@ twonodessuite "REST API": check response.status == 200 check (await response.body) != "" - test "upload fails if content disposition contains bad filename", twoNodesConfig: - let headers = @[("Content-Disposition", "attachment; filename=\"exam*ple.txt\"")] - let response = await client1.uploadRaw("some file contents", headers) - - check response.status == 422 - check (await response.body) == "The filename is not valid." - - test "upload fails if content type is invalid", twoNodesConfig: - let headers = @[("Content-Type", "hello/world")] - let response = await client1.uploadRaw("some file contents", headers) - - check response.status == 422 - check (await response.body) == "The MIME type 'hello/world' is not valid." - test "node retrieve the metadata", twoNodesConfig: let headers = @[ diff --git a/tests/integration/testrestapivalidation.nim b/tests/integration/testrestapivalidation.nim new file mode 100644 index 00000000..00caefdd --- /dev/null +++ b/tests/integration/testrestapivalidation.nim @@ -0,0 +1,368 @@ +import std/httpclient +import std/times +import pkg/ethers +import pkg/codex/manifest +import pkg/codex/conf +import pkg/codex/contracts +from pkg/codex/stores/repostore/types import DefaultQuotaBytes +import ../asynctest +import ../checktest +import ../examples +import ../codex/examples +import ./codexconfig +import ./codexprocess + +from ./multinodes import Role, getTempDirName, jsonRpcProviderUrl, nextFreePort + +# This suite allows to run fast the basic rest api validation. +# It starts only one node for all the checks in order to speed up +# the execution. +asyncchecksuite "Rest API validation": + var node: CodexProcess + var config = CodexConfigs.init(nodes = 1).configs[0] + let starttime = now().format("yyyy-MM-dd'_'HH:mm:ss") + let nodexIdx = 0 + let datadir = getTempDirName(starttime, Role.Client, nodexIdx) + + config.addCliOption("--api-port", $(waitFor nextFreePort(8081))) + config.addCliOption("--data-dir", datadir) + config.addCliOption("--nat", "none") + config.addCliOption("--listen-addrs", "/ip4/127.0.0.1/tcp/0") + config.addCliOption("--disc-port", $(waitFor nextFreePort(8081))) + config.addCliOption(StartUpCmd.persistence, "--eth-provider", jsonRpcProviderUrl) + config.addCliOption(StartUpCmd.persistence, "--eth-account", $EthAddress.example) + + node = + waitFor CodexProcess.startNode(config.cliArgs, config.debugEnabled, $Role.Client) + + waitFor node.waitUntilStarted() + + let client = node.client() + + test "should return 422 when attempting delete of non-existing dataset": + let data = await RandomChunker.example(blocks = 2) + let cid = (await client.upload(data)).get + let duration = 100.uint64 + let pricePerBytePerSecond = 1.u256 + let proofProbability = 3.u256 + let expiry = 30.uint64 + let collateralPerByte = 1.u256 + let nodes = 3 + let tolerance = 0 + + var responseBefore = await client.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, + nodes.uint, tolerance.uint, + ) + + check responseBefore.status == 422 + check (await responseBefore.body) == "Tolerance needs to be bigger then zero" + + test "request storage fails for datasets that are too small": + let cid = (await client.upload("some file contents")).get + let response = ( + await client.requestStorageRaw( + cid, + duration = 10.uint64, + pricePerBytePerSecond = 1.u256, + proofProbability = 3.u256, + collateralPerByte = 1.u256, + expiry = 9.uint64, + ) + ) + + check: + response.status == 422 + (await response.body) == + "Dataset too small for erasure parameters, need at least " & + $(2 * DefaultBlockSize.int) & " bytes" + + test "request storage fails if nodes and tolerance aren't correct": + let data = await RandomChunker.example(blocks = 2) + let cid = (await client.upload(data)).get + let duration = 100.uint64 + let pricePerBytePerSecond = 1.u256 + let proofProbability = 3.u256 + let expiry = 30.uint64 + let collateralPerByte = 1.u256 + let ecParams = @[(1, 1), (2, 1), (3, 2), (3, 3)] + + for ecParam in ecParams: + let (nodes, tolerance) = ecParam + + var responseBefore = ( + await client.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, + expiry, nodes.uint, tolerance.uint, + ) + ) + + check responseBefore.status == 422 + check (await responseBefore.body) == + "Invalid parameters: parameters must satify `1 < (nodes - tolerance) ≥ tolerance`" + + test "request storage fails if tolerance > nodes (underflow protection)": + let data = await RandomChunker.example(blocks = 2) + let cid = (await client.upload(data)).get + let duration = 100.uint64 + let pricePerBytePerSecond = 1.u256 + let proofProbability = 3.u256 + let expiry = 30.uint64 + let collateralPerByte = 1.u256 + let nodes = 3 + let tolerance = 0 + + var responseBefore = ( + await client.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, + expiry, nodes.uint, tolerance.uint, + ) + ) + + check responseBefore.status == 422 + check (await responseBefore.body) == "Tolerance needs to be bigger then zero" + + test "upload fails if content disposition contains bad filename": + let headers = @[("Content-Disposition", "attachment; filename=\"exam*ple.txt\"")] + let response = await client.uploadRaw("some file contents", headers) + + check response.status == 422 + check (await response.body) == "The filename is not valid." + + test "upload fails if content type is invalid": + let headers = @[("Content-Type", "hello/world")] + let response = await client.uploadRaw("some file contents", headers) + + check response.status == 422 + check (await response.body) == "The MIME type 'hello/world' is not valid." + + test "updating non-existing availability": + let nonExistingResponse = await client.patchAvailabilityRaw( + AvailabilityId.example, + duration = 100.uint64.some, + minPricePerBytePerSecond = 2.u256.some, + totalCollateral = 200.u256.some, + ) + check nonExistingResponse.status == 404 + + test "updating availability - freeSize is not allowed to be changed": + let availability = ( + await client.postAvailability( + totalSize = 140000.uint64, + duration = 200.uint64, + minPricePerBytePerSecond = 3.u256, + totalCollateral = 300.u256, + ) + ).get + let freeSizeResponse = + await client.patchAvailabilityRaw(availability.id, freeSize = 110000.uint64.some) + check freeSizeResponse.status == 422 + check "not allowed" in (await freeSizeResponse.body) + + test "creating availability above the node quota returns 422": + let response = await client.postAvailabilityRaw( + totalSize = 24000000000.uint64, + duration = 200.uint64, + minPricePerBytePerSecond = 3.u256, + totalCollateral = 300.u256, + ) + + check response.status == 422 + check (await response.body) == "Not enough storage quota" + + test "updating availability above the node quota returns 422": + let availability = ( + await client.postAvailability( + totalSize = 140000.uint64, + duration = 200.uint64, + minPricePerBytePerSecond = 3.u256, + totalCollateral = 300.u256, + ) + ).get + let response = await client.patchAvailabilityRaw( + availability.id, totalSize = 24000000000.uint64.some + ) + + check response.status == 422 + check (await response.body) == "Not enough storage quota" + + test "creating availability when total size is zero returns 422": + let response = await client.postAvailabilityRaw( + totalSize = 0.uint64, + duration = 200.uint64, + minPricePerBytePerSecond = 3.u256, + totalCollateral = 300.u256, + ) + + check response.status == 422 + check (await response.body) == "Total size must be larger then zero" + + test "updating availability when total size is zero returns 422": + let availability = ( + await client.postAvailability( + totalSize = 140000.uint64, + duration = 200.uint64, + minPricePerBytePerSecond = 3.u256, + totalCollateral = 300.u256, + ) + ).get + let response = + await client.patchAvailabilityRaw(availability.id, totalSize = 0.uint64.some) + + check response.status == 422 + check (await response.body) == "Total size must be larger then zero" + + test "creating availability when total size is negative returns 422": + let json = + %*{ + "totalSize": "-1", + "duration": "200", + "minPricePerBytePerSecond": "3", + "totalCollateral": "300", + } + let response = await client.post(client.buildUrl("/sales/availability"), $json) + + check response.status == 400 + check (await response.body) == "Parsed integer outside of valid range" + + test "updating availability when total size is negative returns 422": + let availability = ( + await client.postAvailability( + totalSize = 140000.uint64, + duration = 200.uint64, + minPricePerBytePerSecond = 3.u256, + totalCollateral = 300.u256, + ) + ).get + + let json = %*{"totalSize": "-1"} + let response = await client.patch( + client.buildUrl("/sales/availability/") & $availability.id, $json + ) + + check response.status == 400 + check (await response.body) == "Parsed integer outside of valid range" + + test "request storage fails if tolerance is zero": + let data = await RandomChunker.example(blocks = 2) + let cid = (await client.upload(data)).get + let duration = 100.uint64 + let pricePerBytePerSecond = 1.u256 + let proofProbability = 3.u256 + let expiry = 30.uint64 + let collateralPerByte = 1.u256 + let nodes = 3 + let tolerance = 0 + + var responseBefore = ( + await client.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, + expiry, nodes.uint, tolerance.uint, + ) + ) + + check responseBefore.status == 422 + check (await responseBefore.body) == "Tolerance needs to be bigger then zero" + + test "request storage fails if duration exceeds limit": + let data = await RandomChunker.example(blocks = 2) + let cid = (await client.upload(data)).get + let duration = (31 * 24 * 60 * 60).uint64 + # 31 days TODO: this should not be hardcoded, but waits for https://github.com/codex-storage/nim-codex/issues/1056 + let proofProbability = 3.u256 + let expiry = 30.uint + let collateralPerByte = 1.u256 + let nodes = 3 + let tolerance = 2 + let pricePerBytePerSecond = 1.u256 + + var responseBefore = ( + await client.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, + expiry, nodes.uint, tolerance.uint, + ) + ) + + check responseBefore.status == 422 + check "Duration exceeds limit of" in (await responseBefore.body) + + test "request storage fails if expiry is zero": + let data = await RandomChunker.example(blocks = 2) + let cid = (await client.upload(data)).get + let duration = 100.uint64 + let pricePerBytePerSecond = 1.u256 + let proofProbability = 3.u256 + let expiry = 0.uint64 + let collateralPerByte = 1.u256 + let nodes = 3 + let tolerance = 1 + + var responseBefore = await client.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, + nodes.uint, tolerance.uint, + ) + + check responseBefore.status == 422 + check (await responseBefore.body) == + "Expiry must be greater than zero and less than the request's duration" + + test "request storage fails if proof probability is zero": + let data = await RandomChunker.example(blocks = 2) + let cid = (await client.upload(data)).get + let duration = 100.uint64 + let pricePerBytePerSecond = 1.u256 + let proofProbability = 0.u256 + let expiry = 30.uint64 + let collateralPerByte = 1.u256 + let nodes = 3 + let tolerance = 1 + + var responseBefore = await client.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, + nodes.uint, tolerance.uint, + ) + + check responseBefore.status == 422 + check (await responseBefore.body) == "Proof probability must be greater than zero" + + test "request storage fails if price per byte per second is zero": + let data = await RandomChunker.example(blocks = 2) + let cid = (await client.upload(data)).get + let duration = 100.uint64 + let pricePerBytePerSecond = 0.u256 + let proofProbability = 3.u256 + let expiry = 30.uint64 + let collateralPerByte = 1.u256 + let nodes = 3 + let tolerance = 1 + + var responseBefore = await client.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, + nodes.uint, tolerance.uint, + ) + + check responseBefore.status == 422 + check (await responseBefore.body) == + "Price per byte per second must be greater than zero" + + test "request storage fails if collareral per byte is zero": + let data = await RandomChunker.example(blocks = 2) + let cid = (await client.upload(data)).get + let duration = 100.uint64 + let pricePerBytePerSecond = 1.u256 + let proofProbability = 3.u256 + let expiry = 30.uint64 + let collateralPerByte = 0.u256 + let nodes = 3 + let tolerance = 1 + + var responseBefore = await client.requestStorageRaw( + cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, + nodes.uint, tolerance.uint, + ) + + check responseBefore.status == 422 + check (await responseBefore.body) == "Collateral per byte must be greater than zero" + + waitFor node.stop() + node.removeDataDir() diff --git a/tests/integration/testsales.nim b/tests/integration/testsales.nim index 9bf8a97c..5e9b26df 100644 --- a/tests/integration/testsales.nim +++ b/tests/integration/testsales.nim @@ -1,5 +1,6 @@ import std/httpclient import pkg/codex/contracts +from pkg/codex/stores/repostore/types import DefaultQuotaBytes import ./twonodes import ../codex/examples import ../contracts/time @@ -69,15 +70,6 @@ multinodesuite "Sales": ).get check availability in (await host.getAvailabilities()).get - test "updating non-existing availability", salesConfig: - let nonExistingResponse = await host.patchAvailabilityRaw( - AvailabilityId.example, - duration = 100.uint64.some, - minPricePerBytePerSecond = 2.u256.some, - totalCollateral = 200.u256.some, - ) - check nonExistingResponse.status == 404 - test "updating availability", salesConfig: let availability = ( await host.postAvailability( @@ -103,20 +95,6 @@ multinodesuite "Sales": check updatedAvailability.totalSize == 140000.uint64 check updatedAvailability.freeSize == 140000.uint64 - test "updating availability - freeSize is not allowed to be changed", salesConfig: - let availability = ( - await host.postAvailability( - totalSize = 140000.uint64, - duration = 200.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 300.u256, - ) - ).get - let freeSizeResponse = - await host.patchAvailabilityRaw(availability.id, freeSize = 110000.uint64.some) - check freeSizeResponse.status == 400 - check "not allowed" in (await freeSizeResponse.body) - test "updating availability - updating totalSize", salesConfig: let availability = ( await host.postAvailability( @@ -176,7 +154,7 @@ multinodesuite "Sales": availability.id, totalSize = (utilizedSize - 1).some ) ) - check totalSizeResponse.status == 400 + check totalSizeResponse.status == 422 check "totalSize must be larger then current totalSize" in (await totalSizeResponse.body) diff --git a/tests/testIntegration.nim b/tests/testIntegration.nim index 9a2dc472..152d22dd 100644 --- a/tests/testIntegration.nim +++ b/tests/testIntegration.nim @@ -1,5 +1,6 @@ import ./integration/testcli import ./integration/testrestapi +import ./integration/testrestapivalidation import ./integration/testupdownload import ./integration/testsales import ./integration/testpurchasing From 60b6996eb0c4c2b85e49c65c7464d83222d7cdf2 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 26 Mar 2025 09:06:37 +0100 Subject: [PATCH 07/24] chore(marketplace): define raises for async pragma (#1165) * Define raises for async pragma * Update nim ethers * Replace CatchableError by MarketError --- codex/contracts/market.nim | 30 ++++++++++++++++++-------- codex/market.nim | 28 +++++++++++++++++------- tests/codex/helpers/mockmarket.nim | 34 ++++++++++++++++++++---------- vendor/nim-ethers | 2 +- 4 files changed, 65 insertions(+), 29 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 0b846099..74694285 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -76,7 +76,9 @@ proc config( return resolvedConfig -proc approveFunds(market: OnChainMarket, amount: UInt256) {.async.} = +proc approveFunds( + market: OnChainMarket, amount: UInt256 +) {.async: (raises: [CancelledError, MarketError]).} = debug "Approving tokens", amount convertEthersError("Failed to approve funds"): let tokenAddress = await market.contract.token() @@ -105,7 +107,9 @@ method getZkeyHash*( let config = await market.config() return some config.proofs.zkeyHash -method getSigner*(market: OnChainMarket): Future[Address] {.async.} = +method getSigner*( + market: OnChainMarket +): Future[Address] {.async: (raises: [CancelledError, MarketError]).} = convertEthersError("Failed to get signer address"): return await market.signer.getAddress() @@ -159,7 +163,9 @@ method mySlots*(market: OnChainMarket): Future[seq[SlotId]] {.async.} = return slots -method requestStorage(market: OnChainMarket, request: StorageRequest) {.async.} = +method requestStorage( + market: OnChainMarket, request: StorageRequest +) {.async: (raises: [CancelledError, MarketError]).} = convertEthersError("Failed to request storage"): debug "Requesting storage" await market.approveFunds(request.totalPrice()) @@ -243,7 +249,7 @@ method fillSlot( slotIndex: uint64, proof: Groth16Proof, collateral: UInt256, -) {.async.} = +) {.async: (raises: [CancelledError, MarketError]).} = convertEthersError("Failed to fill slot"): logScope: requestId @@ -260,7 +266,9 @@ method fillSlot( parent, ) -method freeSlot*(market: OnChainMarket, slotId: SlotId) {.async.} = +method freeSlot*( + market: OnChainMarket, slotId: SlotId +) {.async: (raises: [CancelledError, MarketError]).} = convertEthersError("Failed to free slot"): var freeSlot: Future[Confirmable] if rewardRecipient =? market.rewardRecipient: @@ -279,7 +287,9 @@ method freeSlot*(market: OnChainMarket, slotId: SlotId) {.async.} = discard await freeSlot.confirm(1) -method withdrawFunds(market: OnChainMarket, requestId: RequestId) {.async.} = +method withdrawFunds( + market: OnChainMarket, requestId: RequestId +) {.async: (raises: [CancelledError, MarketError]).} = convertEthersError("Failed to withdraw funds"): discard await market.contract.withdrawFunds(requestId).confirm(1) @@ -306,13 +316,15 @@ method getChallenge*( let overrides = CallOverrides(blockTag: some BlockTag.pending) return await market.contract.getChallenge(id, overrides) -method submitProof*(market: OnChainMarket, id: SlotId, proof: Groth16Proof) {.async.} = +method submitProof*( + market: OnChainMarket, id: SlotId, proof: Groth16Proof +) {.async: (raises: [CancelledError, MarketError]).} = convertEthersError("Failed to submit proof"): discard await market.contract.submitProof(id, proof).confirm(1) method markProofAsMissing*( market: OnChainMarket, id: SlotId, period: Period -) {.async.} = +) {.async: (raises: [CancelledError, MarketError]).} = convertEthersError("Failed to mark proof as missing"): discard await market.contract.markProofAsMissing(id, period).confirm(1) @@ -331,7 +343,7 @@ method canProofBeMarkedAsMissing*( method reserveSlot*( market: OnChainMarket, requestId: RequestId, slotIndex: uint64 -) {.async.} = +) {.async: (raises: [CancelledError, MarketError]).} = convertEthersError("Failed to reserve slot"): try: discard await market.contract diff --git a/codex/market.nim b/codex/market.nim index dd8e14ba..71cad9a9 100644 --- a/codex/market.nim +++ b/codex/market.nim @@ -74,7 +74,9 @@ method getZkeyHash*( ): Future[?string] {.base, async: (raises: [CancelledError, MarketError]).} = raiseAssert("not implemented") -method getSigner*(market: Market): Future[Address] {.base, async.} = +method getSigner*( + market: Market +): Future[Address] {.base, async: (raises: [CancelledError, MarketError]).} = raiseAssert("not implemented") method periodicity*( @@ -108,7 +110,9 @@ proc inDowntime*(market: Market, slotId: SlotId): Future[bool] {.async.} = let pntr = await market.getPointer(slotId) return pntr < downtime -method requestStorage*(market: Market, request: StorageRequest) {.base, async.} = +method requestStorage*( + market: Market, request: StorageRequest +) {.base, async: (raises: [CancelledError, MarketError]).} = raiseAssert("not implemented") method myRequests*(market: Market): Future[seq[RequestId]] {.base, async.} = @@ -161,13 +165,17 @@ method fillSlot*( slotIndex: uint64, proof: Groth16Proof, collateral: UInt256, -) {.base, async.} = +) {.base, async: (raises: [CancelledError, MarketError]).} = raiseAssert("not implemented") -method freeSlot*(market: Market, slotId: SlotId) {.base, async.} = +method freeSlot*( + market: Market, slotId: SlotId +) {.base, async: (raises: [CancelledError, MarketError]).} = raiseAssert("not implemented") -method withdrawFunds*(market: Market, requestId: RequestId) {.base, async.} = +method withdrawFunds*( + market: Market, requestId: RequestId +) {.base, async: (raises: [CancelledError, MarketError]).} = raiseAssert("not implemented") method subscribeRequests*( @@ -186,10 +194,14 @@ method getChallenge*( ): Future[ProofChallenge] {.base, async.} = raiseAssert("not implemented") -method submitProof*(market: Market, id: SlotId, proof: Groth16Proof) {.base, async.} = +method submitProof*( + market: Market, id: SlotId, proof: Groth16Proof +) {.base, async: (raises: [CancelledError, MarketError]).} = raiseAssert("not implemented") -method markProofAsMissing*(market: Market, id: SlotId, period: Period) {.base, async.} = +method markProofAsMissing*( + market: Market, id: SlotId, period: Period +) {.base, async: (raises: [CancelledError, MarketError]).} = raiseAssert("not implemented") method canProofBeMarkedAsMissing*( @@ -199,7 +211,7 @@ method canProofBeMarkedAsMissing*( method reserveSlot*( market: Market, requestId: RequestId, slotIndex: uint64 -) {.base, async.} = +) {.base, async: (raises: [CancelledError, MarketError]).} = raiseAssert("not implemented") method canReserveSlot*( diff --git a/tests/codex/helpers/mockmarket.nim b/tests/codex/helpers/mockmarket.nim index edf8a62d..03e76762 100644 --- a/tests/codex/helpers/mockmarket.nim +++ b/tests/codex/helpers/mockmarket.nim @@ -47,7 +47,7 @@ type config*: MarketplaceConfig canReserveSlot*: bool errorOnReserveSlot*: ?(ref MarketError) - errorOnFillSlot*: ?(ref CatchableError) + errorOnFillSlot*: ?(ref MarketError) clock: ?Clock Fulfillment* = object @@ -144,7 +144,9 @@ method loadConfig*( ): Future[?!void] {.async: (raises: [CancelledError]).} = discard -method getSigner*(market: MockMarket): Future[Address] {.async.} = +method getSigner*( + market: MockMarket +): Future[Address] {.async: (raises: [CancelledError, MarketError]).} = return market.signer method periodicity*( @@ -173,7 +175,9 @@ method repairRewardPercentage*( method getPointer*(market: MockMarket, slotId: SlotId): Future[uint8] {.async.} = return market.proofPointer -method requestStorage*(market: MockMarket, request: StorageRequest) {.async.} = +method requestStorage*( + market: MockMarket, request: StorageRequest +) {.async: (raises: [CancelledError, MarketError]).} = market.requested.add(request) var subscriptions = market.subscriptions.onRequest for subscription in subscriptions: @@ -311,10 +315,12 @@ method fillSlot*( slotIndex: uint64, proof: Groth16Proof, collateral: UInt256, -) {.async.} = +) {.async: (raises: [CancelledError, MarketError]).} = market.fillSlot(requestId, slotIndex, proof, market.signer, collateral) -method freeSlot*(market: MockMarket, slotId: SlotId) {.async.} = +method freeSlot*( + market: MockMarket, slotId: SlotId +) {.async: (raises: [CancelledError, MarketError]).} = market.freed.add(slotId) for s in market.filled: if slotId(s.requestId, s.slotIndex) == slotId: @@ -322,7 +328,9 @@ method freeSlot*(market: MockMarket, slotId: SlotId) {.async.} = break market.slotState[slotId] = SlotState.Free -method withdrawFunds*(market: MockMarket, requestId: RequestId) {.async.} = +method withdrawFunds*( + market: MockMarket, requestId: RequestId +) {.async: (raises: [CancelledError, MarketError]).} = market.withdrawn.add(requestId) if state =? market.requestState .? [requestId] and state == RequestState.Cancelled: @@ -352,12 +360,16 @@ method getChallenge*(mock: MockMarket, id: SlotId): Future[ProofChallenge] {.asy proc setProofEnd*(mock: MockMarket, id: SlotId, proofEnd: UInt256) = mock.proofEnds[id] = proofEnd -method submitProof*(mock: MockMarket, id: SlotId, proof: Groth16Proof) {.async.} = +method submitProof*( + mock: MockMarket, id: SlotId, proof: Groth16Proof +) {.async: (raises: [CancelledError, MarketError]).} = mock.submitted.add(proof) for subscription in mock.subscriptions.onProofSubmitted: subscription.callback(id) -method markProofAsMissing*(market: MockMarket, id: SlotId, period: Period) {.async.} = +method markProofAsMissing*( + market: MockMarket, id: SlotId, period: Period +) {.async: (raises: [CancelledError, MarketError]).} = market.markedAsMissingProofs.add(id) proc setCanProofBeMarkedAsMissing*(mock: MockMarket, id: SlotId, required: bool) = @@ -373,7 +385,7 @@ method canProofBeMarkedAsMissing*( method reserveSlot*( market: MockMarket, requestId: RequestId, slotIndex: uint64 -) {.async.} = +) {.async: (raises: [CancelledError, MarketError]).} = if error =? market.errorOnReserveSlot: raise error @@ -392,10 +404,10 @@ func setErrorOnReserveSlot*(market: MockMarket, error: ref MarketError) = else: some error -func setErrorOnFillSlot*(market: MockMarket, error: ref CatchableError) = +func setErrorOnFillSlot*(market: MockMarket, error: ref MarketError) = market.errorOnFillSlot = if error.isNil: - none (ref CatchableError) + none (ref MarketError) else: some error diff --git a/vendor/nim-ethers b/vendor/nim-ethers index b505ef1a..5d07b5db 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit b505ef1ab889be8161bb1efb4908e3dfde5bc1c9 +Subproject commit 5d07b5dbcf584b020c732e84cc8b7229ab3e1083 From 7deeb7d2b34862889e5bc30e31e44709ca60ff9f Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 26 Mar 2025 12:45:22 +0100 Subject: [PATCH 08/24] feat(marketplace): persistent availabilities (#1099) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add availability enabled parameter * Return bytes to availability when finished * Add until parameter * Remove debug message * Clean up and fix tests * Update documentations and cleanup * Avoid swallowing CancelledError * Move until validation to reservations module * Call onAvailabilityAdded callabck when the availability is enabled in sales * Remove until validation in restapi when creating an availability * Add openapi documentation * Use results instead of stew/results (#1112) * feat: request duration limit (#1057) * feat: request duration limit * Fix tests and duration type * Add custom error * Remove merge issue * Update codex contracts eth * Update market config and fix test * Fix SlotReservationsConfig syntax * Update dependencies * test: remove doubled test * chore: update contracts repo --------- Co-authored-by: Arnaud * fix(statemachine): do not raise from state.run (#1115) * fix(statemachine): do not raise from state.run * fix rebase * fix exception handling in SaleProvingSimulated.prove - re-raise CancelledError - don't return State on CatchableError - expect the Proofs_InvalidProof custom error instead of checking a string * asyncSpawn salesagent.onCancelled This was swallowing a KeyError in one of the tests (fixed in the previous commit) * remove error handling states in asyncstatemachine * revert unneeded changes * formatting * PR feedback, logging updates * chore(integration): simplify block expiration integration test (#1100) * chore(integration): simplify block expiration integration test * clean up * fix after rebase * perf: contract storage optimizations (#1094) * perf: contract storage optimizations * Apply optimization changes * Apply optimizing parameters sizing * Update codex-contracts-eth * bump latest changes in contracts branch * Change requestDurationLimit to uint64 * fix tests * fix tests --------- Co-authored-by: Arnaud Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com> * bump contracts to master (#1122) * Add availability enabled parameter * Return bytes to availability when finished * Add until parameter * Clean up and fix tests * Move until validation to reservations module * Apply suggestion changes: return the reservation module error * Apply suggestion changes for until dates * Apply suggestion changes: reorganize tests * Fix indent * Remove test related to timing issue * Add raises errors to async pragram and remove useless try except * Update open api documentation * Fix wording * Remove the httpClient restart statements * Use market.getRequestEnd to set validUntil * Remove returnBytes * Use clock.now in testing * Move the api validation file to the right file --------- Co-authored-by: Adam Uhlíř Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com> --- codex/rest/api.nim | 29 ++- codex/rest/json.nim | 2 + codex/sales.nim | 20 +- codex/sales/reservations.nim | 195 ++++++++++++------ codex/sales/salesagent.nim | 2 +- codex/sales/states/cancelled.nim | 4 +- codex/sales/states/errored.nim | 2 +- codex/sales/states/filling.nim | 2 +- codex/sales/states/finished.nim | 3 + codex/sales/states/ignored.nim | 5 +- codex/sales/states/preparing.nim | 8 +- codex/sales/states/slotreserving.nim | 4 +- codex/stores/repostore/operations.nim | 2 +- codex/stores/repostore/store.nim | 8 +- openapi.yaml | 13 +- tests/codex/examples.nim | 2 + tests/codex/helpers/mockreservations.nim | 3 + tests/codex/sales/states/testcancelled.nim | 7 +- tests/codex/sales/states/testerrored.nim | 7 +- tests/codex/sales/states/testfilling.nim | 1 - tests/codex/sales/states/testfinished.nim | 11 +- tests/codex/sales/states/testignored.nim | 7 +- tests/codex/sales/states/testpreparing.nim | 22 +- .../codex/sales/states/testslotreserving.nim | 1 - tests/codex/sales/testreservations.nim | 161 +++++++++++++-- tests/codex/sales/testsales.nim | 64 +++++- tests/integration/codexclient.nim | 29 ++- tests/integration/testmarketplace.nim | 2 + tests/integration/testproofs.nim | 4 +- tests/integration/testrestapi.nim | 1 + tests/integration/testrestapivalidation.nim | 16 ++ tests/integration/testsales.nim | 97 +++++++-- vendor/nim-datastore | 2 +- 33 files changed, 564 insertions(+), 172 deletions(-) diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 243d4ed6..ee493e03 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -484,10 +484,19 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = without availability =? ( await reservations.createAvailability( - restAv.totalSize, restAv.duration, restAv.minPricePerBytePerSecond, + restAv.totalSize, + restAv.duration, + restAv.minPricePerBytePerSecond, restAv.totalCollateral, + enabled = restAv.enabled |? true, + until = restAv.until |? 0, ) ), error: + if error of CancelledError: + raise error + if error of UntilOutOfBoundsError: + return RestApiResponse.error(Http422, error.msg) + return RestApiResponse.error(Http500, error.msg, headers = headers) return RestApiResponse.response( @@ -524,6 +533,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = ## tokens) to be matched against the request's pricePerBytePerSecond ## totalCollateral - total collateral (in amount of ## tokens) that can be distributed among matching requests + try: without contracts =? node.contracts.host: return RestApiResponse.error(Http503, "Persistence is not enabled") @@ -577,10 +587,21 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = if totalCollateral =? restAv.totalCollateral: availability.totalCollateral = totalCollateral - if err =? (await reservations.update(availability)).errorOption: - return RestApiResponse.error(Http500, err.msg) + if until =? restAv.until: + availability.until = until - return RestApiResponse.response(Http200) + if enabled =? restAv.enabled: + availability.enabled = enabled + + if err =? (await reservations.update(availability)).errorOption: + if err of CancelledError: + raise err + if err of UntilOutOfBoundsError: + return RestApiResponse.error(Http422, err.msg) + else: + return RestApiResponse.error(Http500, err.msg) + + return RestApiResponse.response(Http204) except CatchableError as exc: trace "Excepting processing request", exc = exc.msg return RestApiResponse.error(Http500) diff --git a/codex/rest/json.nim b/codex/rest/json.nim index 50c8b514..1b9459c1 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -33,6 +33,8 @@ type minPricePerBytePerSecond* {.serialize.}: UInt256 totalCollateral* {.serialize.}: UInt256 freeSize* {.serialize.}: ?uint64 + enabled* {.serialize.}: ?bool + until* {.serialize.}: ?SecondsSince1970 RestSalesAgent* = object state* {.serialize.}: string diff --git a/codex/sales.nim b/codex/sales.nim index a4a174c1..37e2c06a 100644 --- a/codex/sales.nim +++ b/codex/sales.nim @@ -113,7 +113,6 @@ proc remove(sales: Sales, agent: SalesAgent) {.async.} = proc cleanUp( sales: Sales, agent: SalesAgent, - returnBytes: bool, reprocessSlot: bool, returnedCollateral: ?UInt256, processing: Future[void], @@ -132,7 +131,7 @@ proc cleanUp( # if reservation for the SalesAgent was not created, then it means # that the cleanUp was called before the sales process really started, so # there are not really any bytes to be returned - if returnBytes and request =? data.request and reservation =? data.reservation: + if request =? data.request and reservation =? data.reservation: if returnErr =? ( await sales.context.reservations.returnBytesToAvailability( reservation.availabilityId, reservation.id, request.ask.slotSize @@ -203,9 +202,9 @@ proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) = newSalesAgent(sales.context, item.requestId, item.slotIndex, none StorageRequest) agent.onCleanUp = proc( - returnBytes = false, reprocessSlot = false, returnedCollateral = UInt256.none + reprocessSlot = false, returnedCollateral = UInt256.none ) {.async.} = - await sales.cleanUp(agent, returnBytes, reprocessSlot, returnedCollateral, done) + await sales.cleanUp(agent, reprocessSlot, returnedCollateral, done) agent.onFilled = some proc(request: StorageRequest, slotIndex: uint64) = sales.filled(request, slotIndex, done) @@ -271,12 +270,12 @@ proc load*(sales: Sales) {.async.} = newSalesAgent(sales.context, slot.request.id, slot.slotIndex, some slot.request) agent.onCleanUp = proc( - returnBytes = false, reprocessSlot = false, returnedCollateral = UInt256.none + reprocessSlot = false, returnedCollateral = UInt256.none ) {.async.} = # since workers are not being dispatched, this future has not been created # by a worker. Create a dummy one here so we can call sales.cleanUp let done: Future[void] = nil - await sales.cleanUp(agent, returnBytes, reprocessSlot, returnedCollateral, done) + await sales.cleanUp(agent, reprocessSlot, returnedCollateral, done) # There is no need to assign agent.onFilled as slots loaded from `mySlots` # are inherently already filled and so assigning agent.onFilled would be @@ -285,7 +284,9 @@ proc load*(sales: Sales) {.async.} = agent.start(SaleUnknown()) sales.agents.add agent -proc OnAvailabilitySaved(sales: Sales, availability: Availability) {.async.} = +proc OnAvailabilitySaved( + sales: Sales, availability: Availability +) {.async: (raises: []).} = ## When availabilities are modified or added, the queue should be unpaused if ## it was paused and any slots in the queue should have their `seen` flag ## cleared. @@ -533,8 +534,9 @@ proc startSlotQueue(sales: Sales) = slotQueue.start() - proc OnAvailabilitySaved(availability: Availability) {.async.} = - await sales.OnAvailabilitySaved(availability) + proc OnAvailabilitySaved(availability: Availability) {.async: (raises: []).} = + if availability.enabled: + await sales.OnAvailabilitySaved(availability) reservations.OnAvailabilitySaved = OnAvailabilitySaved diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index 25ee2b99..b717cc1c 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -35,6 +35,7 @@ import std/sequtils import std/sugar import std/typetraits import std/sequtils +import std/times import pkg/chronos import pkg/datastore import pkg/nimcrypto @@ -70,6 +71,12 @@ type minPricePerBytePerSecond* {.serialize.}: UInt256 totalCollateral {.serialize.}: UInt256 totalRemainingCollateral* {.serialize.}: UInt256 + # If set to false, the availability will not accept new slots. + # If enabled, it will not impact any existing slots that are already being hosted. + enabled* {.serialize.}: bool + # Specifies the latest timestamp after which the availability will no longer host any slots. + # If set to 0, there will be no restrictions. + until* {.serialize.}: SecondsSince1970 Reservation* = ref object id* {.serialize.}: ReservationId @@ -77,6 +84,7 @@ type size* {.serialize.}: uint64 requestId* {.serialize.}: RequestId slotIndex* {.serialize.}: uint64 + validUntil* {.serialize.}: SecondsSince1970 Reservations* = ref object of RootObj availabilityLock: AsyncLock @@ -84,10 +92,14 @@ type repo: RepoStore OnAvailabilitySaved: ?OnAvailabilitySaved - GetNext* = proc(): Future[?seq[byte]] {.upraises: [], gcsafe, closure.} - IterDispose* = proc(): Future[?!void] {.gcsafe, closure.} - OnAvailabilitySaved* = - proc(availability: Availability): Future[void] {.upraises: [], gcsafe.} + GetNext* = proc(): Future[?seq[byte]] {. + upraises: [], gcsafe, async: (raises: [CancelledError]), closure + .} + IterDispose* = + proc(): Future[?!void] {.gcsafe, async: (raises: [CancelledError]), closure.} + OnAvailabilitySaved* = proc(availability: Availability): Future[void] {. + upraises: [], gcsafe, async: (raises: []) + .} StorableIter* = ref object finished*: bool next*: GetNext @@ -102,13 +114,20 @@ type SerializationError* = object of ReservationsError UpdateFailedError* = object of ReservationsError BytesOutOfBoundsError* = object of ReservationsError + UntilOutOfBoundsError* = object of ReservationsError const SalesKey = (CodexMetaKey / "sales").tryGet # TODO: move to sales module ReservationsKey = (SalesKey / "reservations").tryGet proc hash*(x: AvailabilityId): Hash {.borrow.} -proc all*(self: Reservations, T: type SomeStorableObject): Future[?!seq[T]] {.async.} +proc all*( + self: Reservations, T: type SomeStorableObject +): Future[?!seq[T]] {.async: (raises: [CancelledError]).} + +proc all*( + self: Reservations, T: type SomeStorableObject, availabilityId: AvailabilityId +): Future[?!seq[T]] {.async: (raises: [CancelledError]).} template withLock(lock, body) = try: @@ -128,6 +147,8 @@ proc init*( duration: uint64, minPricePerBytePerSecond: UInt256, totalCollateral: UInt256, + enabled: bool, + until: SecondsSince1970, ): Availability = var id: array[32, byte] doAssert randomBytes(id) == 32 @@ -139,6 +160,8 @@ proc init*( minPricePerBytePerSecond: minPricePerBytePerSecond, totalCollateral: totalCollateral, totalRemainingCollateral: totalCollateral, + enabled: enabled, + until: until, ) func totalCollateral*(self: Availability): UInt256 {.inline.} = @@ -154,6 +177,7 @@ proc init*( size: uint64, requestId: RequestId, slotIndex: uint64, + validUntil: SecondsSince1970, ): Reservation = var id: array[32, byte] doAssert randomBytes(id) == 32 @@ -163,6 +187,7 @@ proc init*( size: size, requestId: requestId, slotIndex: slotIndex, + validUntil: validUntil, ) func toArray(id: SomeStorableId): array[32, byte] = @@ -217,11 +242,19 @@ func available*(self: Reservations): uint = func hasAvailable*(self: Reservations, bytes: uint): bool = self.repo.available(bytes.NBytes) -proc exists*(self: Reservations, key: Key): Future[bool] {.async.} = +proc exists*( + self: Reservations, key: Key +): Future[bool] {.async: (raises: [CancelledError]).} = let exists = await self.repo.metaDs.ds.contains(key) return exists -proc getImpl(self: Reservations, key: Key): Future[?!seq[byte]] {.async.} = +iterator items(self: StorableIter): Future[?seq[byte]] = + while not self.finished: + yield self.next() + +proc getImpl( + self: Reservations, key: Key +): Future[?!seq[byte]] {.async: (raises: [CancelledError]).} = if not await self.exists(key): let err = newException(NotExistsError, "object with key " & $key & " does not exist") @@ -234,7 +267,7 @@ proc getImpl(self: Reservations, key: Key): Future[?!seq[byte]] {.async.} = proc get*( self: Reservations, key: Key, T: type SomeStorableObject -): Future[?!T] {.async.} = +): Future[?!T] {.async: (raises: [CancelledError]).} = without serialized =? await self.getImpl(key), error: return failure(error) @@ -243,7 +276,9 @@ proc get*( return success obj -proc updateImpl(self: Reservations, obj: SomeStorableObject): Future[?!void] {.async.} = +proc updateImpl( + self: Reservations, obj: SomeStorableObject +): Future[?!void] {.async: (raises: [CancelledError]).} = trace "updating " & $(obj.type), id = obj.id without key =? obj.key, error: @@ -256,10 +291,15 @@ proc updateImpl(self: Reservations, obj: SomeStorableObject): Future[?!void] {.a proc updateAvailability( self: Reservations, obj: Availability -): Future[?!void] {.async.} = +): Future[?!void] {.async: (raises: [CancelledError]).} = logScope: availabilityId = obj.id + if obj.until < 0: + let error = + newException(UntilOutOfBoundsError, "Cannot set until to a negative value") + return failure(error) + without key =? obj.key, error: return failure(error) @@ -269,21 +309,25 @@ proc updateAvailability( let res = await self.updateImpl(obj) # inform subscribers that Availability has been added if OnAvailabilitySaved =? self.OnAvailabilitySaved: - # when chronos v4 is implemented, and OnAvailabilitySaved is annotated - # with async:(raises:[]), we can remove this try/catch as we know, with - # certainty, that nothing will be raised - try: - await OnAvailabilitySaved(obj) - except CancelledError as e: - raise e - except CatchableError as e: - # we don't have any insight into types of exceptions that - # `OnAvailabilitySaved` can raise because it is caller-defined - warn "Unknown error during 'OnAvailabilitySaved' callback", error = e.msg + await OnAvailabilitySaved(obj) return res else: return failure(err) + if obj.until > 0: + without allReservations =? await self.all(Reservation, obj.id), error: + error.msg = "Error updating reservation: " & error.msg + return failure(error) + + let requestEnds = allReservations.mapIt(it.validUntil) + + if requestEnds.len > 0 and requestEnds.max > obj.until: + let error = newException( + UntilOutOfBoundsError, + "Until parameter must be greater or equal to the longest currently hosted slot", + ) + return failure(error) + # Sizing of the availability changed, we need to adjust the repo reservation accordingly if oldAvailability.totalSize != obj.totalSize: trace "totalSize changed, updating repo reservation" @@ -306,26 +350,23 @@ proc updateAvailability( # inform subscribers that Availability has been modified (with increased # size) if OnAvailabilitySaved =? self.OnAvailabilitySaved: - # when chronos v4 is implemented, and OnAvailabilitySaved is annotated - # with async:(raises:[]), we can remove this try/catch as we know, with - # certainty, that nothing will be raised - try: - await OnAvailabilitySaved(obj) - except CancelledError as e: - raise e - except CatchableError as e: - # we don't have any insight into types of exceptions that - # `OnAvailabilitySaved` can raise because it is caller-defined - warn "Unknown error during 'OnAvailabilitySaved' callback", error = e.msg - + await OnAvailabilitySaved(obj) return res -proc update*(self: Reservations, obj: Reservation): Future[?!void] {.async.} = +proc update*( + self: Reservations, obj: Reservation +): Future[?!void] {.async: (raises: [CancelledError]).} = return await self.updateImpl(obj) -proc update*(self: Reservations, obj: Availability): Future[?!void] {.async.} = - withLock(self.availabilityLock): - return await self.updateAvailability(obj) +proc update*( + self: Reservations, obj: Availability +): Future[?!void] {.async: (raises: [CancelledError]).} = + try: + withLock(self.availabilityLock): + return await self.updateAvailability(obj) + except AsyncLockError as e: + error "Lock error when trying to update the availability", err = e.msg + return failure(e) proc delete(self: Reservations, key: Key): Future[?!void] {.async.} = trace "deleting object", key @@ -391,12 +432,20 @@ proc createAvailability*( duration: uint64, minPricePerBytePerSecond: UInt256, totalCollateral: UInt256, + enabled: bool, + until: SecondsSince1970, ): Future[?!Availability] {.async.} = trace "creating availability", - size, duration, minPricePerBytePerSecond, totalCollateral + size, duration, minPricePerBytePerSecond, totalCollateral, enabled, until - let availability = - Availability.init(size, size, duration, minPricePerBytePerSecond, totalCollateral) + if until < 0: + let error = + newException(UntilOutOfBoundsError, "Cannot set until to a negative value") + return failure(error) + + let availability = Availability.init( + size, size, duration, minPricePerBytePerSecond, totalCollateral, enabled, until + ) let bytes = availability.freeSize if reserveErr =? (await self.repo.reserve(bytes.NBytes)).errorOption: @@ -420,6 +469,7 @@ method createReservation*( requestId: RequestId, slotIndex: uint64, collateralPerByte: UInt256, + validUntil: SecondsSince1970, ): Future[?!Reservation] {.async, base.} = withLock(self.availabilityLock): without availabilityKey =? availabilityId.key, error: @@ -436,9 +486,11 @@ method createReservation*( ) return failure(error) - trace "Creating reservation", availabilityId, slotSize, requestId, slotIndex + trace "Creating reservation", + availabilityId, slotSize, requestId, slotIndex, validUntil = validUntil - let reservation = Reservation.init(availabilityId, slotSize, requestId, slotIndex) + let reservation = + Reservation.init(availabilityId, slotSize, requestId, slotIndex, validUntil) if createResErr =? (await self.update(reservation)).errorOption: return failure(createResErr) @@ -448,7 +500,7 @@ method createReservation*( availability.freeSize -= slotSize # adjust the remaining totalRemainingCollateral - availability.totalRemainingCollateral -= slotSize.stuint(256) * collateralPerByte + availability.totalRemainingCollateral -= slotSize.u256 * collateralPerByte # update availability with reduced size trace "Updating availability with reduced size" @@ -527,7 +579,7 @@ proc release*( reservationId: ReservationId, availabilityId: AvailabilityId, bytes: uint, -): Future[?!void] {.async.} = +): Future[?!void] {.async: (raises: [CancelledError]).} = logScope: topics = "release" bytes @@ -565,13 +617,9 @@ proc release*( return success() -iterator items(self: StorableIter): Future[?seq[byte]] = - while not self.finished: - yield self.next() - proc storables( self: Reservations, T: type SomeStorableObject, queryKey: Key = ReservationsKey -): Future[?!StorableIter] {.async.} = +): Future[?!StorableIter] {.async: (raises: [CancelledError]).} = var iter = StorableIter() let query = Query.init(queryKey) when T is Availability: @@ -589,7 +637,7 @@ proc storables( return failure(error) # /sales/reservations - proc next(): Future[?seq[byte]] {.async.} = + proc next(): Future[?seq[byte]] {.async: (raises: [CancelledError]).} = await idleAsync() iter.finished = results.finished if not results.finished and res =? (await results.next()) and res.data.len > 0 and @@ -598,7 +646,7 @@ proc storables( return none seq[byte] - proc dispose(): Future[?!void] {.async.} = + proc dispose(): Future[?!void] {.async: (raises: [CancelledError]).} = return await results.dispose() iter.next = next @@ -607,32 +655,40 @@ proc storables( proc allImpl( self: Reservations, T: type SomeStorableObject, queryKey: Key = ReservationsKey -): Future[?!seq[T]] {.async.} = +): Future[?!seq[T]] {.async: (raises: [CancelledError]).} = var ret: seq[T] = @[] without storables =? (await self.storables(T, queryKey)), error: return failure(error) for storable in storables.items: - without bytes =? (await storable): - continue + try: + without bytes =? (await storable): + continue - without obj =? T.fromJson(bytes), error: - error "json deserialization error", - json = string.fromBytes(bytes), error = error.msg - continue + without obj =? T.fromJson(bytes), error: + error "json deserialization error", + json = string.fromBytes(bytes), error = error.msg + continue - ret.add obj + ret.add obj + except CancelledError as err: + raise err + except CatchableError as err: + error "Error when retrieving storable", error = err.msg + continue return success(ret) -proc all*(self: Reservations, T: type SomeStorableObject): Future[?!seq[T]] {.async.} = +proc all*( + self: Reservations, T: type SomeStorableObject +): Future[?!seq[T]] {.async: (raises: [CancelledError]).} = return await self.allImpl(T) proc all*( self: Reservations, T: type SomeStorableObject, availabilityId: AvailabilityId -): Future[?!seq[T]] {.async.} = - without key =? (ReservationsKey / $availabilityId): +): Future[?!seq[T]] {.async: (raises: [CancelledError]).} = + without key =? key(availabilityId): return failure("no key") return await self.allImpl(T, key) @@ -641,6 +697,7 @@ proc findAvailability*( self: Reservations, size, duration: uint64, pricePerBytePerSecond, collateralPerByte: UInt256, + validUntil: SecondsSince1970, ): Future[?Availability] {.async.} = without storables =? (await self.storables(Availability)), e: error "failed to get all storables", error = e.msg @@ -648,11 +705,14 @@ proc findAvailability*( for item in storables.items: if bytes =? (await item) and availability =? Availability.fromJson(bytes): - if size <= availability.freeSize and duration <= availability.duration and + if availability.enabled and size <= availability.freeSize and + duration <= availability.duration and collateralPerByte <= availability.maxCollateralPerByte and - pricePerBytePerSecond >= availability.minPricePerBytePerSecond: + pricePerBytePerSecond >= availability.minPricePerBytePerSecond and + (availability.until == 0 or availability.until >= validUntil): trace "availability matched", id = availability.id, + enabled = availability.enabled, size, availFreeSize = availability.freeSize, duration, @@ -660,7 +720,8 @@ proc findAvailability*( pricePerBytePerSecond, availMinPricePerBytePerSecond = availability.minPricePerBytePerSecond, collateralPerByte, - availMaxCollateralPerByte = availability.maxCollateralPerByte + availMaxCollateralPerByte = availability.maxCollateralPerByte, + until = availability.until # TODO: As soon as we're on ARC-ORC, we can use destructors # to automatically dispose our iterators when they fall out of scope. @@ -672,6 +733,7 @@ proc findAvailability*( trace "availability did not match", id = availability.id, + enabled = availability.enabled, size, availFreeSize = availability.freeSize, duration, @@ -679,4 +741,5 @@ proc findAvailability*( pricePerBytePerSecond, availMinPricePerBytePerSecond = availability.minPricePerBytePerSecond, collateralPerByte, - availMaxCollateralPerByte = availability.maxCollateralPerByte + availMaxCollateralPerByte = availability.maxCollateralPerByte, + until = availability.until diff --git a/codex/sales/salesagent.nim b/codex/sales/salesagent.nim index f0abf3ee..61f3a9d3 100644 --- a/codex/sales/salesagent.nim +++ b/codex/sales/salesagent.nim @@ -27,7 +27,7 @@ type onFilled*: ?OnFilled OnCleanUp* = proc( - returnBytes = false, reprocessSlot = false, returnedCollateral = UInt256.none + reprocessSlot = false, returnedCollateral = UInt256.none ): Future[void] {.gcsafe, upraises: [].} OnFilled* = proc(request: StorageRequest, slotIndex: uint64) {.gcsafe, upraises: [].} diff --git a/codex/sales/states/cancelled.nim b/codex/sales/states/cancelled.nim index 3bdf8c2f..2c240e15 100644 --- a/codex/sales/states/cancelled.nim +++ b/codex/sales/states/cancelled.nim @@ -34,9 +34,7 @@ method run*( if onCleanUp =? agent.onCleanUp: await onCleanUp( - returnBytes = true, - reprocessSlot = false, - returnedCollateral = some currentCollateral, + reprocessSlot = false, returnedCollateral = some currentCollateral ) warn "Sale cancelled due to timeout", diff --git a/codex/sales/states/errored.nim b/codex/sales/states/errored.nim index 77bf08d3..95848fd3 100644 --- a/codex/sales/states/errored.nim +++ b/codex/sales/states/errored.nim @@ -34,7 +34,7 @@ method run*( onClear(request, data.slotIndex) if onCleanUp =? agent.onCleanUp: - await onCleanUp(returnBytes = true, reprocessSlot = state.reprocessSlot) + await onCleanUp(reprocessSlot = state.reprocessSlot) except CancelledError as e: trace "SaleErrored.run was cancelled", error = e.msgDetail except CatchableError as e: diff --git a/codex/sales/states/filling.nim b/codex/sales/states/filling.nim index 13644223..1b76150a 100644 --- a/codex/sales/states/filling.nim +++ b/codex/sales/states/filling.nim @@ -50,7 +50,7 @@ method run*( await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral) except SlotStateMismatchError as e: debug "Slot is already filled, ignoring slot" - return some State(SaleIgnored(reprocessSlot: false, returnBytes: true)) + return some State(SaleIgnored(reprocessSlot: false)) except MarketError as e: return some State(SaleErrored(error: e)) # other CatchableErrors are handled "automatically" by the SaleState diff --git a/codex/sales/states/finished.nim b/codex/sales/states/finished.nim index 2aba69eb..16e66d27 100644 --- a/codex/sales/states/finished.nim +++ b/codex/sales/states/finished.nim @@ -36,6 +36,9 @@ method run*( requestId = data.requestId, slotIndex = data.slotIndex try: + if onClear =? agent.context.onClear: + onClear(request, data.slotIndex) + if onCleanUp =? agent.onCleanUp: await onCleanUp(returnedCollateral = state.returnedCollateral) except CancelledError as e: diff --git a/codex/sales/states/ignored.nim b/codex/sales/states/ignored.nim index b07a201c..7f2ae5b1 100644 --- a/codex/sales/states/ignored.nim +++ b/codex/sales/states/ignored.nim @@ -14,7 +14,6 @@ logScope: type SaleIgnored* = ref object of SaleState reprocessSlot*: bool # readd slot to queue with `seen` flag - returnBytes*: bool # return unreleased bytes from Reservation to Availability method `$`*(state: SaleIgnored): string = "SaleIgnored" @@ -26,9 +25,7 @@ method run*( try: if onCleanUp =? agent.onCleanUp: - await onCleanUp( - reprocessSlot = state.reprocessSlot, returnBytes = state.returnBytes - ) + await onCleanUp(reprocessSlot = state.reprocessSlot) except CancelledError as e: trace "SaleIgnored.run was cancelled", error = e.msgDetail except CatchableError as e: diff --git a/codex/sales/states/preparing.nim b/codex/sales/states/preparing.nim index 443aee0b..a3aee4c9 100644 --- a/codex/sales/states/preparing.nim +++ b/codex/sales/states/preparing.nim @@ -56,7 +56,7 @@ method run*( let slotId = slotId(data.requestId, data.slotIndex) let state = await market.slotState(slotId) if state != SlotState.Free and state != SlotState.Repair: - return some State(SaleIgnored(reprocessSlot: false, returnBytes: false)) + return some State(SaleIgnored(reprocessSlot: false)) # TODO: Once implemented, check to ensure the host is allowed to fill the slot, # due to the [sliding window mechanism](https://github.com/codex-storage/codex-research/blob/master/design/marketplace.md#dispersal) @@ -68,10 +68,12 @@ method run*( pricePerBytePerSecond = request.ask.pricePerBytePerSecond collateralPerByte = request.ask.collateralPerByte + let requestEnd = await market.getRequestEnd(data.requestId) + without availability =? await reservations.findAvailability( request.ask.slotSize, request.ask.duration, request.ask.pricePerBytePerSecond, - request.ask.collateralPerByte, + request.ask.collateralPerByte, requestEnd, ): debug "No availability found for request, ignoring" @@ -82,7 +84,7 @@ method run*( without reservation =? await reservations.createReservation( availability.id, request.ask.slotSize, request.id, data.slotIndex, - request.ask.collateralPerByte, + request.ask.collateralPerByte, requestEnd, ), error: trace "Creation of reservation failed" # Race condition: diff --git a/codex/sales/states/slotreserving.nim b/codex/sales/states/slotreserving.nim index e9ac8dcd..780dadfc 100644 --- a/codex/sales/states/slotreserving.nim +++ b/codex/sales/states/slotreserving.nim @@ -46,7 +46,7 @@ method run*( await market.reserveSlot(data.requestId, data.slotIndex) except SlotReservationNotAllowedError as e: debug "Slot cannot be reserved, ignoring", error = e.msg - return some State(SaleIgnored(reprocessSlot: false, returnBytes: true)) + return some State(SaleIgnored(reprocessSlot: false)) except MarketError as e: return some State(SaleErrored(error: e)) # other CatchableErrors are handled "automatically" by the SaleState @@ -57,7 +57,7 @@ method run*( # do not re-add this slot to the queue, and return bytes from Reservation to # the Availability debug "Slot cannot be reserved, ignoring" - return some State(SaleIgnored(reprocessSlot: false, returnBytes: true)) + return some State(SaleIgnored(reprocessSlot: false)) except CancelledError as e: trace "SaleSlotReserving.run was cancelled", error = e.msgDetail except CatchableError as e: diff --git a/codex/stores/repostore/operations.nim b/codex/stores/repostore/operations.nim index 125741e1..cc488240 100644 --- a/codex/stores/repostore/operations.nim +++ b/codex/stores/repostore/operations.nim @@ -105,7 +105,7 @@ proc updateQuotaUsage*( minusUsed: NBytes = 0.NBytes, plusReserved: NBytes = 0.NBytes, minusReserved: NBytes = 0.NBytes, -): Future[?!void] {.async.} = +): Future[?!void] {.async: (raises: [CancelledError]).} = await self.metaDs.modify( QuotaUsedKey, proc(maybeCurrUsage: ?QuotaUsage): Future[?QuotaUsage] {.async.} = diff --git a/codex/stores/repostore/store.nim b/codex/stores/repostore/store.nim index d7305107..130ab15e 100644 --- a/codex/stores/repostore/store.nim +++ b/codex/stores/repostore/store.nim @@ -380,7 +380,9 @@ method close*(self: RepoStore): Future[void] {.async.} = # RepoStore procs ########################################################### -proc reserve*(self: RepoStore, bytes: NBytes): Future[?!void] {.async.} = +proc reserve*( + self: RepoStore, bytes: NBytes +): Future[?!void] {.async: (raises: [CancelledError]).} = ## Reserve bytes ## @@ -388,7 +390,9 @@ proc reserve*(self: RepoStore, bytes: NBytes): Future[?!void] {.async.} = await self.updateQuotaUsage(plusReserved = bytes) -proc release*(self: RepoStore, bytes: NBytes): Future[?!void] {.async.} = +proc release*( + self: RepoStore, bytes: NBytes +): Future[?!void] {.async: (raises: [CancelledError]).} = ## Release bytes ## diff --git a/openapi.yaml b/openapi.yaml index c2088cc5..8bae1b10 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -163,6 +163,14 @@ components: totalCollateral: type: string description: Total collateral (in amount of tokens) that can be used for matching requests + enabled: + type: boolean + description: Enable the ability to receive sales on this availability. + default: true + until: + type: integer + description: Specifies the latest timestamp, after which the availability will no longer host any slots. If set to 0, there will be no restrictions. + default: 0 SalesAvailabilityREAD: allOf: @@ -239,6 +247,9 @@ components: slotIndex: type: string description: Slot Index as decimal string + validUntil: + type: integer + description: Timestamp after which the reservation will no longer be valid. StorageRequestCreation: type: object @@ -704,7 +715,7 @@ paths: "400": description: Invalid data input "422": - description: The provided parameters did not pass validation + description: Not enough node's storage quota available or the provided parameters did not pass validation "500": description: Error reserving availability "503": diff --git a/tests/codex/examples.nim b/tests/codex/examples.nim index ed1dd52a..52b8a0b8 100644 --- a/tests/codex/examples.nim +++ b/tests/codex/examples.nim @@ -75,6 +75,8 @@ proc example*( duration = uint16.example.uint64, minPricePerBytePerSecond = uint8.example.u256, totalCollateral = totalSize.u256 * collateralPerByte, + enabled = true, + until = 0.SecondsSince1970, ) proc example*(_: type Reservation): Reservation = diff --git a/tests/codex/helpers/mockreservations.nim b/tests/codex/helpers/mockreservations.nim index 1bc76a09..91ed04ec 100644 --- a/tests/codex/helpers/mockreservations.nim +++ b/tests/codex/helpers/mockreservations.nim @@ -2,6 +2,7 @@ import pkg/chronos import pkg/codex/sales import pkg/codex/stores import pkg/questionable/results +import pkg/codex/clock type MockReservations* = ref object of Reservations createReservationThrowBytesOutOfBoundsError: bool @@ -28,6 +29,7 @@ method createReservation*( requestId: RequestId, slotIndex: uint64, collateralPerByte: UInt256, + validUntil: SecondsSince1970, ): Future[?!Reservation] {.async.} = if self.createReservationThrowBytesOutOfBoundsError: let error = newException( @@ -45,4 +47,5 @@ method createReservation*( requestId, slotIndex, collateralPerByte, + validUntil, ) diff --git a/tests/codex/sales/states/testcancelled.nim b/tests/codex/sales/states/testcancelled.nim index 48f3e8a0..ab450200 100644 --- a/tests/codex/sales/states/testcancelled.nim +++ b/tests/codex/sales/states/testcancelled.nim @@ -22,16 +22,14 @@ asyncchecksuite "sales state 'cancelled'": var market: MockMarket var state: SaleCancelled var agent: SalesAgent - var returnBytesWas = bool.none var reprocessSlotWas = bool.none var returnedCollateralValue = UInt256.none setup: market = MockMarket.new() let onCleanUp = proc( - returnBytes = false, reprocessSlot = false, returnedCollateral = UInt256.none + reprocessSlot = false, returnedCollateral = UInt256.none ) {.async.} = - returnBytesWas = some returnBytes reprocessSlotWas = some reprocessSlot returnedCollateralValue = returnedCollateral @@ -40,7 +38,7 @@ asyncchecksuite "sales state 'cancelled'": agent.onCleanUp = onCleanUp state = SaleCancelled.new() - test "calls onCleanUp with returnBytes = false, reprocessSlot = true, and returnedCollateral = currentCollateral": + test "calls onCleanUp with reprocessSlot = true, and returnedCollateral = currentCollateral": market.fillSlot( requestId = request.id, slotIndex = slotIndex, @@ -49,6 +47,5 @@ asyncchecksuite "sales state 'cancelled'": collateral = currentCollateral, ) discard await state.run(agent) - check eventually returnBytesWas == some true check eventually reprocessSlotWas == some false check eventually returnedCollateralValue == some currentCollateral diff --git a/tests/codex/sales/states/testerrored.nim b/tests/codex/sales/states/testerrored.nim index 07e325e3..0cc26cf8 100644 --- a/tests/codex/sales/states/testerrored.nim +++ b/tests/codex/sales/states/testerrored.nim @@ -20,14 +20,12 @@ asyncchecksuite "sales state 'errored'": var state: SaleErrored var agent: SalesAgent - var returnBytesWas = false var reprocessSlotWas = false setup: let onCleanUp = proc( - returnBytes = false, reprocessSlot = false, returnedCollateral = UInt256.none + reprocessSlot = false, returnedCollateral = UInt256.none ) {.async.} = - returnBytesWas = returnBytes reprocessSlotWas = reprocessSlot let context = SalesContext(market: market, clock: clock) @@ -35,8 +33,7 @@ asyncchecksuite "sales state 'errored'": agent.onCleanUp = onCleanUp state = SaleErrored(error: newException(ValueError, "oh no!")) - test "calls onCleanUp with returnBytes = false and reprocessSlot = true": + test "calls onCleanUp with reprocessSlot = true": state = SaleErrored(error: newException(ValueError, "oh no!"), reprocessSlot: true) discard await state.run(agent) - check eventually returnBytesWas == true check eventually reprocessSlotWas == true diff --git a/tests/codex/sales/states/testfilling.nim b/tests/codex/sales/states/testfilling.nim index f746b5a8..54536a4c 100644 --- a/tests/codex/sales/states/testfilling.nim +++ b/tests/codex/sales/states/testfilling.nim @@ -47,7 +47,6 @@ suite "sales state 'filling'": let next = !(await state.run(agent)) check next of SaleIgnored check SaleIgnored(next).reprocessSlot == false - check SaleIgnored(next).returnBytes test "run switches to errored with other error ": let error = newException(MarketError, "some error") diff --git a/tests/codex/sales/states/testfinished.nim b/tests/codex/sales/states/testfinished.nim index 0c33a7b3..1648df3a 100644 --- a/tests/codex/sales/states/testfinished.nim +++ b/tests/codex/sales/states/testfinished.nim @@ -23,22 +23,23 @@ asyncchecksuite "sales state 'finished'": var market: MockMarket var state: SaleFinished var agent: SalesAgent - var returnBytesWas = bool.none var reprocessSlotWas = bool.none var returnedCollateralValue = UInt256.none + var saleCleared = bool.none setup: market = MockMarket.new() let onCleanUp = proc( - returnBytes = false, reprocessSlot = false, returnedCollateral = UInt256.none + reprocessSlot = false, returnedCollateral = UInt256.none ) {.async.} = - returnBytesWas = some returnBytes reprocessSlotWas = some reprocessSlot returnedCollateralValue = returnedCollateral let context = SalesContext(market: market, clock: clock) agent = newSalesAgent(context, request.id, slotIndex, request.some) agent.onCleanUp = onCleanUp + agent.context.onClear = some proc(request: StorageRequest, idx: uint64) = + saleCleared = some true state = SaleFinished(returnedCollateral: some currentCollateral) test "switches to cancelled state when request expires": @@ -49,8 +50,8 @@ asyncchecksuite "sales state 'finished'": let next = state.onFailed(request) check !next of SaleFailed - test "calls onCleanUp with returnBytes = false, reprocessSlot = true, and returnedCollateral = currentCollateral": + test "calls onCleanUp with reprocessSlot = true, and returnedCollateral = currentCollateral": discard await state.run(agent) - check eventually returnBytesWas == some false check eventually reprocessSlotWas == some false check eventually returnedCollateralValue == some currentCollateral + check eventually saleCleared == some true diff --git a/tests/codex/sales/states/testignored.nim b/tests/codex/sales/states/testignored.nim index 2e1c6e91..5eea7d16 100644 --- a/tests/codex/sales/states/testignored.nim +++ b/tests/codex/sales/states/testignored.nim @@ -20,14 +20,12 @@ asyncchecksuite "sales state 'ignored'": var state: SaleIgnored var agent: SalesAgent - var returnBytesWas = false var reprocessSlotWas = false setup: let onCleanUp = proc( - returnBytes = false, reprocessSlot = false, returnedCollateral = UInt256.none + reprocessSlot = false, returnedCollateral = UInt256.none ) {.async.} = - returnBytesWas = returnBytes reprocessSlotWas = reprocessSlot let context = SalesContext(market: market, clock: clock) @@ -36,7 +34,6 @@ asyncchecksuite "sales state 'ignored'": state = SaleIgnored.new() test "calls onCleanUp with values assigned to SaleIgnored": - state = SaleIgnored(reprocessSlot: true, returnBytes: true) + state = SaleIgnored(reprocessSlot: true) discard await state.run(agent) - check eventually returnBytesWas == true check eventually reprocessSlotWas == true diff --git a/tests/codex/sales/states/testpreparing.nim b/tests/codex/sales/states/testpreparing.nim index 99d9c7fe..802489a1 100644 --- a/tests/codex/sales/states/testpreparing.nim +++ b/tests/codex/sales/states/testpreparing.nim @@ -13,6 +13,7 @@ import pkg/codex/sales/salesagent import pkg/codex/sales/salescontext import pkg/codex/sales/reservations import pkg/codex/stores/repostore +import times import ../../../asynctest import ../../helpers import ../../examples @@ -39,6 +40,8 @@ asyncchecksuite "sales state 'preparing'": duration = request.ask.duration + 60.uint64, minPricePerBytePerSecond = request.ask.pricePerBytePerSecond, totalCollateral = request.ask.collateralPerSlot * request.ask.slots.u256, + enabled = true, + until = 0.SecondsSince1970, ) let repoDs = SQLiteDatastore.new(Memory).tryGet() let metaDs = SQLiteDatastore.new(Memory).tryGet() @@ -52,6 +55,8 @@ asyncchecksuite "sales state 'preparing'": context.reservations = reservations agent = newSalesAgent(context, request.id, slotIndex, request.some) + market.requestEnds[request.id] = clock.now() + cast[int64](request.ask.duration) + teardown: await repo.stop() @@ -67,10 +72,14 @@ asyncchecksuite "sales state 'preparing'": let next = state.onSlotFilled(request.id, slotIndex) check !next of SaleFilled - proc createAvailability() {.async.} = + proc createAvailability(enabled = true) {.async.} = let a = await reservations.createAvailability( - availability.totalSize, availability.duration, - availability.minPricePerBytePerSecond, availability.totalCollateral, + availability.totalSize, + availability.duration, + availability.minPricePerBytePerSecond, + availability.totalCollateral, + enabled, + until = 0.SecondsSince1970, ) availability = a.get @@ -79,7 +88,11 @@ asyncchecksuite "sales state 'preparing'": check next of SaleIgnored let ignored = SaleIgnored(next) check ignored.reprocessSlot - check ignored.returnBytes == false + + test "run switches to ignored when availability is not enabled": + await createAvailability(enabled = false) + let next = !(await state.run(agent)) + check next of SaleIgnored test "run switches to slot reserving state after reservation created": await createAvailability() @@ -94,7 +107,6 @@ asyncchecksuite "sales state 'preparing'": check next of SaleIgnored let ignored = SaleIgnored(next) check ignored.reprocessSlot - check ignored.returnBytes == false test "run switches to errored when reserve fails with other error": await createAvailability() diff --git a/tests/codex/sales/states/testslotreserving.nim b/tests/codex/sales/states/testslotreserving.nim index 0e2e2cc7..b223338a 100644 --- a/tests/codex/sales/states/testslotreserving.nim +++ b/tests/codex/sales/states/testslotreserving.nim @@ -67,4 +67,3 @@ asyncchecksuite "sales state 'SlotReserving'": let next = !(await state.run(agent)) check next of SaleIgnored check SaleIgnored(next).reprocessSlot == false - check SaleIgnored(next).returnBytes diff --git a/tests/codex/sales/testreservations.nim b/tests/codex/sales/testreservations.nim index 49df059d..ff5e153c 100644 --- a/tests/codex/sales/testreservations.nim +++ b/tests/codex/sales/testreservations.nim @@ -1,5 +1,5 @@ import std/random - +import std/times import pkg/questionable import pkg/questionable/results import pkg/chronos @@ -8,6 +8,7 @@ import pkg/datastore import pkg/codex/stores import pkg/codex/errors import pkg/codex/sales +import pkg/codex/clock import pkg/codex/utils/json import ../../asynctest @@ -39,19 +40,22 @@ asyncchecksuite "Reservations module": await repoTmp.destroyDb() await metaTmp.destroyDb() - proc createAvailability(): Availability = + proc createAvailability(enabled = true, until = 0.SecondsSince1970): Availability = let example = Availability.example(collateralPerByte) let totalSize = rand(100000 .. 200000).uint64 let totalCollateral = totalSize.u256 * collateralPerByte let availability = waitFor reservations.createAvailability( - totalSize, example.duration, example.minPricePerBytePerSecond, totalCollateral + totalSize, example.duration, example.minPricePerBytePerSecond, totalCollateral, + enabled, until, ) return availability.get proc createReservation(availability: Availability): Reservation = let size = rand(1 ..< availability.freeSize.int) + let validUntil = getTime().toUnix() + 30.SecondsSince1970 let reservation = waitFor reservations.createReservation( - availability.id, size.uint64, RequestId.example, uint64.example, 1.u256 + availability.id, size.uint64, RequestId.example, uint64.example, 1.u256, + validUntil, ) return reservation.get @@ -64,8 +68,12 @@ asyncchecksuite "Reservations module": check (await reservations.all(Availability)).get.len == 0 test "generates unique ids for storage availability": - let availability1 = Availability.init(1.uint64, 2.uint64, 3.uint64, 4.u256, 5.u256) - let availability2 = Availability.init(1.uint64, 2.uint64, 3.uint64, 4.u256, 5.u256) + let availability1 = Availability.init( + 1.uint64, 2.uint64, 3.uint64, 4.u256, 5.u256, true, 0.SecondsSince1970 + ) + let availability2 = Availability.init( + 1.uint64, 2.uint64, 3.uint64, 4.u256, 5.u256, true, 0.SecondsSince1970 + ) check availability1.id != availability2.id test "can reserve available storage": @@ -128,20 +136,24 @@ asyncchecksuite "Reservations module": test "cannot create reservation with non-existant availability": let availability = Availability.example + let validUntil = getTime().toUnix() + 30.SecondsSince1970 let created = await reservations.createReservation( - availability.id, uint64.example, RequestId.example, uint64.example, 1.u256 + availability.id, uint64.example, RequestId.example, uint64.example, 1.u256, + validUntil, ) check created.isErr check created.error of NotExistsError test "cannot create reservation larger than availability size": let availability = createAvailability() + let validUntil = getTime().toUnix() + 30.SecondsSince1970 let created = await reservations.createReservation( availability.id, availability.totalSize + 1, RequestId.example, uint64.example, UInt256.example, + validUntil, ) check created.isErr check created.error of BytesOutOfBoundsError @@ -149,23 +161,26 @@ asyncchecksuite "Reservations module": test "cannot create reservation larger than availability size - concurrency test": proc concurrencyTest(): Future[void] {.async.} = let availability = createAvailability() + let validUntil = getTime().toUnix() + 30.SecondsSince1970 let one = reservations.createReservation( availability.id, availability.totalSize - 1, RequestId.example, uint64.example, UInt256.example, + validUntil, ) let two = reservations.createReservation( availability.id, availability.totalSize, RequestId.example, uint64.example, - UInt256.example, + UInt256.example, validUntil, ) let oneResult = await one let twoResult = await two check oneResult.isErr or twoResult.isErr + if oneResult.isErr: check oneResult.error of BytesOutOfBoundsError if twoResult.isErr: @@ -259,6 +274,48 @@ asyncchecksuite "Reservations module": check isOk await reservations.update(availability) check (repo.quotaReservedBytes - origQuota) == 100.NBytes + test "create availability set enabled to true by default": + let availability = createAvailability() + check availability.enabled == true + + test "create availability set until to 0 by default": + let availability = createAvailability() + check availability.until == 0.SecondsSince1970 + + test "create availability whith correct values": + var until = getTime().toUnix() + + let availability = createAvailability(enabled = false, until = until) + check availability.enabled == false + check availability.until == until + + test "create an availability fails when trying set until with a negative value": + let totalSize = rand(100000 .. 200000).uint64 + let example = Availability.example(collateralPerByte) + let totalCollateral = totalSize.u256 * collateralPerByte + + let result = await reservations.createAvailability( + totalSize, + example.duration, + example.minPricePerBytePerSecond, + totalCollateral, + enabled = true, + until = -1.SecondsSince1970, + ) + + check result.isErr + check result.error of UntilOutOfBoundsError + + test "update an availability fails when trying set until with a negative value": + let until = getTime().toUnix() + let availability = createAvailability(until = until) + + availability.until = -1 + + let result = await reservations.update(availability) + check result.isErr + check result.error of UntilOutOfBoundsError + test "reservation can be partially released": let availability = createAvailability() let reservation = createReservation(availability) @@ -285,7 +342,9 @@ asyncchecksuite "Reservations module": test "OnAvailabilitySaved called when availability is created": var added: Availability - reservations.OnAvailabilitySaved = proc(a: Availability) {.async.} = + reservations.OnAvailabilitySaved = proc( + a: Availability + ) {.gcsafe, async: (raises: []).} = added = a let availability = createAvailability() @@ -295,7 +354,9 @@ asyncchecksuite "Reservations module": test "OnAvailabilitySaved called when availability size is increased": var availability = createAvailability() var added: Availability - reservations.OnAvailabilitySaved = proc(a: Availability) {.async.} = + reservations.OnAvailabilitySaved = proc( + a: Availability + ) {.gcsafe, async: (raises: []).} = added = a availability.freeSize += 1 discard await reservations.update(availability) @@ -305,7 +366,21 @@ asyncchecksuite "Reservations module": test "OnAvailabilitySaved is not called when availability size is decreased": var availability = createAvailability() var called = false - reservations.OnAvailabilitySaved = proc(a: Availability) {.async.} = + reservations.OnAvailabilitySaved = proc( + a: Availability + ) {.gcsafe, async: (raises: []).} = + called = true + availability.freeSize -= 1.uint64 + discard await reservations.update(availability) + + check not called + + test "OnAvailabilitySaved is not called when availability is disabled": + var availability = createAvailability(enabled = false) + var called = false + reservations.OnAvailabilitySaved = proc( + a: Availability + ) {.gcsafe, async: (raises: []).} = called = true availability.freeSize -= 1 discard await reservations.update(availability) @@ -315,7 +390,7 @@ asyncchecksuite "Reservations module": test "OnAvailabilitySaved called when availability duration is increased": var availability = createAvailability() var added: Availability - reservations.OnAvailabilitySaved = proc(a: Availability) {.async.} = + reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = added = a availability.duration += 1 discard await reservations.update(availability) @@ -325,7 +400,7 @@ asyncchecksuite "Reservations module": test "OnAvailabilitySaved is not called when availability duration is decreased": var availability = createAvailability() var called = false - reservations.OnAvailabilitySaved = proc(a: Availability) {.async.} = + reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = called = true availability.duration -= 1 discard await reservations.update(availability) @@ -335,7 +410,7 @@ asyncchecksuite "Reservations module": test "OnAvailabilitySaved called when availability minPricePerBytePerSecond is increased": var availability = createAvailability() var added: Availability - reservations.OnAvailabilitySaved = proc(a: Availability) {.async.} = + reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = added = a availability.minPricePerBytePerSecond += 1.u256 discard await reservations.update(availability) @@ -345,7 +420,7 @@ asyncchecksuite "Reservations module": test "OnAvailabilitySaved is not called when availability minPricePerBytePerSecond is decreased": var availability = createAvailability() var called = false - reservations.OnAvailabilitySaved = proc(a: Availability) {.async.} = + reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = called = true availability.minPricePerBytePerSecond -= 1.u256 discard await reservations.update(availability) @@ -355,7 +430,7 @@ asyncchecksuite "Reservations module": test "OnAvailabilitySaved called when availability totalCollateral is increased": var availability = createAvailability() var added: Availability - reservations.OnAvailabilitySaved = proc(a: Availability) {.async.} = + reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = added = a availability.totalCollateral = availability.totalCollateral + 1.u256 discard await reservations.update(availability) @@ -365,7 +440,7 @@ asyncchecksuite "Reservations module": test "OnAvailabilitySaved is not called when availability totalCollateral is decreased": var availability = createAvailability() var called = false - reservations.OnAvailabilitySaved = proc(a: Availability) {.async.} = + reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = called = true availability.totalCollateral = availability.totalCollateral - 1.u256 discard await reservations.update(availability) @@ -374,32 +449,69 @@ asyncchecksuite "Reservations module": test "availabilities can be found": let availability = createAvailability() - + let validUntil = getTime().toUnix() + 30.SecondsSince1970 let found = await reservations.findAvailability( availability.freeSize, availability.duration, - availability.minPricePerBytePerSecond, collateralPerByte, + availability.minPricePerBytePerSecond, collateralPerByte, validUntil, ) check found.isSome check found.get == availability + test "does not find an availability when is it disabled": + let availability = createAvailability(enabled = false) + let validUntil = getTime().toUnix() + 30.SecondsSince1970 + let found = await reservations.findAvailability( + availability.freeSize, availability.duration, + availability.minPricePerBytePerSecond, collateralPerByte, validUntil, + ) + + check found.isNone + + test "finds an availability when the until date is after the duration": + let example = Availability.example(collateralPerByte) + let until = getTime().toUnix() + example.duration.SecondsSince1970 + let availability = createAvailability(until = until) + let validUntil = getTime().toUnix() + 30.SecondsSince1970 + let found = await reservations.findAvailability( + availability.freeSize, availability.duration, + availability.minPricePerBytePerSecond, collateralPerByte, validUntil, + ) + + check found.isSome + check found.get == availability + + test "does not find an availability when the until date is before the duration": + let example = Availability.example(collateralPerByte) + let until = getTime().toUnix() + 1.SecondsSince1970 + let availability = createAvailability(until = until) + let validUntil = getTime().toUnix() + 30.SecondsSince1970 + let found = await reservations.findAvailability( + availability.freeSize, availability.duration, + availability.minPricePerBytePerSecond, collateralPerByte, validUntil, + ) + + check found.isNone + test "non-matching availabilities are not found": let availability = createAvailability() - + let validUntil = getTime().toUnix() + 30.SecondsSince1970 let found = await reservations.findAvailability( availability.freeSize + 1, availability.duration, availability.minPricePerBytePerSecond, collateralPerByte, + validUntil, ) check found.isNone test "non-existent availability cannot be found": let availability = Availability.example + let validUntil = getTime().toUnix() + 30.SecondsSince1970 let found = await reservations.findAvailability( availability.freeSize, availability.duration, - availability.minPricePerBytePerSecond, collateralPerByte, + availability.minPricePerBytePerSecond, collateralPerByte, validUntil, ) check found.isNone @@ -420,7 +532,12 @@ asyncchecksuite "Reservations module": test "fails to create availability with size that is larger than available quota": let created = await reservations.createAvailability( - DefaultQuotaBytes.uint64 + 1, uint64.example, UInt256.example, UInt256.example + DefaultQuotaBytes.uint64 + 1, + uint64.example, + UInt256.example, + UInt256.example, + enabled = true, + until = 0.SecondsSince1970, ) check created.isErr check created.error of ReserveFailedError diff --git a/tests/codex/sales/testsales.nim b/tests/codex/sales/testsales.nim index 74ea8a2b..f4d9cbae 100644 --- a/tests/codex/sales/testsales.nim +++ b/tests/codex/sales/testsales.nim @@ -14,6 +14,7 @@ import pkg/codex/stores/repostore import pkg/codex/blocktype as bt import pkg/codex/node import pkg/codex/utils/asyncstatemachine +import times import ../../asynctest import ../helpers import ../helpers/mockmarket @@ -152,6 +153,8 @@ asyncchecksuite "Sales": duration = 60.uint64, minPricePerBytePerSecond = minPricePerBytePerSecond, totalCollateral = totalCollateral, + enabled = true, + until = 0.SecondsSince1970, ) request = StorageRequest( ask: StorageAsk( @@ -221,10 +224,11 @@ asyncchecksuite "Sales": let key = availability.id.key.get (waitFor reservations.get(key, Availability)).get - proc createAvailability() = + proc createAvailability(enabled = true, until = 0.SecondsSince1970) = let a = waitFor reservations.createAvailability( availability.totalSize, availability.duration, - availability.minPricePerBytePerSecond, availability.totalCollateral, + availability.minPricePerBytePerSecond, availability.totalCollateral, enabled, + until, ) availability = a.get # update id @@ -380,14 +384,14 @@ asyncchecksuite "Sales": check eventually getAvailability().freeSize == availability.freeSize - request.ask.slotSize - test "non-downloaded bytes are returned to availability once finished": + test "bytes are returned to availability once finished": var slotIndex = 0.uint64 sales.onStore = proc( request: StorageRequest, slot: uint64, onBatch: BatchProc, isRepairing = false ): Future[?!void] {.async.} = slotIndex = slot let blk = bt.Block.new(@[1.byte]).get - await onBatch(@[blk]) + await onBatch(blk.repeat(request.ask.slotSize)) let sold = newFuture[void]() sales.onSale = proc(request: StorageRequest, slotIndex: uint64) = @@ -403,7 +407,7 @@ asyncchecksuite "Sales": market.slotState[request.slotId(slotIndex)] = SlotState.Finished clock.advance(request.ask.duration.int64) - check eventually getAvailability().freeSize == origSize - 1 + check eventually getAvailability().freeSize == origSize test "ignores download when duration not long enough": availability.duration = request.ask.duration - 1 @@ -439,6 +443,34 @@ asyncchecksuite "Sales": market.slotState[request.slotId(3.uint64)] = SlotState.Filled check wasIgnored() + test "ignores request when availability is not enabled": + createAvailability(enabled = false) + await market.requestStorage(request) + check wasIgnored() + + test "ignores request when availability until terminates before the duration": + let until = getTime().toUnix() + createAvailability(until = until) + await market.requestStorage(request) + + check wasIgnored() + + test "retrieves request when availability until terminates after the duration": + let requestEnd = getTime().toUnix() + cast[int64](request.ask.duration) + let until = requestEnd + 1 + createAvailability(until = until) + + var storingRequest: StorageRequest + sales.onStore = proc( + request: StorageRequest, slot: uint64, onBatch: BatchProc, isRepairing = false + ): Future[?!void] {.async.} = + storingRequest = request + return success() + + market.requestEnds[request.id] = requestEnd + await market.requestStorage(request) + check eventually storingRequest == request + test "retrieves and stores data locally": var storingRequest: StorageRequest var storingSlot: uint64 @@ -563,6 +595,8 @@ asyncchecksuite "Sales": # by other slots request.ask.slots = 1 market.requestExpiry[request.id] = expiry + market.requestEnds[request.id] = + getTime().toUnix() + cast[int64](request.ask.duration) let origSize = availability.freeSize sales.onStore = proc( @@ -621,10 +655,28 @@ asyncchecksuite "Sales": test "deletes inactive reservations on load": createAvailability() + let validUntil = getTime().toUnix() + 30.SecondsSince1970 discard await reservations.createReservation( - availability.id, 100.uint64, RequestId.example, 0.uint64, UInt256.example + availability.id, 100.uint64, RequestId.example, 0.uint64, UInt256.example, + validUntil, ) check (await reservations.all(Reservation)).get.len == 1 await sales.load() check (await reservations.all(Reservation)).get.len == 0 check getAvailability().freeSize == availability.freeSize # was restored + + test "update an availability fails when trying change the until date before an existing reservation": + let until = getTime().toUnix() + 300.SecondsSince1970 + createAvailability(until = until) + + market.requestEnds[request.id] = + getTime().toUnix() + cast[int64](request.ask.duration) + + await market.requestStorage(request) + await allowRequestToStart() + + availability.until = getTime().toUnix() + + let result = await reservations.update(availability) + check result.isErr + check result.error of UntilOutOfBoundsError diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index 5d5f0cc2..d7ed3df2 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -294,6 +294,8 @@ proc postAvailabilityRaw*( client: CodexClient, totalSize, duration: uint64, minPricePerBytePerSecond, totalCollateral: UInt256, + enabled: ?bool = bool.none, + until: ?SecondsSince1970 = SecondsSince1970.none, ): Future[HttpClientResponseRef] {.async: (raises: [CancelledError, HttpError]).} = ## Post sales availability endpoint ## @@ -304,18 +306,27 @@ proc postAvailabilityRaw*( "duration": duration, "minPricePerBytePerSecond": minPricePerBytePerSecond, "totalCollateral": totalCollateral, + "enabled": enabled, + "until": until, } - return await client.post(url, $json) proc postAvailability*( client: CodexClient, totalSize, duration: uint64, minPricePerBytePerSecond, totalCollateral: UInt256, + enabled: ?bool = bool.none, + until: ?SecondsSince1970 = SecondsSince1970.none, ): Future[?!Availability] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.postAvailabilityRaw( - totalSize, duration, minPricePerBytePerSecond, totalCollateral + totalSize = totalSize, + duration = duration, + minPricePerBytePerSecond = minPricePerBytePerSecond, + totalCollateral = totalCollateral, + enabled = enabled, + until = until, ) + let body = await response.body doAssert response.status == 201, @@ -327,6 +338,8 @@ proc patchAvailabilityRaw*( availabilityId: AvailabilityId, totalSize, freeSize, duration: ?uint64 = uint64.none, minPricePerBytePerSecond, totalCollateral: ?UInt256 = UInt256.none, + enabled: ?bool = bool.none, + until: ?SecondsSince1970 = SecondsSince1970.none, ): Future[HttpClientResponseRef] {. async: (raw: true, raises: [CancelledError, HttpError]) .} = @@ -352,6 +365,12 @@ proc patchAvailabilityRaw*( if totalCollateral =? totalCollateral: json["totalCollateral"] = %totalCollateral + if enabled =? enabled: + json["enabled"] = %enabled + + if until =? until: + json["until"] = %until + client.patch(url, $json) proc patchAvailability*( @@ -359,6 +378,8 @@ proc patchAvailability*( availabilityId: AvailabilityId, totalSize, duration: ?uint64 = uint64.none, minPricePerBytePerSecond, totalCollateral: ?UInt256 = UInt256.none, + enabled: ?bool = bool.none, + until: ?SecondsSince1970 = SecondsSince1970.none, ): Future[void] {.async: (raises: [CancelledError, HttpError]).} = let response = await client.patchAvailabilityRaw( availabilityId, @@ -366,8 +387,10 @@ proc patchAvailability*( duration = duration, minPricePerBytePerSecond = minPricePerBytePerSecond, totalCollateral = totalCollateral, + enabled = enabled, + until = until, ) - doAssert response.status == 200, "expected 200 OK, got " & $response.status + doAssert response.status == 204, "expected No Content, got " & $response.status proc getAvailabilities*( client: CodexClient diff --git a/tests/integration/testmarketplace.nim b/tests/integration/testmarketplace.nim index dee3645e..40f394e0 100644 --- a/tests/integration/testmarketplace.nim +++ b/tests/integration/testmarketplace.nim @@ -1,3 +1,5 @@ +import std/times +import std/httpclient import ../examples import ../contracts/time import ../contracts/deployment diff --git a/tests/integration/testproofs.nim b/tests/integration/testproofs.nim index b0ede765..c49b7b6f 100644 --- a/tests/integration/testproofs.nim +++ b/tests/integration/testproofs.nim @@ -275,7 +275,9 @@ marketplacesuite "Simulate invalid proofs": # totalSize=slotSize, # should match 1 slot only # duration=totalPeriods.periods.u256, # minPricePerBytePerSecond=minPricePerBytePerSecond, - # totalCollateral=slotSize * minPricePerBytePerSecond + # totalCollateral=slotSize * minPricePerBytePerSecond, + # enabled = true.some, + # until = 0.SecondsSince1970.some, # ) # let cid = client0.upload(data).get diff --git a/tests/integration/testrestapi.nim b/tests/integration/testrestapi.nim index 415658c1..57e38b39 100644 --- a/tests/integration/testrestapi.nim +++ b/tests/integration/testrestapi.nim @@ -35,6 +35,7 @@ twonodessuite "REST API": duration = 2.uint64, minPricePerBytePerSecond = minPricePerBytePerSecond, totalCollateral = totalCollateral, + enabled = true.some, ) ).get let space = (await client1.space()).tryGet() diff --git a/tests/integration/testrestapivalidation.nim b/tests/integration/testrestapivalidation.nim index 00caefdd..adeffa77 100644 --- a/tests/integration/testrestapivalidation.nim +++ b/tests/integration/testrestapivalidation.nim @@ -364,5 +364,21 @@ asyncchecksuite "Rest API validation": check responseBefore.status == 422 check (await responseBefore.body) == "Collateral per byte must be greater than zero" + test "creating availability fails when until is negative": + let totalSize = 12.uint64 + let minPricePerBytePerSecond = 1.u256 + let totalCollateral = totalSize.u256 * minPricePerBytePerSecond + let response = await client.postAvailabilityRaw( + totalSize = totalSize, + duration = 2.uint64, + minPricePerBytePerSecond = minPricePerBytePerSecond, + totalCollateral = totalCollateral, + until = -1.SecondsSince1970.some, + ) + + check: + response.status == 422 + (await response.body) == "Cannot set until to a negative value" + waitFor node.stop() node.removeDataDir() diff --git a/tests/integration/testsales.nim b/tests/integration/testsales.nim index 5e9b26df..ef999990 100644 --- a/tests/integration/testsales.nim +++ b/tests/integration/testsales.nim @@ -1,4 +1,5 @@ import std/httpclient +import std/times import pkg/codex/contracts from pkg/codex/stores/repostore/types import DefaultQuotaBytes import ./twonodes @@ -17,22 +18,14 @@ proc findItem[T](items: seq[T], item: T): ?!T = multinodesuite "Sales": let salesConfig = NodeConfigs( - clients: CodexConfigs - .init(nodes = 1) - .withLogFile() - .withLogTopics( - "node", "marketplace", "sales", "reservations", "node", "proving", "clock" - ).some, - providers: CodexConfigs - .init(nodes = 1) - .withLogFile() - .withLogTopics( - "node", "marketplace", "sales", "reservations", "node", "proving", "clock" - ).some, + clients: CodexConfigs.init(nodes = 1).some, + providers: CodexConfigs.init(nodes = 1) + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("node", "marketplace", "sales", "reservations", "node", "proving", "clock") + .some, ) - let minPricePerBytePerSecond = 1.u256 - var host: CodexClient var client: CodexClient @@ -80,11 +73,15 @@ multinodesuite "Sales": ) ).get + var until = getTime().toUnix() + await host.patchAvailability( availability.id, duration = 100.uint64.some, minPricePerBytePerSecond = 2.u256.some, totalCollateral = 200.u256.some, + enabled = false.some, + until = until.some, ) let updatedAvailability = @@ -94,6 +91,8 @@ multinodesuite "Sales": check updatedAvailability.totalCollateral == 200 check updatedAvailability.totalSize == 140000.uint64 check updatedAvailability.freeSize == 140000.uint64 + check updatedAvailability.enabled == false + check updatedAvailability.until == until test "updating availability - updating totalSize", salesConfig: let availability = ( @@ -105,6 +104,7 @@ multinodesuite "Sales": ) ).get await host.patchAvailability(availability.id, totalSize = 100000.uint64.some) + let updatedAvailability = ((await host.getAvailabilities()).get).findItem(availability).get check updatedAvailability.totalSize == 100000 @@ -165,3 +165,72 @@ multinodesuite "Sales": ((await host.getAvailabilities()).get).findItem(availability).get check newUpdatedAvailability.totalSize == originalSize + 20000 check newUpdatedAvailability.freeSize - updatedAvailability.freeSize == 20000 + + test "updating availability fails with until negative", salesConfig: + let availability = ( + await host.postAvailability( + totalSize = 140000.uint64, + duration = 200.uint64, + minPricePerBytePerSecond = 3.u256, + totalCollateral = 300.u256, + ) + ).get + + let response = + await host.patchAvailabilityRaw(availability.id, until = -1.SecondsSince1970.some) + + check: + (await response.body) == "Cannot set until to a negative value" + + test "returns an error when trying to update the until date before an existing a request is finished", + salesConfig: + let size = 0xFFFFFF.uint64 + let data = await RandomChunker.example(blocks = 8) + let duration = 20 * 60.uint64 + let minPricePerBytePerSecond = 3.u256 + let collateralPerByte = 1.u256 + let ecNodes = 3.uint + let ecTolerance = 1.uint + + # host makes storage available + let availability = ( + await host.postAvailability( + totalSize = size, + duration = duration, + minPricePerBytePerSecond = minPricePerBytePerSecond, + totalCollateral = size.u256 * minPricePerBytePerSecond, + ) + ).get + + # client requests storage + let cid = (await client.upload(data)).get + let id = ( + await client.requestStorage( + cid, + duration = duration, + pricePerBytePerSecond = minPricePerBytePerSecond, + proofProbability = 3.u256, + expiry = 10 * 60.uint64, + collateralPerByte = collateralPerByte, + nodes = ecNodes, + tolerance = ecTolerance, + ) + ).get + + check eventually( + await client.purchaseStateIs(id, "started"), timeout = 10 * 60 * 1000 + ) + let purchase = (await client.getPurchase(id)).get + check purchase.error == none string + + let unixNow = getTime().toUnix() + let until = unixNow + 1.SecondsSince1970 + + let response = await host.patchAvailabilityRaw( + availabilityId = availability.id, until = until.some + ) + + check: + response.status == 422 + (await response.body) == + "Until parameter must be greater or equal to the longest currently hosted slot" diff --git a/vendor/nim-datastore b/vendor/nim-datastore index d67860ad..5778e373 160000 --- a/vendor/nim-datastore +++ b/vendor/nim-datastore @@ -1 +1 @@ -Subproject commit d67860add63fd23cdacde1d3da8f4739c2660c2d +Subproject commit 5778e373fa97286f389e0aef61f1e8f30a934dab From 0032e60398f800527e30315ed6c39985cc79e11c Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 26 Mar 2025 16:17:39 +0100 Subject: [PATCH 09/24] fix(marketplace): catch Marketplace_SlotIsFree and continue the cancelled process (#1139) * Catch Marketplace_SlotIsFree and continue the cancelled process * Add log message when the slot if free during failed state * Reduce log level to debug for slot free error * Separate slot mock errors * Initialize variable in setyp * Improve tests * Remove non-meaningful checks and rename test * Remove the Option in the error setters * Return collateral when the state is cancelled only if the slot is filled by the host * Do not propagate AsyncLockError * Wrap contract error into specific error type * Remove debug message * Catch only SlotStateMismatchError in cancelled * Fix error * Remove returnBytesWas * Use MarketError after raises pragma were defined * Fix typo * Fix lint --- codex/contracts/market.nim | 39 ++++++------ codex/market.nim | 4 +- codex/sales/states/cancelled.nim | 32 +++++++--- codex/sales/states/failed.nim | 1 + tests/codex/helpers/mockmarket.nim | 27 ++++++++- tests/codex/sales/states/testcancelled.nim | 69 ++++++++++++++++++++-- 6 files changed, 139 insertions(+), 33 deletions(-) diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 74694285..8b235876 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -221,7 +221,7 @@ method requestExpiresAt*( method getHost( market: OnChainMarket, requestId: RequestId, slotIndex: uint64 -): Future[?Address] {.async.} = +): Future[?Address] {.async: (raises: [CancelledError, MarketError]).} = convertEthersError("Failed to get slot's host"): let slotId = slotId(requestId, slotIndex) let address = await market.contract.getHost(slotId) @@ -232,7 +232,7 @@ method getHost( method currentCollateral*( market: OnChainMarket, slotId: SlotId -): Future[UInt256] {.async.} = +): Future[UInt256] {.async: (raises: [MarketError, CancelledError]).} = convertEthersError("Failed to get slot's current collateral"): return await market.contract.currentCollateral(slotId) @@ -270,22 +270,27 @@ method freeSlot*( market: OnChainMarket, slotId: SlotId ) {.async: (raises: [CancelledError, MarketError]).} = convertEthersError("Failed to free slot"): - var freeSlot: Future[Confirmable] - if rewardRecipient =? market.rewardRecipient: - # If --reward-recipient specified, use it as the reward recipient, and use - # the SP's address as the collateral recipient - let collateralRecipient = await market.getSigner() - freeSlot = market.contract.freeSlot( - slotId, - rewardRecipient, # --reward-recipient - collateralRecipient, - ) # SP's address - else: - # Otherwise, use the SP's address as both the reward and collateral - # recipient (the contract will use msg.sender for both) - freeSlot = market.contract.freeSlot(slotId) + try: + var freeSlot: Future[Confirmable] + if rewardRecipient =? market.rewardRecipient: + # If --reward-recipient specified, use it as the reward recipient, and use + # the SP's address as the collateral recipient + let collateralRecipient = await market.getSigner() + freeSlot = market.contract.freeSlot( + slotId, + rewardRecipient, # --reward-recipient + collateralRecipient, + ) # SP's address + else: + # Otherwise, use the SP's address as both the reward and collateral + # recipient (the contract will use msg.sender for both) + freeSlot = market.contract.freeSlot(slotId) - discard await freeSlot.confirm(1) + discard await freeSlot.confirm(1) + except Marketplace_SlotIsFree as parent: + raise newException( + SlotStateMismatchError, "Failed to free slot, slot is already free", parent + ) method withdrawFunds( market: OnChainMarket, requestId: RequestId diff --git a/codex/market.nim b/codex/market.nim index 71cad9a9..31c0687f 100644 --- a/codex/market.nim +++ b/codex/market.nim @@ -148,12 +148,12 @@ method requestExpiresAt*( method getHost*( market: Market, requestId: RequestId, slotIndex: uint64 -): Future[?Address] {.base, async.} = +): Future[?Address] {.base, async: (raises: [CancelledError, MarketError]).} = raiseAssert("not implemented") method currentCollateral*( market: Market, slotId: SlotId -): Future[UInt256] {.base, async.} = +): Future[UInt256] {.base, async: (raises: [MarketError, CancelledError]).} = raiseAssert("not implemented") method getActiveSlot*(market: Market, slotId: SlotId): Future[?Slot] {.base, async.} = diff --git a/codex/sales/states/cancelled.nim b/codex/sales/states/cancelled.nim index 2c240e15..f3c755a3 100644 --- a/codex/sales/states/cancelled.nim +++ b/codex/sales/states/cancelled.nim @@ -12,6 +12,14 @@ type SaleCancelled* = ref object of SaleState method `$`*(state: SaleCancelled): string = "SaleCancelled" +proc slotIsFilledByMe( + market: Market, requestId: RequestId, slotIndex: uint64 +): Future[bool] {.async: (raises: [CancelledError, MarketError]).} = + let host = await market.getHost(requestId, slotIndex) + let me = await market.getSigner() + + return host == me.some + method run*( state: SaleCancelled, machine: Machine ): Future[?State] {.async: (raises: []).} = @@ -23,19 +31,27 @@ method run*( raiseAssert "no sale request" try: - let slot = Slot(request: request, slotIndex: data.slotIndex) - debug "Collecting collateral and partial payout", - requestId = data.requestId, slotIndex = data.slotIndex - let currentCollateral = await market.currentCollateral(slot.id) - await market.freeSlot(slot.id) + var returnedCollateral = UInt256.none + + if await slotIsFilledByMe(market, data.requestId, data.slotIndex): + debug "Collecting collateral and partial payout", + requestId = data.requestId, slotIndex = data.slotIndex + + let slot = Slot(request: request, slotIndex: data.slotIndex) + let currentCollateral = await market.currentCollateral(slot.id) + + try: + await market.freeSlot(slot.id) + except SlotStateMismatchError as e: + warn "Failed to free slot because slot is already free", error = e.msg + + returnedCollateral = currentCollateral.some if onClear =? agent.context.onClear and request =? data.request: onClear(request, data.slotIndex) if onCleanUp =? agent.onCleanUp: - await onCleanUp( - reprocessSlot = false, returnedCollateral = some currentCollateral - ) + await onCleanUp(reprocessSlot = false, returnedCollateral = returnedCollateral) warn "Sale cancelled due to timeout", requestId = data.requestId, slotIndex = data.slotIndex diff --git a/codex/sales/states/failed.nim b/codex/sales/states/failed.nim index b0d6a7cd..f1490d20 100644 --- a/codex/sales/states/failed.nim +++ b/codex/sales/states/failed.nim @@ -28,6 +28,7 @@ method run*( let slot = Slot(request: request, slotIndex: data.slotIndex) debug "Removing slot from mySlots", requestId = data.requestId, slotIndex = data.slotIndex + await market.freeSlot(slot.id) let error = newException(SaleFailedError, "Sale failed") diff --git a/tests/codex/helpers/mockmarket.nim b/tests/codex/helpers/mockmarket.nim index 03e76762..55abeb14 100644 --- a/tests/codex/helpers/mockmarket.nim +++ b/tests/codex/helpers/mockmarket.nim @@ -8,6 +8,7 @@ import pkg/codex/market import pkg/codex/contracts/requests import pkg/codex/contracts/proofs import pkg/codex/contracts/config +import pkg/questionable/results from pkg/ethers import BlockTag import codex/clock @@ -48,6 +49,8 @@ type canReserveSlot*: bool errorOnReserveSlot*: ?(ref MarketError) errorOnFillSlot*: ?(ref MarketError) + errorOnFreeSlot*: ?(ref MarketError) + errorOnGetHost*: ?(ref MarketError) clock: ?Clock Fulfillment* = object @@ -232,7 +235,10 @@ method requestExpiresAt*( method getHost*( market: MockMarket, requestId: RequestId, slotIndex: uint64 -): Future[?Address] {.async.} = +): Future[?Address] {.async: (raises: [CancelledError, MarketError]).} = + if error =? market.errorOnGetHost: + raise error + for slot in market.filled: if slot.requestId == requestId and slot.slotIndex == slotIndex: return some slot.host @@ -240,7 +246,7 @@ method getHost*( method currentCollateral*( market: MockMarket, slotId: SlotId -): Future[UInt256] {.async.} = +): Future[UInt256] {.async: (raises: [MarketError, CancelledError]).} = for slot in market.filled: if slotId == slotId(slot.requestId, slot.slotIndex): return slot.collateral @@ -321,6 +327,9 @@ method fillSlot*( method freeSlot*( market: MockMarket, slotId: SlotId ) {.async: (raises: [CancelledError, MarketError]).} = + if error =? market.errorOnFreeSlot: + raise error + market.freed.add(slotId) for s in market.filled: if slotId(s.requestId, s.slotIndex) == slotId: @@ -411,6 +420,20 @@ func setErrorOnFillSlot*(market: MockMarket, error: ref MarketError) = else: some error +func setErrorOnFreeSlot*(market: MockMarket, error: ref MarketError) = + market.errorOnFreeSlot = + if error.isNil: + none (ref MarketError) + else: + some error + +func setErrorOnGetHost*(market: MockMarket, error: ref MarketError) = + market.errorOnGetHost = + if error.isNil: + none (ref MarketError) + else: + some error + method subscribeRequests*( market: MockMarket, callback: OnRequest ): Future[Subscription] {.async.} = diff --git a/tests/codex/sales/states/testcancelled.nim b/tests/codex/sales/states/testcancelled.nim index ab450200..6eaf1f5a 100644 --- a/tests/codex/sales/states/testcancelled.nim +++ b/tests/codex/sales/states/testcancelled.nim @@ -2,9 +2,11 @@ import pkg/questionable import pkg/chronos import pkg/codex/contracts/requests import pkg/codex/sales/states/cancelled +import pkg/codex/sales/states/errored import pkg/codex/sales/salesagent import pkg/codex/sales/salescontext import pkg/codex/market +from pkg/codex/utils/asyncstatemachine import State import ../../../asynctest import ../../examples @@ -22,8 +24,8 @@ asyncchecksuite "sales state 'cancelled'": var market: MockMarket var state: SaleCancelled var agent: SalesAgent - var reprocessSlotWas = bool.none - var returnedCollateralValue = UInt256.none + var reprocessSlotWas: ?bool + var returnedCollateralValue: ?UInt256 setup: market = MockMarket.new() @@ -37,8 +39,43 @@ asyncchecksuite "sales state 'cancelled'": agent = newSalesAgent(context, request.id, slotIndex, request.some) agent.onCleanUp = onCleanUp state = SaleCancelled.new() + reprocessSlotWas = bool.none + returnedCollateralValue = UInt256.none + teardown: + reprocessSlotWas = bool.none + returnedCollateralValue = UInt256.none test "calls onCleanUp with reprocessSlot = true, and returnedCollateral = currentCollateral": + market.fillSlot( + requestId = request.id, + slotIndex = slotIndex, + proof = Groth16Proof.default, + host = await market.getSigner(), + collateral = currentCollateral, + ) + discard await state.run(agent) + check eventually reprocessSlotWas == some false + check eventually returnedCollateralValue == some currentCollateral + + test "completes the cancelled state when free slot error is raised and the collateral is returned when a host is hosting a slot": + market.fillSlot( + requestId = request.id, + slotIndex = slotIndex, + proof = Groth16Proof.default, + host = await market.getSigner(), + collateral = currentCollateral, + ) + + let error = + newException(SlotStateMismatchError, "Failed to free slot, slot is already free") + market.setErrorOnFreeSlot(error) + + let next = await state.run(agent) + check next == none State + check eventually reprocessSlotWas == some false + check eventually returnedCollateralValue == some currentCollateral + + test "completes the cancelled state when free slot error is raised and the collateral is not returned when a host is not hosting a slot": market.fillSlot( requestId = request.id, slotIndex = slotIndex, @@ -46,6 +83,30 @@ asyncchecksuite "sales state 'cancelled'": host = Address.example, collateral = currentCollateral, ) - discard await state.run(agent) + + let error = + newException(SlotStateMismatchError, "Failed to free slot, slot is already free") + market.setErrorOnFreeSlot(error) + + let next = await state.run(agent) + check next == none State check eventually reprocessSlotWas == some false - check eventually returnedCollateralValue == some currentCollateral + check eventually returnedCollateralValue == UInt256.none + + test "calls onCleanUp and returns the collateral when an error is raised": + market.fillSlot( + requestId = request.id, + slotIndex = slotIndex, + proof = Groth16Proof.default, + host = Address.example, + collateral = currentCollateral, + ) + + let error = newException(MarketError, "") + market.setErrorOnGetHost(error) + + let next = !(await state.run(agent)) + + check next of SaleErrored + let errored = SaleErrored(next) + check errored.error == error From 0ec52abc981772f547f58083e191594aa0f078a7 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Mon, 31 Mar 2025 06:48:22 +0200 Subject: [PATCH 10/24] fixes RandomChunker not respecting padding (#1170) --- tests/codex/helpers/randomchunker.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/codex/helpers/randomchunker.nim b/tests/codex/helpers/randomchunker.nim index cf857595..d1383e84 100644 --- a/tests/codex/helpers/randomchunker.nim +++ b/tests/codex/helpers/randomchunker.nim @@ -33,10 +33,10 @@ proc new*( return 0 var read = 0 - while read < len: + while read < len and (pad or read < size - consumed): rng.shuffle(alpha) for a in alpha: - if read >= len: + if read >= len or (not pad and read >= size - consumed): break data[read] = a From 5ec3b2b0275046cbf4a4743b18714e49e54adbfc Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Mon, 31 Mar 2025 06:57:55 +0200 Subject: [PATCH 11/24] make sure we do not call "get" on unverified Result while fetching in batches (#1169) * makes sure we do not call "get" on unverified result * make handling of failed blocks in fetchBatched even more explicit * simplifies allFinishedValues and makes it independent from allFinishedFailed * only sleep if not iter.finished in fetchBatched --- codex/errors.nim | 29 ++++++++++++++++++++++++++++- codex/node.nim | 22 ++++++++++++++-------- tests/codex/node/testnode.nim | 26 ++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/codex/errors.nim b/codex/errors.nim index fadf7299..1a571e0f 100644 --- a/codex/errors.nim +++ b/codex/errors.nim @@ -8,6 +8,8 @@ ## those terms. import std/options +import std/sugar +import std/sequtils import pkg/results import pkg/chronos @@ -42,7 +44,9 @@ func toFailure*[T](exp: Option[T]): Result[T, ref CatchableError] {.inline.} = else: T.failure("Option is None") -proc allFinishedFailed*[T](futs: seq[Future[T]]): Future[FinishedFailed[T]] {.async.} = +proc allFinishedFailed*[T]( + futs: seq[Future[T]] +): Future[FinishedFailed[T]] {.async: (raises: [CancelledError]).} = ## Check if all futures have finished or failed ## ## TODO: wip, not sure if we want this - at the minimum, @@ -57,3 +61,26 @@ proc allFinishedFailed*[T](futs: seq[Future[T]]): Future[FinishedFailed[T]] {.as res.success.add f return res + +proc allFinishedValues*[T]( + futs: seq[Future[T]] +): Future[?!seq[T]] {.async: (raises: [CancelledError]).} = + ## If all futures have finished, return corresponding values, + ## otherwise return failure + ## + + # wait for all futures to be either completed, failed or canceled + await allFutures(futs) + + let numOfFailed = futs.countIt(it.failed) + + if numOfFailed > 0: + return failure "Some futures failed (" & $numOfFailed & "))" + + # here, we know there are no failed futures in "futs" + # and we are only interested in those that completed successfully + let values = collect: + for b in futs: + if b.finished: + b.value + return success values diff --git a/codex/node.nim b/codex/node.nim index 9932deb6..fb653c0d 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -183,23 +183,29 @@ proc fetchBatched*( # ) while not iter.finished: - let blocks = collect: + let blockFutures = collect: for i in 0 ..< batchSize: if not iter.finished: let address = BlockAddress.init(cid, iter.next()) if not (await address in self.networkStore) or fetchLocal: self.networkStore.getBlock(address) - let res = await allFinishedFailed(blocks) - if res.failure.len > 0: - trace "Some blocks failed to fetch", len = res.failure.len - return failure("Some blocks failed to fetch (" & $res.failure.len & " )") + without blockResults =? await allFinishedValues(blockFutures), err: + trace "Some blocks failed to fetch", err = err.msg + return failure(err) - if not onBatch.isNil and - batchErr =? (await onBatch(blocks.mapIt(it.read.get))).errorOption: + let blocks = blockResults.filterIt(it.isSuccess()).mapIt(it.value) + + let numOfFailedBlocks = blockResults.len - blocks.len + if numOfFailedBlocks > 0: + return + failure("Some blocks failed (Result) to fetch (" & $numOfFailedBlocks & ")") + + if not onBatch.isNil and batchErr =? (await onBatch(blocks)).errorOption: return failure(batchErr) - await sleepAsync(1.millis) + if not iter.finished: + await sleepAsync(1.millis) success() diff --git a/tests/codex/node/testnode.nim b/tests/codex/node/testnode.nim index 511badef..bd535336 100644 --- a/tests/codex/node/testnode.nim +++ b/tests/codex/node/testnode.nim @@ -30,6 +30,7 @@ import pkg/codex/discovery import pkg/codex/erasure import pkg/codex/merkletree import pkg/codex/blocktype as bt +import pkg/codex/rng import pkg/codex/node {.all.} @@ -78,6 +79,31 @@ asyncchecksuite "Test Node - Basic": ) ).tryGet() + test "Block Batching with corrupted blocks": + let blocks = await makeRandomBlocks(datasetSize = 64.KiBs.int, blockSize = 64.KiBs) + assert blocks.len == 1 + + let blk = blocks[0] + + # corrupt block + let pos = rng.Rng.instance.rand(blk.data.len - 1) + blk.data[pos] = byte 0 + + let manifest = await storeDataGetManifest(localStore, blocks) + + let batchSize = manifest.blocksCount + let res = ( + await node.fetchBatched( + manifest, + batchSize = batchSize, + proc(blocks: seq[bt.Block]): Future[?!void] {.gcsafe, async.} = + return failure("Should not be called"), + ) + ) + check res.isFailure + check res.error of CatchableError + check res.error.msg == "Some blocks failed (Result) to fetch (1)" + test "Should store Data Stream": let stream = BufferStream.new() From e9c6d198732874203755c473363b624562725df8 Mon Sep 17 00:00:00 2001 From: munna0908 <88337208+munna0908@users.noreply.github.com> Date: Mon, 31 Mar 2025 12:11:08 +0530 Subject: [PATCH 12/24] use constantine sha256 for codex tree hashing (#1168) --- codex/merkletree/codex/codex.nim | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/codex/merkletree/codex/codex.nim b/codex/merkletree/codex/codex.nim index e287dfac..0eec92e4 100644 --- a/codex/merkletree/codex/codex.nim +++ b/codex/merkletree/codex/codex.nim @@ -15,7 +15,7 @@ import std/sequtils import pkg/questionable import pkg/questionable/results import pkg/libp2p/[cid, multicodec, multihash] - +import pkg/constantine/hashes import ../../utils import ../../rng import ../../errors @@ -132,9 +132,13 @@ func compress*(x, y: openArray[byte], key: ByteTreeKey, mhash: MHash): ?!ByteHas ## Compress two hashes ## - var digest = newSeq[byte](mhash.size) - mhash.coder(@x & @y & @[key.byte], digest) - success digest + # Using Constantine's SHA256 instead of mhash for optimal performance on 32-byte merkle node hashing + # See: https://github.com/codex-storage/nim-codex/issues/1162 + + let input = @x & @y & @[key.byte] + var digest = hashes.sha256.hash(input) + + success @digest func init*( _: type CodexTree, mcodec: MultiCodec = Sha256HashCodec, leaves: openArray[ByteHash] From 1213377ac40d8c5634052a507332a09bd8707742 Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Wed, 2 Apr 2025 12:09:43 +0300 Subject: [PATCH 13/24] ci: switch out from ubuntu 20.04 (#1184) * ci: use ubuntu-latest for coverage (#1141) * ci: pass --keep-going to lcov and genhtml (#1141) * ci: use ubuntu-22.04 for release workflow (#1141) * ci: install gcc-14 on linux (#1141) * chore: bump nim-leveldbstatic to 0.2.1 --- .github/actions/nimbus-build-system/action.yml | 14 ++++++++++---- .github/workflows/ci.yml | 6 +----- .github/workflows/release.yml | 2 +- Makefile | 6 +++--- build.nims | 6 +++--- vendor/nim-leveldbstatic | 2 +- 6 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/actions/nimbus-build-system/action.yml b/.github/actions/nimbus-build-system/action.yml index 5d1917e3..2128bba8 100644 --- a/.github/actions/nimbus-build-system/action.yml +++ b/.github/actions/nimbus-build-system/action.yml @@ -92,10 +92,16 @@ runs: if : ${{ inputs.os == 'linux' && inputs.coverage != 'true' }} shell: ${{ inputs.shell }} {0} run: | - # Add GCC-14 to alternatives - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 14 - # Set GCC-14 as the default - sudo update-alternatives --set gcc /usr/bin/gcc-14 + # Skip for older Ubuntu versions + if [[ $(lsb_release -r | awk -F '[^0-9]+' '{print $2}') -ge 24 ]]; then + # Install GCC-14 + sudo apt-get update -qq + sudo apt-get install -yq gcc-14 + # Add GCC-14 to alternatives + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 14 + # Set GCC-14 as the default + sudo update-alternatives --set gcc /usr/bin/gcc-14 + fi - name: Install ccache on Linux/Mac if: inputs.os == 'linux' || inputs.os == 'macos' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d660a029..6917e16b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,11 +61,7 @@ jobs: suggest: true coverage: - # Force to stick to ubuntu 20.04 for coverage because - # lcov was updated to 2.x version in ubuntu-latest - # and cause a lot of issues. - # See https://github.com/linux-test-project/lcov/issues/238 - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout sources uses: actions/checkout@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 19170528..9d433257 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,7 @@ jobs: uses: fabiocaccamo/create-matrix-action@v5 with: matrix: | - os {linux}, cpu {amd64}, builder {ubuntu-20.04}, nim_version {${{ env.nim_version }}}, rust_version {${{ env.rust_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-22.04}, nim_version {${{ env.nim_version }}}, rust_version {${{ env.rust_version }}}, shell {bash --noprofile --norc -e -o pipefail} os {linux}, cpu {arm64}, builder {ubuntu-22.04-arm}, nim_version {${{ env.nim_version }}}, rust_version {${{ env.rust_version }}}, shell {bash --noprofile --norc -e -o pipefail} os {macos}, cpu {amd64}, builder {macos-13}, nim_version {${{ env.nim_version }}}, rust_version {${{ env.rust_version }}}, shell {bash --noprofile --norc -e -o pipefail} os {macos}, cpu {arm64}, builder {macos-14}, nim_version {${{ env.nim_version }}}, rust_version {${{ env.rust_version }}}, shell {bash --noprofile --norc -e -o pipefail} diff --git a/Makefile b/Makefile index 29d6c11d..c9f1f10e 100644 --- a/Makefile +++ b/Makefile @@ -179,11 +179,11 @@ coverage: $(MAKE) NIMFLAGS="$(NIMFLAGS) --lineDir:on --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage" test cd nimcache/release/testCodex && rm -f *.c mkdir -p coverage - lcov --capture --directory nimcache/release/testCodex --output-file coverage/coverage.info + lcov --capture --keep-going --directory nimcache/release/testCodex --output-file coverage/coverage.info shopt -s globstar && ls $$(pwd)/codex/{*,**/*}.nim - shopt -s globstar && lcov --extract coverage/coverage.info $$(pwd)/codex/{*,**/*}.nim --output-file coverage/coverage.f.info + shopt -s globstar && lcov --extract coverage/coverage.info --keep-going $$(pwd)/codex/{*,**/*}.nim --output-file coverage/coverage.f.info echo -e $(BUILD_MSG) "coverage/report/index.html" - genhtml coverage/coverage.f.info --output-directory coverage/report + genhtml coverage/coverage.f.info --keep-going --output-directory coverage/report show-coverage: if which open >/dev/null; then (echo -e "\e[92mOpening\e[39m HTML coverage report in browser..." && open coverage/report/index.html) || true; fi diff --git a/build.nims b/build.nims index baf21e03..88660321 100644 --- a/build.nims +++ b/build.nims @@ -107,14 +107,14 @@ task coverage, "generates code coverage report": mkDir("coverage") echo " ======== Running LCOV ======== " exec( - "lcov --capture --directory nimcache/coverage --output-file coverage/coverage.info" + "lcov --capture --keep-going --directory nimcache/coverage --output-file coverage/coverage.info" ) exec( - "lcov --extract coverage/coverage.info --output-file coverage/coverage.f.info " & + "lcov --extract coverage/coverage.info --keep-going --output-file coverage/coverage.f.info " & nimSrcs ) echo " ======== Generating HTML coverage report ======== " - exec("genhtml coverage/coverage.f.info --output-directory coverage/report ") + exec("genhtml coverage/coverage.f.info --keep-going --output-directory coverage/report ") echo " ======== Coverage report Done ======== " task showCoverage, "open coverage html": diff --git a/vendor/nim-leveldbstatic b/vendor/nim-leveldbstatic index 4da61d23..378ef63e 160000 --- a/vendor/nim-leveldbstatic +++ b/vendor/nim-leveldbstatic @@ -1 +1 @@ -Subproject commit 4da61d231a5e73c5daf85eb23f146242b90b144f +Subproject commit 378ef63e261e3b5834a3567404edc3ce838498b3 From 4e2a321ad5ad1e299dbba8ab08e78665714a84d9 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Wed, 2 Apr 2025 16:09:23 +0200 Subject: [PATCH 14/24] chore(openapi): add required parameters (#1178) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update the openapi file * Fix typo * Remove SalesAvailabilityCREATE and add collateralPerByte * Fix SalesAvailability reference * chore: adding perf optimization tweaks to openapi (#1182) * chore: adding perf optimization tweaks to openapi * chore: slotsize integer --------- Co-authored-by: Adam Uhlíř --- openapi.yaml | 162 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 122 insertions(+), 40 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index 8bae1b10..6a8e8b76 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -27,10 +27,6 @@ components: maxLength: 66 example: 0x... - BigInt: - type: string - description: Integer represented as decimal string - Cid: type: string description: Content Identifier as specified at https://github.com/multiformats/cid @@ -55,17 +51,18 @@ components: description: The amount of tokens paid per byte per second per slot to hosts the client is willing to pay Duration: - type: string - description: The duration of the request in seconds as decimal string + type: integer + format: int64 + description: The duration of the request in seconds ProofProbability: type: string description: How often storage proofs are required as decimal string Expiry: - type: string + type: integer + format: int64 description: A timestamp as seconds since unix epoch at which this request expires if the Request does not find requested amount of nodes to host the data. - default: 10 minutes SPR: type: string @@ -73,6 +70,8 @@ components: SPRRead: type: object + required: + - spr properties: spr: $ref: "#/components/schemas/SPR" @@ -85,6 +84,8 @@ components: Content: type: object + required: + - cid description: Parameters specifying the content properties: cid: @@ -92,6 +93,12 @@ components: Node: type: object + required: + - nodeId + - peerId + - record + - address + - seen properties: nodeId: type: string @@ -116,6 +123,9 @@ components: PeersTable: type: object + required: + - localNode + - nodes properties: localNode: $ref: "#/components/schemas/Node" @@ -126,6 +136,14 @@ components: DebugInfo: type: object + required: + - id + - addrs + - repo + - spr + - announceAddresses + - table + - codex properties: id: $ref: "#/components/schemas/PeerId" @@ -149,12 +167,16 @@ components: SalesAvailability: type: object + required: + - totalSize + - duration + - minPricePerBytePerSecond + - totalCollateral properties: - id: - $ref: "#/components/schemas/Id" totalSize: - type: string - description: Total size of availability's storage in bytes as decimal string + type: integer + format: int64 + description: Total size of availability's storage in bytes duration: $ref: "#/components/schemas/Duration" minPricePerBytePerSecond: @@ -173,42 +195,53 @@ components: default: 0 SalesAvailabilityREAD: + required: + - id + - totalRemainingCollateral allOf: - $ref: "#/components/schemas/SalesAvailability" - type: object properties: + id: + $ref: "#/components/schemas/Id" + readonly: true freeSize: - type: string + type: integer + format: int64 description: Unused size of availability's storage in bytes as decimal string - - SalesAvailabilityCREATE: - allOf: - - $ref: "#/components/schemas/SalesAvailability" - - required: - - totalSize - - minPricePerBytePerSecond - - totalCollateral - - duration + readOnly: true + totalRemainingCollateral: + type: string + description: Total collateral effective (in amount of tokens) that can be used for matching requests + readOnly: true Slot: type: object + required: + - id + - request + - slotIndex properties: id: $ref: "#/components/schemas/SlotId" request: $ref: "#/components/schemas/StorageRequest" slotIndex: - type: string - description: Slot Index as decimal string + type: integer + format: int64 + description: Slot Index number SlotAgent: type: object + required: + - state + - requestId + - slotIndex properties: - id: - $ref: "#/components/schemas/SlotId" slotIndex: - type: string - description: Slot Index as decimal string + type: integer + format: int64 + description: Slot Index number requestId: $ref: "#/components/schemas/Id" request: @@ -235,18 +268,28 @@ components: Reservation: type: object + required: + - id + - availabilityId + - size + - requestId + - slotIndex + - validUntil properties: id: $ref: "#/components/schemas/Id" availabilityId: $ref: "#/components/schemas/Id" size: - $ref: "#/components/schemas/BigInt" + type: integer + format: int64 + description: Size of the slot in bytes requestId: $ref: "#/components/schemas/Id" slotIndex: - type: string - description: Slot Index as decimal string + type: integer + format: int64 + description: Slot Index number validUntil: type: integer description: Timestamp after which the reservation will no longer be valid. @@ -269,28 +312,39 @@ components: nodes: description: Minimal number of nodes the content should be stored on type: integer - default: 1 + default: 3 + minimum: 3 tolerance: description: Additional number of nodes on top of the `nodes` property that can be lost before pronouncing the content lost type: integer - default: 0 + default: 1 + minimum: 1 collateralPerByte: type: string description: Number as decimal string that represents how much collateral per byte is asked from hosts that wants to fill a slots expiry: - type: string - description: Number as decimal string that represents expiry threshold in seconds from when the Request is submitted. When the threshold is reached and the Request does not find requested amount of nodes to host the data, the Request is voided. The number of seconds can not be higher then the Request's duration itself. + type: integer + format: int64 + description: Number that represents expiry threshold in seconds from when the Request is submitted. When the threshold is reached and the Request does not find requested amount of nodes to host the data, the Request is voided. The number of seconds can not be higher then the Request's duration itself. StorageAsk: type: object required: + - slots + - slotSize + - duration + - proofProbability - pricePerBytePerSecond + - collateralPerByte + - maxSlotLoss properties: slots: description: Number of slots (eq. hosts) that the Request want to have the content spread over type: integer + format: int64 slotSize: - type: string - description: Amount of storage per slot (in bytes) as decimal string + type: integer + format: int64 + description: Amount of storage per slot in bytes duration: $ref: "#/components/schemas/Duration" proofProbability: @@ -299,10 +353,18 @@ components: $ref: "#/components/schemas/PricePerBytePerSecond" maxSlotLoss: type: integer + format: int64 description: Max slots that can be lost without data considered to be lost StorageRequest: type: object + required: + - id + - client + - ask + - content + - expiry + - nonce properties: id: type: string @@ -321,6 +383,9 @@ components: Purchase: type: object + required: + - state + - requestId properties: state: type: string @@ -340,9 +405,13 @@ components: description: If Request failed, then here is presented the error message request: $ref: "#/components/schemas/StorageRequest" + requestId: + $ref: "#/components/schemas/Id" DataList: type: object + required: + - content properties: content: type: array @@ -351,6 +420,9 @@ components: DataItem: type: object + required: + - cid + - manifest properties: cid: $ref: "#/components/schemas/Cid" @@ -359,6 +431,11 @@ components: ManifestItem: type: object + required: + - treeCid + - datasetSize + - blockSize + - protected properties: treeCid: $ref: "#/components/schemas/Cid" @@ -386,6 +463,11 @@ components: Space: type: object + required: + - totalBlocks + - quotaMaxBytes + - quotaUsedBytes + - quotaReservedBytes properties: totalBlocks: description: "Number of blocks stored by the node" @@ -704,7 +786,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/SalesAvailabilityCREATE" + $ref: "#/components/schemas/SalesAvailability" responses: "201": description: Created storage availability @@ -870,7 +952,7 @@ paths: "200": description: Node's SPR content: - plain/text: + text/plain: schema: $ref: "#/components/schemas/SPR" application/json: @@ -888,7 +970,7 @@ paths: "200": description: Node's Peer ID content: - plain/text: + text/plain: schema: $ref: "#/components/schemas/PeerId" application/json: From 6f62afef7520060d385bd22aea97ee51533fe536 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Fri, 4 Apr 2025 14:58:23 +0200 Subject: [PATCH 15/24] Apply changes to the openapi file (#1187) --- openapi.yaml | 13 ++++++++++--- redocly.yaml | 7 +++++++ 2 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 redocly.yaml diff --git a/openapi.yaml b/openapi.yaml index 6a8e8b76..551e2fe2 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -50,6 +50,10 @@ components: type: string description: The amount of tokens paid per byte per second per slot to hosts the client is willing to pay + CollateralPerByte: + type: string + description: Number as decimal string that represents how much collateral per byte is asked from hosts that wants to fill a slots + Duration: type: integer format: int64 @@ -320,8 +324,7 @@ components: default: 1 minimum: 1 collateralPerByte: - type: string - description: Number as decimal string that represents how much collateral per byte is asked from hosts that wants to fill a slots + $ref: "#/components/schemas/CollateralPerByte" expiry: type: integer format: int64 @@ -351,6 +354,8 @@ components: $ref: "#/components/schemas/ProofProbability" pricePerBytePerSecond: $ref: "#/components/schemas/PricePerBytePerSecond" + collateralPerByte: + $ref: "#/components/schemas/CollateralPerByte" maxSlotLoss: type: integer format: int64 @@ -392,7 +397,7 @@ components: description: Description of the Request's state enum: - cancelled - - error + - errored - failed - finished - pending @@ -586,6 +591,8 @@ paths: text/plain: schema: type: string + "422": + description: The mimetype of the filename is invalid "500": description: Well it was bad-bad and the upload did not work out diff --git a/redocly.yaml b/redocly.yaml new file mode 100644 index 00000000..78fa9e60 --- /dev/null +++ b/redocly.yaml @@ -0,0 +1,7 @@ +extends: + - recommended + +rules: + info-license: off + no-required-schema-properties-undefined: error + no-server-example.com: off \ No newline at end of file From b92f79a6543cae2b074bad363aa91add20d0b802 Mon Sep 17 00:00:00 2001 From: markspanbroek Date: Tue, 15 Apr 2025 12:31:06 +0200 Subject: [PATCH 16/24] Increase gas estimates (#1192) * update nim-ethers to version 2.0.0 To allow for gas estimation of contract calls * contracts: add 10% extra gas to contract calls These calls could otherwise run out of gas because the on-chain state may have changed between the time of the estimate and the time of processing the transaction. --- codex/contracts/config.nim | 2 +- codex/contracts/market.nim | 54 +++++++++++++++++++++++++++--------- codex/contracts/proofs.nim | 2 +- codex/contracts/requests.nim | 3 +- vendor/nim-ethers | 2 +- 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/codex/contracts/config.nim b/codex/contracts/config.nim index 3c31c8b5..83b39c0a 100644 --- a/codex/contracts/config.nim +++ b/codex/contracts/config.nim @@ -1,5 +1,5 @@ import pkg/contractabi -import pkg/ethers/fields +import pkg/ethers/contracts/fields import pkg/questionable/results export contractabi diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 8b235876..9d0799f9 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -257,8 +257,17 @@ method fillSlot( try: await market.approveFunds(collateral) + + # Add 10% to gas estimate to deal with different evm code flow when we + # happen to be the last one to fill a slot in this request + trace "estimating gas for fillSlot" + let gas = await market.contract.estimateGas.fillSlot(requestId, slotIndex, proof) + let overrides = TransactionOverrides(gasLimit: some (gas * 110) div 100) + trace "calling fillSlot on contract" - discard await market.contract.fillSlot(requestId, slotIndex, proof).confirm(1) + discard await market.contract + .fillSlot(requestId, slotIndex, proof, overrides) + .confirm(1) trace "fillSlot transaction completed" except Marketplace_SlotNotFree as parent: raise newException( @@ -276,15 +285,30 @@ method freeSlot*( # If --reward-recipient specified, use it as the reward recipient, and use # the SP's address as the collateral recipient let collateralRecipient = await market.getSigner() + + # Add 10% to gas estimate to deal with different evm code flow when we + # happen to be the one to make the request fail + let gas = await market.contract.estimateGas.freeSlot( + slotId, rewardRecipient, collateralRecipient + ) + let overrides = TransactionOverrides(gasLimit: some (gas * 110) div 100) + freeSlot = market.contract.freeSlot( slotId, rewardRecipient, # --reward-recipient - collateralRecipient, - ) # SP's address + collateralRecipient, # SP's address + overrides, + ) else: # Otherwise, use the SP's address as both the reward and collateral # recipient (the contract will use msg.sender for both) - freeSlot = market.contract.freeSlot(slotId) + + # Add 10% to gas estimate to deal with different evm code flow when we + # happen to be the one to make the request fail + let gas = await market.contract.estimateGas.freeSlot(slotId) + let overrides = TransactionOverrides(gasLimit: some (gas * 110) div 100) + + freeSlot = market.contract.freeSlot(slotId, overrides) discard await freeSlot.confirm(1) except Marketplace_SlotIsFree as parent: @@ -331,7 +355,12 @@ method markProofAsMissing*( market: OnChainMarket, id: SlotId, period: Period ) {.async: (raises: [CancelledError, MarketError]).} = convertEthersError("Failed to mark proof as missing"): - discard await market.contract.markProofAsMissing(id, period).confirm(1) + # Add 10% to gas estimate to deal with different evm code flow when we + # happen to be the one to make the request fail + let gas = await market.contract.estimateGas.markProofAsMissing(id, period) + let overrides = TransactionOverrides(gasLimit: some (gas * 110) div 100) + + discard await market.contract.markProofAsMissing(id, period, overrides).confirm(1) method canProofBeMarkedAsMissing*( market: OnChainMarket, id: SlotId, period: Period @@ -351,14 +380,13 @@ method reserveSlot*( ) {.async: (raises: [CancelledError, MarketError]).} = convertEthersError("Failed to reserve slot"): try: - discard await market.contract - .reserveSlot( - requestId, - slotIndex, - # reserveSlot runs out of gas for unknown reason, but 100k gas covers it - TransactionOverrides(gasLimit: some 100000.u256), - ) - .confirm(1) + # Add 10% to gas estimate to deal with different evm code flow when we + # happen to be the last one that is allowed to reserve the slot + let gas = await market.contract.estimateGas.reserveSlot(requestId, slotIndex) + let overrides = TransactionOverrides(gasLimit: some (gas * 110) div 100) + + discard + await market.contract.reserveSlot(requestId, slotIndex, overrides).confirm(1) except SlotReservations_ReservationNotAllowed: raise newException( SlotReservationNotAllowedError, diff --git a/codex/contracts/proofs.nim b/codex/contracts/proofs.nim index 771d685b..c0d80b7d 100644 --- a/codex/contracts/proofs.nim +++ b/codex/contracts/proofs.nim @@ -1,6 +1,6 @@ import pkg/stint import pkg/contractabi -import pkg/ethers/fields +import pkg/ethers/contracts/fields type Groth16Proof* = object diff --git a/codex/contracts/requests.nim b/codex/contracts/requests.nim index 2b3811c3..035e9648 100644 --- a/codex/contracts/requests.nim +++ b/codex/contracts/requests.nim @@ -3,13 +3,12 @@ import std/sequtils import std/typetraits import pkg/contractabi import pkg/nimcrypto -import pkg/ethers/fields +import pkg/ethers/contracts/fields import pkg/questionable/results import pkg/stew/byteutils import pkg/libp2p/[cid, multicodec] import ../logutils import ../utils/json -import ../clock from ../errors import mapFailure export contractabi diff --git a/vendor/nim-ethers b/vendor/nim-ethers index 5d07b5db..bbced467 160000 --- a/vendor/nim-ethers +++ b/vendor/nim-ethers @@ -1 +1 @@ -Subproject commit 5d07b5dbcf584b020c732e84cc8b7229ab3e1083 +Subproject commit bbced4673316763c6ef931b4d0a08069cde2474c From 7c7871ac7502199cd03848b4aed60b6e29325f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Uhl=C3=AD=C5=99?= Date: Tue, 15 Apr 2025 15:52:19 +0200 Subject: [PATCH 17/24] ci: add compatible contracts image for nim-codex docker image (#1186) * ci: add compatible contracts image for nim-codex docker image * ci: with submodules * ci: with submodules on correct place * ci: remove double dash * ci: avoiding artifact conflicts * ci: add labels to arch images * ci: correct way to add label to arch images * ci: correct contract label * ci: avoid building contracts image and use contracts commit hash * refactor: better way to get the hash --- .github/workflows/docker-reusable.yml | 36 ++++++++++++++++++++++++--- .github/workflows/docker.yml | 18 +++++++++++++- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-reusable.yml b/.github/workflows/docker-reusable.yml index 7d937f78..2a614316 100644 --- a/.github/workflows/docker-reusable.yml +++ b/.github/workflows/docker-reusable.yml @@ -59,6 +59,10 @@ on: required: false type: string default: false + contract_image: + description: Specifies compatible smart contract image + required: false + type: string env: @@ -71,6 +75,7 @@ env: TAG_LATEST: ${{ inputs.tag_latest }} TAG_SHA: ${{ inputs.tag_sha }} TAG_SUFFIX: ${{ inputs.tag_suffix }} + CONTRACT_IMAGE: ${{ inputs.contract_image }} # Tests TESTS_SOURCE: codex-storage/cs-codex-dist-tests TESTS_BRANCH: master @@ -80,8 +85,19 @@ env: jobs: + compute: + name: Compute build ID + runs-on: ubuntu-latest + outputs: + build_id: ${{ steps.build_id.outputs.build_id }} + steps: + - name: Generate unique build id + id: build_id + run: echo "build_id=$(openssl rand -hex 5)" >> $GITHUB_OUTPUT + # Build platform specific image build: + needs: compute strategy: fail-fast: true matrix: @@ -108,11 +124,19 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Docker - Variables + run: | + # Create contract label for compatible contract image if specified + if [[ -n "${{ env.CONTRACT_IMAGE }}" ]]; then + echo "CONTRACT_LABEL=storage.codex.nim-codex.blockchain-image=${{ env.CONTRACT_IMAGE }}" >>$GITHUB_ENV + fi + - name: Docker - Meta id: meta uses: docker/metadata-action@v5 with: images: ${{ env.DOCKER_REPO }} + labels: ${{ env.CONTRACT_LABEL }} - name: Docker - Set up Buildx uses: docker/setup-buildx-action@v3 @@ -147,7 +171,7 @@ jobs: - name: Docker - Upload digest uses: actions/upload-artifact@v4 with: - name: digests-${{ matrix.target.arch }} + name: digests-${{ needs.compute.outputs.build_id }}-${{ matrix.target.arch }} path: /tmp/digests/* if-no-files-found: error retention-days: 1 @@ -159,7 +183,7 @@ jobs: runs-on: ubuntu-latest outputs: version: ${{ steps.meta.outputs.version }} - needs: build + needs: [build, compute] steps: - name: Docker - Variables run: | @@ -183,11 +207,16 @@ jobs: else echo "TAG_RAW=false" >>$GITHUB_ENV fi + + # Create contract label for compatible contract image if specified + if [[ -n "${{ env.CONTRACT_IMAGE }}" ]]; then + echo "CONTRACT_LABEL=storage.codex.nim-codex.blockchain-image=${{ env.CONTRACT_IMAGE }}" >>$GITHUB_ENV + fi - name: Docker - Download digests uses: actions/download-artifact@v4 with: - pattern: digests-* + pattern: digests-${{ needs.compute.outputs.build_id }}-* merge-multiple: true path: /tmp/digests @@ -199,6 +228,7 @@ jobs: uses: docker/metadata-action@v5 with: images: ${{ env.DOCKER_REPO }} + labels: ${{ env.CONTRACT_LABEL }} flavor: | latest=${{ env.TAG_LATEST }} suffix=${{ env.TAG_SUFFIX }},onlatest=true diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fb97c339..1a1573bb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -20,9 +20,25 @@ on: jobs: + get-contracts-hash: + runs-on: ubuntu-latest + outputs: + hash: ${{ steps.get-hash.outputs.hash }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Get submodule short hash + id: get-hash + run: | + hash=$(git rev-parse --short HEAD:vendor/codex-contracts-eth) + echo "hash=$hash" >> $GITHUB_OUTPUT build-and-push: name: Build and Push uses: ./.github/workflows/docker-reusable.yml + needs: get-contracts-hash with: tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }} - secrets: inherit + contract_image: "codexstorage/codex-contracts-eth:sha-${{ needs.get-contracts-hash.outputs.hash }}" + secrets: inherit \ No newline at end of file From acf81d0571cbdf2c2d3bd976951313aade69b1ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Uhl=C3=AD=C5=99?= Date: Thu, 17 Apr 2025 06:05:53 +0200 Subject: [PATCH 18/24] chore: add marketplace topic to reservations (#1193) --- codex/sales/reservations.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index b717cc1c..07e3f406 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -56,7 +56,7 @@ export requests export logutils logScope: - topics = "sales reservations" + topics = "marketplace sales reservations" type AvailabilityId* = distinct array[32, byte] From 0f152d333c635d049df8b0596fc2d629a79ddc16 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 17 Apr 2025 16:13:43 +1000 Subject: [PATCH 19/24] chore: bump contracts to master (#1197) Bump contracts to master branch. There was a change that allowed hardhat to have multiple blocks with the same timestamp, so this needed to be reflected in two tests. --- tests/contracts/testMarket.nim | 4 ++-- vendor/codex-contracts-eth | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index 068a4d2e..1d8095c7 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -548,7 +548,7 @@ ethersuite "On-Chain Market": switchAccount(host) await market.reserveSlot(request.id, 0.uint64) await market.fillSlot(request.id, 0.uint64, proof, request.ask.collateralPerSlot) - let filledAt = (await ethProvider.currentTime()) - 1.u256 + let filledAt = (await ethProvider.currentTime()) for slotIndex in 1 ..< request.ask.slots: await market.reserveSlot(request.id, slotIndex.uint64) @@ -575,7 +575,7 @@ ethersuite "On-Chain Market": switchAccount(host) await market.reserveSlot(request.id, 0.uint64) await market.fillSlot(request.id, 0.uint64, proof, request.ask.collateralPerSlot) - let filledAt = (await ethProvider.currentTime()) - 1.u256 + let filledAt = (await ethProvider.currentTime()) for slotIndex in 1 ..< request.ask.slots: await market.reserveSlot(request.id, slotIndex.uint64) diff --git a/vendor/codex-contracts-eth b/vendor/codex-contracts-eth index c00152e6..0bf13851 160000 --- a/vendor/codex-contracts-eth +++ b/vendor/codex-contracts-eth @@ -1 +1 @@ -Subproject commit c00152e6213a3ad4e6760a670213bfae22b0aabf +Subproject commit 0bf138512b7c1c3b8d77c48376e47f702e47106c From 22f5150d1dbea8e49ae4c71b9daec60f8fbe536f Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Fri, 18 Apr 2025 17:21:24 +0300 Subject: [PATCH 20/24] ci: add compatible contracts image for nim-codex dist-tests docker image (#1204) --- .github/workflows/docker-dist-tests.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/docker-dist-tests.yml b/.github/workflows/docker-dist-tests.yml index 1cbc528a..c6c133f2 100644 --- a/.github/workflows/docker-dist-tests.yml +++ b/.github/workflows/docker-dist-tests.yml @@ -26,13 +26,29 @@ on: jobs: + get-contracts-hash: + runs-on: ubuntu-latest + outputs: + hash: ${{ steps.get-hash.outputs.hash }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Get submodule short hash + id: get-hash + run: | + hash=$(git rev-parse --short HEAD:vendor/codex-contracts-eth) + echo "hash=$hash" >> $GITHUB_OUTPUT build-and-push: name: Build and Push uses: ./.github/workflows/docker-reusable.yml + needs: get-contracts-hash with: nimflags: '-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_proof_failures=true -d:codex_enable_log_counter=true -d:verify_circuit=true' nat_ip_auto: true tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }} tag_suffix: dist-tests + contract_image: "codexstorage/codex-contracts-eth:sha-${{ needs.get-contracts-hash.outputs.hash }}-dist-tests" run_release_tests: ${{ inputs.run_release_tests }} secrets: inherit From 2eb83a0ebb4f80984baf65d41e6b68d90e9866de Mon Sep 17 00:00:00 2001 From: Ben Bierens <39762930+benbierens@users.noreply.github.com> Date: Tue, 22 Apr 2025 16:32:32 +0200 Subject: [PATCH 21/24] Codex-contracts hash in version information. (#1207) * Adds revision hash of codex-contracts to version information. * Update codex/conf.nim Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Ben Bierens <39762930+benbierens@users.noreply.github.com> * Update codex/conf.nim Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Ben Bierens <39762930+benbierens@users.noreply.github.com> * Update codex/rest/api.nim Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Ben Bierens <39762930+benbierens@users.noreply.github.com> * simplified git command * Remove space Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Giuliano Mega * Updates openapi.yaml --------- Signed-off-by: Ben Bierens <39762930+benbierens@users.noreply.github.com> Signed-off-by: Giuliano Mega Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Giuliano Mega --- codex/conf.nim | 7 ++++++- codex/rest/api.nim | 6 +++++- openapi.yaml | 3 +++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/codex/conf.nim b/codex/conf.nim index 986a53d6..af55861f 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -477,17 +477,22 @@ proc getCodexRevision(): string = var res = strip(staticExec("git rev-parse --short HEAD")) return res +proc getCodexContractsRevision(): string = + let res = strip(staticExec("git rev-parse --short HEAD:vendor/codex-contracts-eth")) + return res + proc getNimBanner(): string = staticExec("nim --version | grep Version") const codexVersion* = getCodexVersion() codexRevision* = getCodexRevision() + codexContractsRevision* = getCodexContractsRevision() nimBanner* = getNimBanner() codexFullVersion* = "Codex version: " & codexVersion & "\p" & "Codex revision: " & codexRevision & "\p" & - nimBanner + "Codex contracts revision: " & codexContractsRevision & "\p" & nimBanner proc parseCmdArg*( T: typedesc[MultiAddress], input: string diff --git a/codex/rest/api.nim b/codex/rest/api.nim index ee493e03..7c7dcd34 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -914,7 +914,11 @@ proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) = "", "announceAddresses": node.discovery.announceAddrs, "table": table, - "codex": {"version": $codexVersion, "revision": $codexRevision}, + "codex": { + "version": $codexVersion, + "revision": $codexRevision, + "contracts": $codexContractsRevision, + }, } # return pretty json for human readability diff --git a/openapi.yaml b/openapi.yaml index 551e2fe2..0c3ca9fe 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -124,6 +124,9 @@ components: revision: type: string example: 0c647d8 + contracts: + type: string + example: 0b537c7 PeersTable: type: object From d220e53fe1535f8867d2cd27afaae0173cedd7d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Uhl=C3=AD=C5=99?= Date: Tue, 22 Apr 2025 16:46:03 +0200 Subject: [PATCH 22/24] ci: trigger python generator upon release (#1208) --- .github/workflows/release.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9d433257..7f154383 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -177,3 +177,12 @@ jobs: files: | /tmp/release/* make_latest: true + + - name: Generate Python SDK + uses: peter-evans/repository-dispatch@v3 + if: startsWith(github.ref, 'refs/tags/') + with: + token: ${{ secrets.DISPATCH_PAT }} + repository: codex-storage/py-codex-api-client + event-type: generate + client-payload: '{"openapi_url": "https://raw.githubusercontent.com/codex-storage/nim-codex/${{ github.ref }}/openapi.yaml"}' From b39d541227ed0c623571c497c6c3eaabfc176b25 Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Wed, 23 Apr 2025 09:18:38 +0300 Subject: [PATCH 23/24] chore: update testnet marketplace address (#1209) https://github.com/codex-storage/nim-codex/issues/1203 --- codex/contracts/deployment.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codex/contracts/deployment.nim b/codex/contracts/deployment.nim index cc125d18..37bb8ea1 100644 --- a/codex/contracts/deployment.nim +++ b/codex/contracts/deployment.nim @@ -18,9 +18,9 @@ const knownAddresses = { # Taiko Alpha-3 Testnet "167005": {"Marketplace": Address.init("0x948CF9291b77Bd7ad84781b9047129Addf1b894F")}.toTable, - # Codex Testnet - Feb 25 2025 07:24:19 AM (+00:00 UTC) + # Codex Testnet - Apr 22 2025 12:42:16 PM (+00:00 UTC) "789987": - {"Marketplace": Address.init("0xfFaF679D5Cbfdd5Dbc9Be61C616ed115DFb597ed")}.toTable, + {"Marketplace": Address.init("0xDB2908d724a15d05c0B6B8e8441a8b36E67476d3")}.toTable, }.toTable proc getKnownAddress(T: type, chainId: UInt256): ?Address = From 19a5e05c1397a26e4d9b02264bdf560829f3d8e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Uhl=C3=AD=C5=99?= Date: Sat, 3 May 2025 18:54:38 +0200 Subject: [PATCH 24/24] docs(openapi): add local data delete endpoint (#1214) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs(openapi): add local data delete endpoint * chore: feedback Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com> Signed-off-by: Adam Uhlíř --------- Signed-off-by: Adam Uhlíř Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com> --- codex/rest/api.nim | 2 +- openapi.yaml | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 7c7dcd34..0d9e5d80 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -288,7 +288,7 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute cid: Cid, resp: HttpResponseRef ) -> RestApiResponse: ## Deletes either a single block or an entire dataset - ## from the local node. Does nothing and returns 200 + ## from the local node. Does nothing and returns 204 ## if the dataset is not locally available. ## var headers = buildCorsHeaders("DELETE", allowedOrigin) diff --git a/openapi.yaml b/openapi.yaml index 0c3ca9fe..23c5ead8 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -627,6 +627,26 @@ paths: "500": description: Well it was bad-bad + delete: + summary: "Deletes either a single block or an entire dataset from the local node." + tags: [Data] + operationId: deleteLocal + parameters: + - in: path + name: cid + required: true + schema: + $ref: "#/components/schemas/Cid" + description: Block or dataset to be deleted. + + responses: + "204": + description: Data was successfully deleted. + "400": + description: Invalid CID is specified + "500": + description: There was an error during deletion + "/data/{cid}/network": post: summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."